From 9d1a261bf626502a0ac3ee2406a0e7d688c41070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 11:42:13 +0800 Subject: [PATCH 001/398] feat: + education industry --- .gitignore | 3 + metagpt/actions/write_teaching_plan.py | 134 +++++++++++++++++++++++++ metagpt/provider/openai_api.py | 20 ++-- metagpt/roles/role.py | 66 +++++++++++- metagpt/roles/teacher.py | 96 ++++++++++++++++++ metagpt/software_company.py | 5 +- requirements.txt | 1 + startup.py | 96 ++++++++++++++++-- tests/metagpt/roles/test_teacher.py | 94 +++++++++++++++++ 9 files changed, 498 insertions(+), 17 deletions(-) create mode 100644 metagpt/actions/write_teaching_plan.py create mode 100644 metagpt/roles/teacher.py create mode 100644 tests/metagpt/roles/test_teacher.py diff --git a/.gitignore b/.gitignore index c4c79c733..3ec71f8b6 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,6 @@ workspace/* *.mmd tmp output.wav + +# output folder +output diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py new file mode 100644 index 000000000..1f0167df3 --- /dev/null +++ b/metagpt/actions/write_teaching_plan.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/11 14:43 +@Author : mashenquan +@File : write_teaching_plan.py +""" +from langchain.llms.base import LLM +from metagpt.logs import logger +from metagpt.actions import Action +from metagpt.schema import Message + + +class TeachingPlanRequirement(Action): + """Teaching Plan Requirement without any implementation details""" + + async def run(self, *args, **kwargs): + raise NotImplementedError + + +class WriteTeachingPlanPart(Action): + """Write Teaching Plan Part""" + + def __init__(self, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): + """ + + Args: + name: action name + context: context + llm: object of :class:`LLM` + topic: topic part of teaching plan + """ + super().__init__(name, context, llm) + self.topic = topic + self.language = language + self.rsp = None + + async def run(self, *args, **kwargs): + if len(args) < 1 or len(args[0]) < 1 or not isinstance(args[0][0], Message): + raise ValueError("Invalid args, a tuple of List[Message] is expected") + + statements = self.TOPIC_STATEMENTS.get(self.topic, []) + formatter = self.PROMPT_TITLE_TEMPLATE if self.topic == self.COURSE_TITLE else self.PROMPT_TEMPLATE + prompt = formatter.format(formation=self.FORMATION, + role=self.prefix, + statements="\n".join(statements), + lesson=args[0][0].content, + topic=self.topic, + language=self.language) + + logger.debug(prompt) + rsp = await self._aask(prompt=prompt) + logger.debug(rsp) + self._set_result(rsp) + return self.rsp + + def _set_result(self, rsp): + if self.DATA_BEGIN_TAG in rsp: + ix = rsp.index(self.DATA_BEGIN_TAG) + rsp = rsp[ix + len(self.DATA_BEGIN_TAG):] + if self.DATA_END_TAG in rsp: + ix = rsp.index(self.DATA_END_TAG) + rsp = rsp[0:ix] + self.rsp = rsp.strip() + if self.topic != self.COURSE_TITLE: + return + if '#' not in self.rsp or self.rsp.index('#') != 0: + self.rsp = "# " + self.rsp + + def __str__(self): + """str()时返回`topic`""" + return self.topic + + def __repr__(self): + """调试时返回`topic`""" + return self.topic + + FORMATION = """ + "\tCapacity and role" defines the role you are currently playing; + "\t[LESSON_BEGIN]" and "[LESSON_END]" tags enclose the content of textbook; + "\tStatement" defines the work detail you need to complete at this stage; + "\tAnswer options" defines the format requirements for your responses; + "\tConstraint" defines the conditions that your responses must comply with. + """ + COURSE_TITLE = "Title" + TOPICS = [COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", + "Teaching Methods and Strategies", "Learning Activities", + "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement"] + + TOPIC_STATEMENTS = { + COURSE_TITLE: ["Statement: Find and return the title of the lesson only in markdown first-level header format, " + "without anything else."], + "Teaching Content": [ + "Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar " + "structures that appear in the textbook, as well as the listening materials and key points.", + "Statement: \"Teaching Content\" must include more examples."], + "Teaching Time Allocation": [ + "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each " + "part of the textbook content."], + "Teaching Methods and Strategies": [ + "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, " + "procedures, in detail." + ] + } + + # Teaching plan title + PROMPT_TITLE_TEMPLATE = """ + Do not refer to the context of the previous conversation records, start the conversation anew.\n\n + Formation: {formation}\n\n + {statements}\n + Constraint: Writing in {language}.\n + Answer options: Encloses the lesson title with "[TEACHING_PLAN_BEGIN]" and "[TEACHING_PLAN_END]" tags.\n + [LESSON_BEGIN]\n + {lesson}\n + [LESSON_END] + """ + + # Teaching plan parts: + PROMPT_TEMPLATE = """ + Do not refer to the context of the previous conversation records, start the conversation anew.\n\n + Formation: {formation}\n\n + Capacity and role: {role}\n + Statement: Write the "{topic}" part of teaching plan, WITHOUT ANY content unrelated to "{topic}"!!\n + {statements}\n + Answer options: Enclose the teaching plan content with "[TEACHING_PLAN_BEGIN]" and "[TEACHING_PLAN_END]" tags.\n + Answer options: Using proper markdown format from second-level header format.\n + Constraint: Writing in {language}.\n + [LESSON_BEGIN]\n + {lesson}\n + [LESSON_END] + """ + + DATA_BEGIN_TAG = "[TEACHING_PLAN_BEGIN]" + DATA_END_TAG = "[TEACHING_PLAN_END]" diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index f6499c643..ba5a655d3 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -4,12 +4,13 @@ @Time : 2023/5/5 23:08 @Author : alexanderwu @File : openai.py +@Modified By: mashenquan, 2023-07-27, + try except. """ import asyncio import time from functools import wraps from typing import NamedTuple - +import traceback import openai from metagpt.config import CONFIG @@ -30,7 +31,9 @@ def retry(max_retries): for i in range(max_retries): try: return await f(*args, **kwargs) - except Exception: + except Exception as e: + error_str = traceback.format_exc() + logger.warning(f"Exception occurred: {str(e)}, stack:{error_str}. Retrying...") if i == max_retries - 1: raise await asyncio.sleep(2 ** i) @@ -148,10 +151,15 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: - response = await openai.ChatCompletion.acreate( - **self._cons_kwargs(messages), - stream=True - ) + try: + response = await openai.ChatCompletion.acreate( + **self._cons_kwargs(messages), + stream=True + ) + except Exception as e: + error_str = traceback.format_exc() + logger.error(f"Exception:{e}, stack:{error_str}") + raise e # create variables to collect the stream of chunks collected_chunks = [] diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1681586cc..3e18257ed 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,9 +4,11 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py +@Modified By: mashenquan, 2023-07-27, :class:`Role` + properties. """ from __future__ import annotations +import traceback from typing import Iterable, Type from pydantic import BaseModel, Field @@ -92,13 +94,22 @@ class RoleContext(BaseModel): class Role: """角色/代理""" - def __init__(self, name="", profile="", goal="", constraints="", desc=""): + def __init__(self, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): + # Enable parameter configurability + name = Role.format_value(name, kwargs) + profile = Role.format_value(profile, kwargs) + goal = Role.format_value(goal, kwargs) + constraints = Role.format_value(constraints, kwargs) + desc = Role.format_value(desc, kwargs) + + # Initialize self._llm = LLM() self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) self._states = [] self._actions = [] self._role_id = str(self._setting) self._rc = RoleContext() + self._options = Role.supply_options(kwargs) def _reset(self): self._states = [] @@ -136,6 +147,26 @@ class Role: """获取角色描述（职位）""" return self._setting.profile + @property + def name(self): + """Return role `name`, read only""" + return self._setting.name + + @property + def desc(self): + """Return role `desc`, read only""" + return self._setting.desc + + @property + def goal(self): + """Return role `goal`, read only""" + return self._setting.goal + + @property + def constraints(self): + """Return role `constraints`, read only""" + return self._setting.constraints + def _get_prefix(self): """获取角色前缀""" if self._setting.desc: @@ -164,7 +195,8 @@ class Role: # history=self.history) logger.info(f"{self._setting}: ready to {self._rc.todo}") - response = await self._rc.todo.run(self._rc.important_memory) + requirement = self._rc.important_memory + response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): msg = Message(content=response.content, instruct_content=response.instruct_content, @@ -238,3 +270,33 @@ class Role: # 将回复发布到环境，等待下一个订阅者处理 self._publish_message(rsp) return rsp + + @staticmethod + def supply_options(options): + """Supply missing options""" + ret = Role.__DEFAULT_OPTIONS__.copy() + if not options: + return ret + ret.update(options) + return ret + + @staticmethod + def format_value(value, options): + """Fill parameters inside `value` with `options`. + """ + if "{" not in value: + return value + + options = Role.supply_options(options) + try: + return value.format(**options) + except KeyError as e: + logger.warning(f"Parameter is missing:{e}") + for k, v in options.items(): + value = value.replace("{" + f"{k}" + "}", v) + return value + + __DEFAULT_OPTIONS__ = { + "teaching_language": "English", + "language": "Chinese" + } \ No newline at end of file diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py new file mode 100644 index 000000000..a007926be --- /dev/null +++ b/metagpt/roles/teacher.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/23 17:25 +@Author : mashenquan +@File : teacher.py +""" +from pathlib import Path + +import aiofiles + +from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart, TeachingPlanRequirement +from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.logs import logger +import re + + +class Teacher(Role): + """Support configurable teacher roles, + with native and teaching languages being replaceable through configurations.""" + def __init__(self, name='Lily', profile='{teaching_language} Teacher', + goal='writing a {language} teaching plan part by part', + constraints='writing in {language}', desc="", *args, **kwargs): + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) + actions = [] + for topic in WriteTeachingPlanPart.TOPICS: + act = WriteTeachingPlanPart(topic=topic, llm=self._llm) + actions.append(act) + self._init_actions(actions) + self._watch({TeachingPlanRequirement}) + + async def _think(self) -> None: + """Everything will be done part by part.""" + if self._rc.todo is None: + self._set_state(0) + return + + if self._rc.state + 1 < len(self._states): + self._set_state(self._rc.state + 1) + else: + self._rc.todo = None + + async def _react(self) -> Message: + ret = Message(content="") + while True: + await self._think() + if self._rc.todo is None: + break + logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") + msg = await self._act() + if ret.content != '': + ret.content += "\n\n\n" + ret.content += msg.content + logger.info(ret.content) + await self.save(ret.content) + return ret + + async def save(self, content): + """Save teaching plan""" + filename = Teacher.new_file_name(self.course_title) + pathname = Path(__file__).resolve().parent.parent.parent / "output" + pathname.mkdir(exist_ok=True) + pathname = pathname / filename + try: + async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer: + await writer.write(content) + except Exception as e: + logger.error(f'Save failed：{e}') + logger.info(f"Save to:{pathname}") + + @staticmethod + def new_file_name(lesson_title, ext=".md"): + """Create a related file name based on `lesson_title` and `ext`.""" + # 定义需要替换的特殊字符 + illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' + # 将特殊字符替换为下划线 + filename = re.sub(illegal_chars, '_', lesson_title) + ext + return re.sub(r'_+', '_', filename) + + @property + def course_title(self): + """Return course title of teaching plan""" + default_title = "teaching_plan" + for act in self._actions: + if act.topic != WriteTeachingPlanPart.COURSE_TITLE: + continue + if act.rsp is None: + return default_title + title = act.rsp.lstrip("# \n") + if '\n' in title: + ix = title.index('\n') + title = title[0: ix] + return title + + return default_title diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 8f173ebf3..10fb025d6 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,6 +4,7 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py +@Modified By: mashenquan, 2023-07-27, Add `role` & `cause_by` parameters to `start_project()`. """ from pydantic import BaseModel, Field @@ -42,10 +43,10 @@ class SoftwareCompany(BaseModel): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea): + def start_project(self, idea, role="BOSS", cause_by=BossRequirement): """Start a project from publishing boss requirement.""" self.idea = idea - self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) + self.environment.publish_message(Message(role=role, content=idea, cause_by=cause_by)) def _save(self): logger.info(self.json()) diff --git a/requirements.txt b/requirements.txt index 32a436962..4d5856c20 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,3 +35,4 @@ tqdm==4.64.0 anthropic==0.3.6 typing-inspect==0.8.0 typing_extensions==4.5.0 +aiofiles \ No newline at end of file diff --git a/startup.py b/startup.py index e062babb5..17f55fb0a 100644 --- a/startup.py +++ b/startup.py @@ -1,15 +1,23 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +""" +@Modified By: mashenquan, 2023-07-27, +industry concept +""" + import asyncio - +from pathlib import Path +import aiofiles import fire - +from metagpt.logs import logger +from metagpt.actions.write_teaching_plan import TeachingPlanRequirement from metagpt.roles import Architect, Engineer, ProductManager, ProjectManager +from metagpt.roles.teacher import Teacher from metagpt.software_company import SoftwareCompany -async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False): - """Run a startup. Be a boss.""" +async def software_startup(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): + """Run a startup. Be a boss in software industry.""" + idea = kwargs['idea'] # Your innovative idea, such as "Creating a snake game." company = SoftwareCompany() company.hire([ProductManager(), Architect(), @@ -20,16 +28,90 @@ async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_rev await company.run(n_round=n_round) -def main(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False): +async def education_startup(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): + """Run a startup. Be a teacher in education industry.""" + + demo_lesson = """ + UNIT 1 Making New Friends + TOPIC 1 Welcome to China! + Section A + + 1a Listen and number the following names. + Jane Mari Kangkang Michael + Look, listen and understand. Then practice the conversation. + Work in groups. Introduce yourself using + I ’m ... Then practice 1a + with your own hometown or the following places. + + 1b Listen and number the following names + Jane Michael Maria Kangkang + 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. + China the USA the UK Hong Kong Beijing + + 2a Look, listen and understand. Then practice the conversation + Hello! + Hello! + Hello! + Hello! Are you Maria? + No, I’m not. I’m Jane. + Oh, nice to meet you, Jane + Nice to meet you, too. + Hi, Maria! + Hi, Kangkang! + Welcome to China! + Thanks. + + 2b Work in groups. Make up a conversation with your own name and the + following structures. + A: Hello! / Good morning! / Hi! I’m ... Are you ... ? + B: ... + + 3a Listen, say and trace + Aa Bb Cc Dd Ee Ff Gg + + 3b Listen and number the following letters. Then circle the letters withthe same sound as Bb. + Aa Bb Cc Dd Ee Ff Gg + + 3c Match the big letters with the small ones. Then write them on the lines. + """ + + lesson = "" + lesson_file = kwargs.get('lesson_file') + if lesson_file is not None and Path(lesson_file).exists(): + async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: + lesson = await reader.read() + logger.info(f"Course content: {lesson}") + if not lesson: + logger.info("No course content provided, using the demo course.") + lesson = demo_lesson + + company = SoftwareCompany() + company.hire([Teacher(*args, **kwargs)]) + company.invest(investment) + company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) + await company.run(n_round=1) + + +def main(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): """ We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea, such as "Creating a snake game." :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. :param n_round: :param code_review: Whether to use code review. + :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` + :param kwargs: Parameters passed in format: `python your_script.py a--param1=value1 --param2=value2` :return: """ - asyncio.run(startup(idea, investment, n_round, code_review)) + industry = kwargs.get("industry", "software") + industries = { + "software": software_startup, + "education": education_startup, + } + startup = industries.get(industry) + if startup is None: + print(f"Available industries:{list(industries.keys())}") + return + asyncio.run(startup(investment, n_round, code_review, *args, **kwargs)) if __name__ == '__main__': diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py new file mode 100644 index 000000000..0dddff3ac --- /dev/null +++ b/tests/metagpt/roles/test_teacher.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/23 17:25 +@Author : mashenquan +@File : teacher.py +""" + +from typing import Dict, Optional +from pydantic import BaseModel + +from metagpt.roles.teacher import Teacher + + +def test_init(): + class Inputs(BaseModel): + name: str + profile: str + goal: str + constraints: str + desc: str + options: Optional[Dict] = None + expect_name: str + expect_profile: str + expect_goal: str + expect_constraints: str + expect_desc: str + + inputs = [ + { + "name": "Lily{language}", + "expect_name": "LilyCN", + "profile": "X {teaching_language}", + "expect_profile": "X EN", + "goal": "Do {something_big}, {language}", + "expect_goal": "Do sleep, CN", + "constraints": "Do in {key1}, {language}", + "expect_constraints": "Do in HaHa, CN", + "options": {"language": "CN", "key1": "HaHa", "something_big": "sleep", "teaching_language": "EN"}, + "desc": "aaa{language}", + "expect_desc": "aaaCN" + }, + { + "name": "Lily{language}", + "expect_name": "LilyChinese", + "profile": "X {teaching_language}", + "expect_profile": "X English", + "goal": "Do {something_big}, {language}", + "expect_goal": "Do {something_big}, Chinese", + "constraints": "Do in {key1}, {language}", + "expect_constraints": "Do in {key1}, Chinese", + "desc": "aaa{language}", + "expect_desc": "aaaChinese" + }, + ] + + for i in inputs: + seed = Inputs(**i) + teacher = Teacher(name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, + desc=seed.desc, options=seed.options) + assert teacher.name == seed.expect_name + assert teacher.desc == seed.expect_desc + assert teacher.profile == seed.expect_profile + assert teacher.goal == seed.expect_goal + assert teacher.constraints == seed.expect_constraints + assert teacher.course_title == "teaching_plan" + + +def test_new_file_name(): + class Inputs(BaseModel): + lesson_title: str + ext: str + expect: str + + inputs = [ + { + "lesson_title": "# @344\n12", + "ext": ".md", + "expect": "_344_12.md" + }, + { + "lesson_title": "1#@$%!*&\\/:*?\"<>|\n\t \'1", + "ext": ".cc", + "expect": "1_1.cc" + } + ] + for i in inputs: + seed = Inputs(**i) + result = Teacher.new_file_name(seed.lesson_title, seed.ext) + assert result == seed.expect + + +if __name__ == '__main__': + test_init() From 5725296b1cdf6f7df494811945e07e4fe797aeb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 14:18:21 +0800 Subject: [PATCH 002/398] fixbug: unit test --- requirements-test.txt | 38 +++++++++++++++++++++++++++++ tests/metagpt/roles/test_teacher.py | 7 +++--- 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 requirements-test.txt diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 000000000..4d5856c20 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,38 @@ +aiohttp==3.8.4 +#azure_storage==0.37.0 +channels==4.0.0 +# chromadb==0.3.22 +# Django==4.1.5 +# docx==0.2.4 +duckduckgo_search==2.9.4 +#faiss==1.5.3 +faiss_cpu==1.7.4 +fire==0.4.0 +# godot==0.1.1 +# google_api_python_client==2.93.0 +langchain==0.0.231 +loguru==0.6.0 +meilisearch==0.21.0 +numpy==1.24.3 +openai==0.27.8 +openpyxl +pandas==1.4.1 +pydantic==1.10.7 +#pygame==2.1.3 +#pymilvus==2.2.8 +pytest==7.2.2 +python_docx==0.8.11 +PyYAML==6.0 +# sentence_transformers==2.2.2 +setuptools==65.6.3 +tenacity==8.2.2 +tiktoken==0.3.3 +tqdm==4.64.0 +#unstructured[local-inference] +# playwright +# selenium>4 +# webdriver_manager<3.9 +anthropic==0.3.6 +typing-inspect==0.8.0 +typing_extensions==4.5.0 +aiofiles \ No newline at end of file diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 0dddff3ac..10789f868 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -19,7 +19,7 @@ def test_init(): goal: str constraints: str desc: str - options: Optional[Dict] = None + kwargs: Optional[Dict] = None expect_name: str expect_profile: str expect_goal: str @@ -36,7 +36,7 @@ def test_init(): "expect_goal": "Do sleep, CN", "constraints": "Do in {key1}, {language}", "expect_constraints": "Do in HaHa, CN", - "options": {"language": "CN", "key1": "HaHa", "something_big": "sleep", "teaching_language": "EN"}, + "kwargs": {"language": "CN", "key1": "HaHa", "something_big": "sleep", "teaching_language": "EN"}, "desc": "aaa{language}", "expect_desc": "aaaCN" }, @@ -49,6 +49,7 @@ def test_init(): "expect_goal": "Do {something_big}, Chinese", "constraints": "Do in {key1}, {language}", "expect_constraints": "Do in {key1}, Chinese", + "kwargs": {}, "desc": "aaa{language}", "expect_desc": "aaaChinese" }, @@ -57,7 +58,7 @@ def test_init(): for i in inputs: seed = Inputs(**i) teacher = Teacher(name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, - desc=seed.desc, options=seed.options) + desc=seed.desc, **seed.kwargs) assert teacher.name == seed.expect_name assert teacher.desc == seed.expect_desc assert teacher.profile == seed.expect_profile From 686ca2347817ee913d8e53f51846a4d988cdb899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 15:33:52 +0800 Subject: [PATCH 003/398] fixbug: startup parameters do not match --- requirements-test.txt | 17 ++++++++++------- startup.py | 15 +++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 4d5856c20..7c03dddd9 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,7 +1,7 @@ aiohttp==3.8.4 -#azure_storage==0.37.0 +azure-cognitiveservices-speech==1.30.0 channels==4.0.0 -# chromadb==0.3.22 +chromadb==0.3.22 # Django==4.1.5 # docx==0.2.4 duckduckgo_search==2.9.4 @@ -19,7 +19,7 @@ openpyxl pandas==1.4.1 pydantic==1.10.7 #pygame==2.1.3 -#pymilvus==2.2.8 +pymilvus==2.2.8 pytest==7.2.2 python_docx==0.8.11 PyYAML==6.0 @@ -29,10 +29,13 @@ tenacity==8.2.2 tiktoken==0.3.3 tqdm==4.64.0 #unstructured[local-inference] -# playwright -# selenium>4 -# webdriver_manager<3.9 +playwright +selenium>4 +webdriver_manager<3.9 anthropic==0.3.6 typing-inspect==0.8.0 typing_extensions==4.5.0 -aiofiles \ No newline at end of file +bs4 +aiofiles +pytest +pytest-asyncio \ No newline at end of file diff --git a/startup.py b/startup.py index 17f55fb0a..c05bbbbf0 100644 --- a/startup.py +++ b/startup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Modified By: mashenquan, 2023-07-27, +industry concept +@Modified By: mashenquan, 2023-07-27, + `industry` concept """ import asyncio @@ -15,9 +15,9 @@ from metagpt.roles.teacher import Teacher from metagpt.software_company import SoftwareCompany -async def software_startup(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): +async def software_startup(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): """Run a startup. Be a boss in software industry.""" - idea = kwargs['idea'] # Your innovative idea, such as "Creating a snake game." + code_review = kwargs.get("code_review", False) # Whether to use code review. company = SoftwareCompany() company.hire([ProductManager(), Architect(), @@ -28,7 +28,7 @@ async def software_startup(investment: float = 3.0, n_round: int = 5, code_revie await company.run(n_round=n_round) -async def education_startup(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): +async def education_startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): """Run a startup. Be a teacher in education industry.""" demo_lesson = """ @@ -76,7 +76,6 @@ async def education_startup(investment: float = 3.0, n_round: int = 5, code_revi """ lesson = "" - lesson_file = kwargs.get('lesson_file') if lesson_file is not None and Path(lesson_file).exists(): async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: lesson = await reader.read() @@ -92,12 +91,12 @@ async def education_startup(investment: float = 3.0, n_round: int = 5, code_revi await company.run(n_round=1) -def main(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): +def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): """ We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + :param idea: Your innovative idea for `software` industry, such as "Creating a snake game."; lesson filename for `education` industry. :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. :param n_round: - :param code_review: Whether to use code review. :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` :param kwargs: Parameters passed in format: `python your_script.py a--param1=value1 --param2=value2` :return: @@ -111,7 +110,7 @@ def main(investment: float = 3.0, n_round: int = 5, code_review: bool = False, * if startup is None: print(f"Available industries:{list(industries.keys())}") return - asyncio.run(startup(investment, n_round, code_review, *args, **kwargs)) + asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) if __name__ == '__main__': From 255c56ca2658a3b2bcf76e8f68edc8b864a32572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:06:32 +0800 Subject: [PATCH 004/398] feat: +test --- .../actions/test_write_teaching_plan.py | 57 +++++++++++++++++++ tests/metagpt/roles/test_teacher.py | 1 + 2 files changed, 58 insertions(+) create mode 100644 tests/metagpt/actions/test_write_teaching_plan.py diff --git a/tests/metagpt/actions/test_write_teaching_plan.py b/tests/metagpt/actions/test_write_teaching_plan.py new file mode 100644 index 000000000..b47d6ab56 --- /dev/null +++ b/tests/metagpt/actions/test_write_teaching_plan.py @@ -0,0 +1,57 @@ +import asyncio +from typing import Optional +from pydantic import BaseModel +from langchain.llms.base import LLM + +from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart +from metagpt.schema import Message + + +class MockWriteTeachingPlanPart(WriteTeachingPlanPart): + def __init__(self, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): + super().__init__(name, context, llm, topic, language) + + async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> str: + return f"{WriteTeachingPlanPart.DATA_BEGIN_TAG}\nprompt\n{WriteTeachingPlanPart.DATA_END_TAG}" + + +async def mock_write_teaching_plan_part(): + class Inputs(BaseModel): + input: str + name: str + topic: str + language: str + + inputs = [ + { + "input": "AABBCC", + "name": "A", + "topic": "B", + "language": "C" + }, + { + "input": "DDEEFFF", + "name": "A1", + "topic": "B1", + "language": "C1" + } + ] + + for i in inputs: + seed = Inputs(**i) + act = MockWriteTeachingPlanPart(name=seed.name, topic=seed.topic, language=seed.language) + await act.run([Message(content="")]) + assert act.topic == seed.topic + assert str(act) == seed.topic + assert act.name == seed.name + assert act.rsp == "prompt" + + +def test_suite(): + loop = asyncio.get_event_loop() + task = loop.create_task(mock_write_teaching_plan_part()) + loop.run_until_complete(task) + + +if __name__ == '__main__': + test_suite() diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 10789f868..3af053338 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -93,3 +93,4 @@ def test_new_file_name(): if __name__ == '__main__': test_init() + test_new_file_name() From 2e6f88d3e4ab556f79626bf91367de7cd945de72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:10:54 +0800 Subject: [PATCH 005/398] feat: + notation --- metagpt/actions/write_teaching_plan.py | 2 +- metagpt/roles/teacher.py | 2 +- tests/metagpt/actions/test_write_teaching_plan.py | 8 ++++++++ tests/metagpt/roles/test_teacher.py | 4 ++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 1f0167df3..76c72651d 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/11 14:43 +@Time : 2023/7/27 @Author : mashenquan @File : write_teaching_plan.py """ diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index a007926be..acaa3860f 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/23 17:25 +@Time : 2023/7/27 @Author : mashenquan @File : teacher.py """ diff --git a/tests/metagpt/actions/test_write_teaching_plan.py b/tests/metagpt/actions/test_write_teaching_plan.py index b47d6ab56..2e34491fb 100644 --- a/tests/metagpt/actions/test_write_teaching_plan.py +++ b/tests/metagpt/actions/test_write_teaching_plan.py @@ -1,3 +1,11 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/7/28 17:25 +@Author : mashenquan +@File : test_write_teaching_plan.py +""" + import asyncio from typing import Optional from pydantic import BaseModel diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 3af053338..5faa43455 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -1,9 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/23 17:25 +@Time : 2023/7/27 13:25 @Author : mashenquan -@File : teacher.py +@File : test_teacher.py """ from typing import Dict, Optional From 34e5658009016cefd0083c0797ef6d3f42b68fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:14:37 +0800 Subject: [PATCH 006/398] feat: + notation --- metagpt/actions/write_teaching_plan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 76c72651d..0778b86b4 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -68,11 +68,11 @@ class WriteTeachingPlanPart(Action): self.rsp = "# " + self.rsp def __str__(self): - """str()时返回`topic`""" + """Return `topic` value when str()""" return self.topic def __repr__(self): - """调试时返回`topic`""" + """Show `topic` value when debug""" return self.topic FORMATION = """ From d7c1d9797f82d80015f7ec3e548466c9b25b938c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:22:28 +0800 Subject: [PATCH 007/398] fixbug: prompt format error --- metagpt/actions/write_teaching_plan.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 0778b86b4..370c70040 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -76,11 +76,11 @@ class WriteTeachingPlanPart(Action): return self.topic FORMATION = """ - "\tCapacity and role" defines the role you are currently playing; - "\t[LESSON_BEGIN]" and "[LESSON_END]" tags enclose the content of textbook; - "\tStatement" defines the work detail you need to complete at this stage; - "\tAnswer options" defines the format requirements for your responses; - "\tConstraint" defines the conditions that your responses must comply with. + \t\"Capacity and role\" defines the role you are currently playing; + \t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook; + \t\"Statement\" defines the work detail you need to complete at this stage; + \t\"Answer options\" defines the format requirements for your responses; + \t\"Constraint\" defines the conditions that your responses must comply with. """ COURSE_TITLE = "Title" TOPICS = [COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", From d79fe56a38596385a6ea5aafdb5752942bbebedc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:34:30 +0800 Subject: [PATCH 008/398] fixbug: prompt format error --- metagpt/actions/write_teaching_plan.py | 60 +++++++++---------- .../actions/test_write_teaching_plan.py | 4 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 370c70040..4c36983ff 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -75,13 +75,12 @@ class WriteTeachingPlanPart(Action): """Show `topic` value when debug""" return self.topic - FORMATION = """ - \t\"Capacity and role\" defines the role you are currently playing; - \t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook; - \t\"Statement\" defines the work detail you need to complete at this stage; - \t\"Answer options\" defines the format requirements for your responses; - \t\"Constraint\" defines the conditions that your responses must comply with. - """ + FORMATION = "\"Capacity and role\" defines the role you are currently playing;\n" \ + "\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n" \ + "\t\"Statement\" defines the work detail you need to complete at this stage;\n" \ + "\t\"Answer options\" defines the format requirements for your responses;\n" \ + "\t\"Constraint\" defines the conditions that your responses must comply with." + COURSE_TITLE = "Title" TOPICS = [COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", "Teaching Methods and Strategies", "Learning Activities", @@ -104,31 +103,32 @@ class WriteTeachingPlanPart(Action): } # Teaching plan title - PROMPT_TITLE_TEMPLATE = """ - Do not refer to the context of the previous conversation records, start the conversation anew.\n\n - Formation: {formation}\n\n - {statements}\n - Constraint: Writing in {language}.\n - Answer options: Encloses the lesson title with "[TEACHING_PLAN_BEGIN]" and "[TEACHING_PLAN_END]" tags.\n - [LESSON_BEGIN]\n - {lesson}\n - [LESSON_END] - """ + PROMPT_TITLE_TEMPLATE = "Do not refer to the context of the previous conversation records, " \ + "start the conversation anew.\n\n" \ + "Formation: {formation}\n\n" \ + "{statements}\n" \ + "Constraint: Writing in {language}.\n" \ + "Answer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" " \ + "and \"[TEACHING_PLAN_END]\" tags.\n" \ + "[LESSON_BEGIN]\n" \ + "{lesson}\n" \ + "[LESSON_END]" # Teaching plan parts: - PROMPT_TEMPLATE = """ - Do not refer to the context of the previous conversation records, start the conversation anew.\n\n - Formation: {formation}\n\n - Capacity and role: {role}\n - Statement: Write the "{topic}" part of teaching plan, WITHOUT ANY content unrelated to "{topic}"!!\n - {statements}\n - Answer options: Enclose the teaching plan content with "[TEACHING_PLAN_BEGIN]" and "[TEACHING_PLAN_END]" tags.\n - Answer options: Using proper markdown format from second-level header format.\n - Constraint: Writing in {language}.\n - [LESSON_BEGIN]\n - {lesson}\n - [LESSON_END] - """ + PROMPT_TEMPLATE = "Do not refer to the context of the previous conversation records, " \ + "start the conversation anew.\n\n" \ + "Formation: {formation}\n\n" \ + "Capacity and role: {role}\n" \ + "Statement: Write the \"{topic}\" part of teaching plan, " \ + "WITHOUT ANY content unrelated to \"{topic}\"!!\n" \ + "{statements}\n" \ + "Answer options: Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" " \ + "and \"[TEACHING_PLAN_END]\" tags.\n" \ + "Answer options: Using proper markdown format from second-level header format.\n" \ + "Constraint: Writing in {language}.\n" \ + "[LESSON_BEGIN]\n" \ + "{lesson}\n" \ + "[LESSON_END]" DATA_BEGIN_TAG = "[TEACHING_PLAN_BEGIN]" DATA_END_TAG = "[TEACHING_PLAN_END]" diff --git a/tests/metagpt/actions/test_write_teaching_plan.py b/tests/metagpt/actions/test_write_teaching_plan.py index 2e34491fb..299a89639 100644 --- a/tests/metagpt/actions/test_write_teaching_plan.py +++ b/tests/metagpt/actions/test_write_teaching_plan.py @@ -34,7 +34,7 @@ async def mock_write_teaching_plan_part(): { "input": "AABBCC", "name": "A", - "topic": "B", + "topic": WriteTeachingPlanPart.COURSE_TITLE, "language": "C" }, { @@ -52,7 +52,7 @@ async def mock_write_teaching_plan_part(): assert act.topic == seed.topic assert str(act) == seed.topic assert act.name == seed.name - assert act.rsp == "prompt" + assert act.rsp == "# prompt" if seed.topic == WriteTeachingPlanPart.COURSE_TITLE else "prompt" def test_suite(): From b95a01079c4a0cc18119ef196f5f0124556fbdc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:42:02 +0800 Subject: [PATCH 009/398] feat: + notation --- metagpt/roles/teacher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index acaa3860f..fede9f74a 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -72,9 +72,9 @@ class Teacher(Role): @staticmethod def new_file_name(lesson_title, ext=".md"): """Create a related file name based on `lesson_title` and `ext`.""" - # 定义需要替换的特殊字符 + # Define the special characters that need to be replaced. illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' - # 将特殊字符替换为下划线 + # Replace the special characters with underscores. filename = re.sub(illegal_chars, '_', lesson_title) + ext return re.sub(r'_+', '_', filename) From 16bad64649cbdc3b498376981dd0039a63e0eff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:44:30 +0800 Subject: [PATCH 010/398] feat: + notation --- startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/startup.py b/startup.py index c05bbbbf0..1fe4a067a 100644 --- a/startup.py +++ b/startup.py @@ -98,7 +98,7 @@ def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. :param n_round: :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` - :param kwargs: Parameters passed in format: `python your_script.py a--param1=value1 --param2=value2` + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` :return: """ industry = kwargs.get("industry", "software") From 7cab03942ebdc9a7fbfcc34f31d0bf8d780d6fb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 31 Jul 2023 15:44:12 +0800 Subject: [PATCH 011/398] =?UTF-8?q?feat:=20+=E7=BB=83=E4=B9=A0=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/write_teaching_plan.py | 57 +++++++++++++++++++++++++- metagpt/roles/role.py | 2 +- startup.py | 2 +- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 4c36983ff..66c370bc9 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -39,7 +39,12 @@ class WriteTeachingPlanPart(Action): if len(args) < 1 or len(args[0]) < 1 or not isinstance(args[0][0], Message): raise ValueError("Invalid args, a tuple of List[Message] is expected") - statements = self.TOPIC_STATEMENTS.get(self.topic, []) + statement_patterns = self.TOPIC_STATEMENTS.get(self.topic, []) + statements = [] + from metagpt.roles import Role + for p in statement_patterns: + s = Role.format_value(p, kwargs) + statements.append(s) formatter = self.PROMPT_TITLE_TEMPLATE if self.topic == self.COURSE_TITLE else self.PROMPT_TEMPLATE prompt = formatter.format(formation=self.FORMATION, role=self.prefix, @@ -84,7 +89,9 @@ class WriteTeachingPlanPart(Action): COURSE_TITLE = "Title" TOPICS = [COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", "Teaching Methods and Strategies", "Learning Activities", - "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement"] + "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", + "Vocabulary Practice", "Grammar Practice", "Reading Comprehension", "Listening Practice", + "Writing Practice", "Speaking Practice", "Translation Practice", "Listening and Speaking Activities"] TOPIC_STATEMENTS = { COURSE_TITLE: ["Statement: Find and return the title of the lesson only in markdown first-level header format, " @@ -99,6 +106,52 @@ class WriteTeachingPlanPart(Action): "Teaching Methods and Strategies": [ "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, " "procedures, in detail." + ], + "Vocabulary Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create vocabulary practice exercises. The exercises should be in either {language} with " + "{teaching_language} answers or {teaching_language} with {language} answers. The key-related vocabulary " + "and phrases in the textbook content must all be included in the exercises." + ], + "Grammar Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create grammar practice exercises. "], + "Reading Comprehension": [ + "Statement: Based on the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create {teaching_language} reading comprehension exercises. ", + "Statement: Prohibit the use of words that are not within the scope of the \"[LESSON_BEGIN]\" " + "and \"[LESSON_END]\" tags.", + "Statement: Prohibit copy the content of the \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + "Answer options: Write the story content in {teaching_language}." + ], + "Listening Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create listening practice exercises. Each exercise should include the audio content and the " + "question-and-answer part." + ], + "Writing Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create writing practice exercises.", + #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + ], + "Speaking Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create speaking practice exercises.", + #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + ], + "Translation Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create Translation practice exercises.", + #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + ], + "Listening and Speaking Activities": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create listening and speaking activities exercises.", + #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." ] } diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 3e18257ed..47aa90197 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -196,7 +196,7 @@ class Role: logger.info(f"{self._setting}: ready to {self._rc.todo}") requirement = self._rc.important_memory - response = await self._rc.todo.run(requirement) + response = await self._rc.todo.run(requirement, **self._options) # logger.info(response) if isinstance(response, ActionOutput): msg = Message(content=response.content, instruct_content=response.instruct_content, diff --git a/startup.py b/startup.py index 1fe4a067a..ee8cd3b6e 100644 --- a/startup.py +++ b/startup.py @@ -69,7 +69,7 @@ async def education_startup(lesson_file: str, investment: float = 3.0, n_round: 3a Listen, say and trace Aa Bb Cc Dd Ee Ff Gg - 3b Listen and number the following letters. Then circle the letters withthe same sound as Bb. + 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. Aa Bb Cc Dd Ee Ff Gg 3c Match the big letters with the small ones. Then write them on the lines. From 8b7eddad86c47e76b200c3ed4cbe1c2377a3767f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 31 Jul 2023 15:57:06 +0800 Subject: [PATCH 012/398] =?UTF-8?q?feat:=20+=E7=BB=83=E4=B9=A0=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/write_teaching_plan.py | 33 +++++++++++++------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 66c370bc9..09b45634c 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -87,7 +87,8 @@ class WriteTeachingPlanPart(Action): "\t\"Constraint\" defines the conditions that your responses must comply with." COURSE_TITLE = "Title" - TOPICS = [COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", + TOPICS = [ + COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", "Teaching Methods and Strategies", "Learning Activities", "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", "Vocabulary Practice", "Grammar Practice", "Reading Comprehension", "Listening Practice", @@ -117,41 +118,41 @@ class WriteTeachingPlanPart(Action): "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create grammar practice exercises. "], "Reading Comprehension": [ - "Statement: Based on the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create {teaching_language} reading comprehension exercises. ", - "Statement: Prohibit the use of words that are not within the scope of the \"[LESSON_BEGIN]\" " - "and \"[LESSON_END]\" tags.", - "Statement: Prohibit copy the content of the \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - "Answer options: Write the story content in {teaching_language}." + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create {teaching_language} reading comprehension exercises. " + # "Statement: Prohibit the use of words that are not within the scope of the \"[LESSON_BEGIN]\" " + # "and \"[LESSON_END]\" tags.", + # "Statement: Prohibit copy the content of the \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + # "Answer options: Write the story content in {teaching_language}." ], "Listening Practice": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create listening practice exercises. Each exercise should include the audio content and the " "question-and-answer part." ], "Writing Practice": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create writing practice exercises.", #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." ], "Speaking Practice": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create speaking practice exercises.", #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." ], "Translation Practice": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create Translation practice exercises.", #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." ], "Listening and Speaking Activities": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create listening and speaking activities exercises.", #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." ] } From 444b609e38c931fa18fcf04d35b7ed4016fb46d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 31 Jul 2023 17:27:09 +0800 Subject: [PATCH 013/398] =?UTF-8?q?feat:=20=E5=88=A0=E6=8E=89=E6=97=A0?= =?UTF-8?q?=E7=94=A8=E7=9A=84part?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/write_teaching_plan.py | 94 ++++++++++++++++++++++++++ metagpt/actions/write_teaching_plan.py | 53 +++------------ 2 files changed, 102 insertions(+), 45 deletions(-) create mode 100644 examples/write_teaching_plan.py diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py new file mode 100644 index 000000000..ec8ad8948 --- /dev/null +++ b/examples/write_teaching_plan.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Modified By: mashenquan, 2023-07-27, + `industry` concept +""" + +import asyncio +from pathlib import Path +import aiofiles +import fire +from metagpt.logs import logger +from metagpt.actions.write_teaching_plan import TeachingPlanRequirement +from metagpt.roles.teacher import Teacher +from metagpt.software_company import SoftwareCompany + + +async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): + """Run a startup. Be a teacher in education industry.""" + + demo_lesson = """ + UNIT 1 Making New Friends + TOPIC 1 Welcome to China! + Section A + + 1a Listen and number the following names. + Jane Mari Kangkang Michael + Look, listen and understand. Then practice the conversation. + Work in groups. Introduce yourself using + I ’m ... Then practice 1a + with your own hometown or the following places. + + 1b Listen and number the following names + Jane Michael Maria Kangkang + 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. + China the USA the UK Hong Kong Beijing + + 2a Look, listen and understand. Then practice the conversation + Hello! + Hello! + Hello! + Hello! Are you Maria? + No, I’m not. I’m Jane. + Oh, nice to meet you, Jane + Nice to meet you, too. + Hi, Maria! + Hi, Kangkang! + Welcome to China! + Thanks. + + 2b Work in groups. Make up a conversation with your own name and the + following structures. + A: Hello! / Good morning! / Hi! I’m ... Are you ... ? + B: ... + + 3a Listen, say and trace + Aa Bb Cc Dd Ee Ff Gg + + 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. + Aa Bb Cc Dd Ee Ff Gg + + 3c Match the big letters with the small ones. Then write them on the lines. + """ + + lesson = "" + if lesson_file is not None and Path(lesson_file).exists(): + async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: + lesson = await reader.read() + logger.info(f"Course content: {lesson}") + if not lesson: + logger.info("No course content provided, using the demo course.") + lesson = demo_lesson + + company = SoftwareCompany() + company.hire([Teacher(*args, **kwargs)]) + company.invest(investment) + company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) + await company.run(n_round=1) + + +def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): + """ + We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + :param idea: Your innovative idea for `software` industry, such as "Creating a snake game."; lesson filename for `education` industry. + :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. + :param n_round: + :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + :return: + """ + asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) + + +if __name__ == '__main__': + fire.Fire(main) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 09b45634c..bd6e96956 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -89,10 +89,10 @@ class WriteTeachingPlanPart(Action): COURSE_TITLE = "Title" TOPICS = [ COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", - "Teaching Methods and Strategies", "Learning Activities", - "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", - "Vocabulary Practice", "Grammar Practice", "Reading Comprehension", "Listening Practice", - "Writing Practice", "Speaking Practice", "Translation Practice", "Listening and Speaking Activities"] + "Teaching Methods and Strategies", "Learning Activities", + "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", + "Vocabulary Cloze", "Grammar Questions" + ] TOPIC_STATEMENTS = { COURSE_TITLE: ["Statement: Find and return the title of the lesson only in markdown first-level header format, " @@ -108,52 +108,15 @@ class WriteTeachingPlanPart(Action): "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, " "procedures, in detail." ], - "Vocabulary Practice": [ + "Vocabulary Cloze": [ "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create vocabulary practice exercises. The exercises should be in either {language} with " + "create vocabulary cloze. The cloze should be in either {language} with " "{teaching_language} answers or {teaching_language} with {language} answers. The key-related vocabulary " "and phrases in the textbook content must all be included in the exercises." ], - "Grammar Practice": [ + "Grammar Questions": [ "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create grammar practice exercises. "], - "Reading Comprehension": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create {teaching_language} reading comprehension exercises. " - # "Statement: Prohibit the use of words that are not within the scope of the \"[LESSON_BEGIN]\" " - # "and \"[LESSON_END]\" tags.", - # "Statement: Prohibit copy the content of the \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - # "Answer options: Write the story content in {teaching_language}." - ], - "Listening Practice": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create listening practice exercises. Each exercise should include the audio content and the " - "question-and-answer part." - ], - "Writing Practice": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create writing practice exercises.", - #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." - ], - "Speaking Practice": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create speaking practice exercises.", - #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." - ], - "Translation Practice": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create Translation practice exercises.", - #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." - ], - "Listening and Speaking Activities": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create listening and speaking activities exercises.", - #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." - ] + "create grammar questions. "] } # Teaching plan title From 3b0f76a1cd1ffada96f5e47ab6c91e7618b5efaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 31 Jul 2023 17:48:06 +0800 Subject: [PATCH 014/398] =?UTF-8?q?fixbug:=20=E7=BB=9F=E4=B8=80=E4=BA=86?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/design_api.py | 3 ++- metagpt/actions/project_management.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 1447eacc3..55213a4b0 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -135,7 +135,8 @@ class WriteDesign(Action): self._save_prd(docs_path, resources_path, context[-1].content) self._save_system_design(docs_path, resources_path, content) - async def run(self, context): + async def run(self, *args, **kwargs): + context = args[0] prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 89c59dcda..394f279e8 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -115,7 +115,8 @@ class WriteTasks(Action): requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt' requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) - async def run(self, context): + async def run(self, *args, **kwargs): + context = args[0] prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) self._save(context, rsp) From 4379d360228e6ded6900ca855658daca02c80172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 31 Jul 2023 17:48:41 +0800 Subject: [PATCH 015/398] =?UTF-8?q?fixbug:=20=E7=BB=9F=E4=B8=80=E4=BA=86?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- startup.py | 97 +++++------------------------------------------------- 1 file changed, 8 insertions(+), 89 deletions(-) diff --git a/startup.py b/startup.py index ee8cd3b6e..e062babb5 100644 --- a/startup.py +++ b/startup.py @@ -1,23 +1,15 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" -@Modified By: mashenquan, 2023-07-27, + `industry` concept -""" - import asyncio -from pathlib import Path -import aiofiles + import fire -from metagpt.logs import logger -from metagpt.actions.write_teaching_plan import TeachingPlanRequirement + from metagpt.roles import Architect, Engineer, ProductManager, ProjectManager -from metagpt.roles.teacher import Teacher from metagpt.software_company import SoftwareCompany -async def software_startup(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): - """Run a startup. Be a boss in software industry.""" - code_review = kwargs.get("code_review", False) # Whether to use code review. +async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False): + """Run a startup. Be a boss.""" company = SoftwareCompany() company.hire([ProductManager(), Architect(), @@ -28,89 +20,16 @@ async def software_startup(idea: str, investment: float = 3.0, n_round: int = 5, await company.run(n_round=n_round) -async def education_startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): - """Run a startup. Be a teacher in education industry.""" - - demo_lesson = """ - UNIT 1 Making New Friends - TOPIC 1 Welcome to China! - Section A - - 1a Listen and number the following names. - Jane Mari Kangkang Michael - Look, listen and understand. Then practice the conversation. - Work in groups. Introduce yourself using - I ’m ... Then practice 1a - with your own hometown or the following places. - - 1b Listen and number the following names - Jane Michael Maria Kangkang - 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. - China the USA the UK Hong Kong Beijing - - 2a Look, listen and understand. Then practice the conversation - Hello! - Hello! - Hello! - Hello! Are you Maria? - No, I’m not. I’m Jane. - Oh, nice to meet you, Jane - Nice to meet you, too. - Hi, Maria! - Hi, Kangkang! - Welcome to China! - Thanks. - - 2b Work in groups. Make up a conversation with your own name and the - following structures. - A: Hello! / Good morning! / Hi! I’m ... Are you ... ? - B: ... - - 3a Listen, say and trace - Aa Bb Cc Dd Ee Ff Gg - - 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. - Aa Bb Cc Dd Ee Ff Gg - - 3c Match the big letters with the small ones. Then write them on the lines. - """ - - lesson = "" - if lesson_file is not None and Path(lesson_file).exists(): - async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: - lesson = await reader.read() - logger.info(f"Course content: {lesson}") - if not lesson: - logger.info("No course content provided, using the demo course.") - lesson = demo_lesson - - company = SoftwareCompany() - company.hire([Teacher(*args, **kwargs)]) - company.invest(investment) - company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) - await company.run(n_round=1) - - -def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): +def main(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False): """ We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea for `software` industry, such as "Creating a snake game."; lesson filename for `education` industry. + :param idea: Your innovative idea, such as "Creating a snake game." :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. :param n_round: - :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` - :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + :param code_review: Whether to use code review. :return: """ - industry = kwargs.get("industry", "software") - industries = { - "software": software_startup, - "education": education_startup, - } - startup = industries.get(industry) - if startup is None: - print(f"Available industries:{list(industries.keys())}") - return - asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) + asyncio.run(startup(idea, investment, n_round, code_review)) if __name__ == '__main__': From 85c7148b6235fb91a8d141b1d6be990e8999ced8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 10:04:21 +0800 Subject: [PATCH 016/398] feat: +param type --- metagpt/actions/write_teaching_plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index bd6e96956..10fc2863f 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -21,7 +21,7 @@ class TeachingPlanRequirement(Action): class WriteTeachingPlanPart(Action): """Write Teaching Plan Part""" - def __init__(self, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): + def __init__(self, name: str = "", context=None, llm: LLM = None, topic: str = "", language: str = "Chinese"): """ Args: From d415ca5dbc5a9622cb65b331d7ca87c224de57a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 10:48:26 +0800 Subject: [PATCH 017/398] fixbug: tests --- metagpt/actions/action.py | 1 + metagpt/actions/design_api.py | 3 +-- metagpt/actions/project_management.py | 3 +-- metagpt/actions/write_teaching_plan.py | 9 ++++----- metagpt/roles/teacher.py | 3 ++- tests/metagpt/actions/test_ui_design.py | 10 ++++------ tests/metagpt/actions/test_write_code.py | 3 ++- 7 files changed, 15 insertions(+), 17 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index fa0d592a3..6b9ea626c 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -15,6 +15,7 @@ from metagpt.llm import LLM from metagpt.utils.common import OutputParser from metagpt.logs import logger + class Action(ABC): def __init__(self, name: str = '', context=None, llm: LLM = None): self.name: str = name diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 55213a4b0..1447eacc3 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -135,8 +135,7 @@ class WriteDesign(Action): self._save_prd(docs_path, resources_path, context[-1].content) self._save_system_design(docs_path, resources_path, content) - async def run(self, *args, **kwargs): - context = args[0] + async def run(self, context): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 394f279e8..89c59dcda 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -115,8 +115,7 @@ class WriteTasks(Action): requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt' requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) - async def run(self, *args, **kwargs): - context = args[0] + async def run(self, context): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) self._save(context, rsp) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 10fc2863f..e8fe110d8 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -5,7 +5,6 @@ @Author : mashenquan @File : write_teaching_plan.py """ -from langchain.llms.base import LLM from metagpt.logs import logger from metagpt.actions import Action from metagpt.schema import Message @@ -21,7 +20,7 @@ class TeachingPlanRequirement(Action): class WriteTeachingPlanPart(Action): """Write Teaching Plan Part""" - def __init__(self, name: str = "", context=None, llm: LLM = None, topic: str = "", language: str = "Chinese"): + def __init__(self, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): """ Args: @@ -35,8 +34,8 @@ class WriteTeachingPlanPart(Action): self.language = language self.rsp = None - async def run(self, *args, **kwargs): - if len(args) < 1 or len(args[0]) < 1 or not isinstance(args[0][0], Message): + async def run(self, messages, *args, **kwargs): + if len(messages) < 1 or not isinstance(messages[0], Message): raise ValueError("Invalid args, a tuple of List[Message] is expected") statement_patterns = self.TOPIC_STATEMENTS.get(self.topic, []) @@ -49,7 +48,7 @@ class WriteTeachingPlanPart(Action): prompt = formatter.format(formation=self.FORMATION, role=self.prefix, statements="\n".join(statements), - lesson=args[0][0].content, + lesson=messages[0].content, topic=self.topic, language=self.language) diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index fede9f74a..5d10c4d17 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -10,6 +10,7 @@ from pathlib import Path import aiofiles from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart, TeachingPlanRequirement +from metagpt.const import WORKSPACE_ROOT from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger @@ -59,7 +60,7 @@ class Teacher(Role): async def save(self, content): """Save teaching plan""" filename = Teacher.new_file_name(self.course_title) - pathname = Path(__file__).resolve().parent.parent.parent / "output" + pathname = WORKSPACE_ROOT / "output" pathname.mkdir(exist_ok=True) pathname = pathname / filename try: diff --git a/tests/metagpt/actions/test_ui_design.py b/tests/metagpt/actions/test_ui_design.py index d284b20f2..dedd0b30e 100644 --- a/tests/metagpt/actions/test_ui_design.py +++ b/tests/metagpt/actions/test_ui_design.py @@ -4,7 +4,7 @@ # from tests.metagpt.roles.ui_role import UIDesign -llm_resp= ''' +llm_resp = ''' # UI Design Description ```The user interface for the snake game will be designed in a way that is simple, clean, and intuitive. The main elements of the game such as the game grid, snake, food, score, and game over message will be clearly defined and easy to understand. The game grid will be centered on the screen with the score displayed at the top. The game controls will be intuitive and easy to use. The design will be modern and minimalist with a pleasing color scheme.``` @@ -100,6 +100,7 @@ body { font-size: 3em; ''' + def test_ui_design_parse_css(): ui_design_work = UIDesign(name="UI design action") @@ -161,7 +162,7 @@ def test_ui_design_parse_css(): transform: translate(-50%, -50%); font-size: 3em; ''' - assert ui_design_work.parse_css_code(context=llm_resp)==css + assert ui_design_work.parse_css_code(context=llm_resp) == css def test_ui_design_parse_html(): @@ -185,7 +186,4 @@ def test_ui_design_parse_html(): ''' - assert ui_design_work.parse_css_code(context=llm_resp)==html - - - + assert ui_design_work.parse_css_code(context=llm_resp) == html diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 7bb18ddf2..2d4c496e1 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_code.py +@Modified By: mashenquan, 2023-8-1, fix-bug: `filename` of `write_code.run()` is missing. """ import pytest @@ -18,7 +19,7 @@ async def test_write_code(): api_design = "设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。" write_code = WriteCode("write_code") - code = await write_code.run(api_design) + code = await write_code.run(context=api_design, filename="test") logger.info(code) # 我们不能精确地预测生成的代码，但我们可以检查某些关键字 From a56e9a29e3243d6584c326a171e19dc38bf2f906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 10:53:34 +0800 Subject: [PATCH 018/398] feat: change save to --- metagpt/roles/teacher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index 5d10c4d17..95d54133b 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -60,7 +60,7 @@ class Teacher(Role): async def save(self, content): """Save teaching plan""" filename = Teacher.new_file_name(self.course_title) - pathname = WORKSPACE_ROOT / "output" + pathname = WORKSPACE_ROOT / "teaching_plan" pathname.mkdir(exist_ok=True) pathname = pathname / filename try: From 8f5b3e076e9655f0dde939dd9ed6e3a9e49ac27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 11:52:50 +0800 Subject: [PATCH 019/398] feat: +Choice Questions, Translation Questions --- metagpt/actions/write_teaching_plan.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index e8fe110d8..2916d7309 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -90,7 +90,7 @@ class WriteTeachingPlanPart(Action): COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", "Teaching Methods and Strategies", "Learning Activities", "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", - "Vocabulary Cloze", "Grammar Questions" + "Vocabulary Cloze", "Choice Questions", "Grammar Questions", "Translation Questions" ] TOPIC_STATEMENTS = { @@ -109,13 +109,20 @@ class WriteTeachingPlanPart(Action): ], "Vocabulary Cloze": [ "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create vocabulary cloze. The cloze should be in either {language} with " - "{teaching_language} answers or {teaching_language} with {language} answers. The key-related vocabulary " - "and phrases in the textbook content must all be included in the exercises." + "create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} " + "answers, and it should also include 10 {teaching_language} questions with {language} answers. " + "The key-related vocabulary and phrases in the textbook content must all be included in the exercises.", ], "Grammar Questions": [ "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create grammar questions. "] + "create grammar questions. 10 questions."], + "Choice Questions": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create choice questions. 10 questions."], + "Translation Questions": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create translation questions. 10 questions." + ] } # Teaching plan title From 0c1febfc77555df93051f00e245d38d626028b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 12:06:03 +0800 Subject: [PATCH 020/398] feat: +Choice Questions, Translation Questions --- metagpt/actions/write_teaching_plan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 2916d7309..1c9b1a86e 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -121,7 +121,9 @@ class WriteTeachingPlanPart(Action): "create choice questions. 10 questions."], "Translation Questions": [ "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create translation questions. 10 questions." + "create translation questions. The translation should include 10 {language} questions with " + "{teaching_language} answers, and it should also include 10 {teaching_language} questions with " + "{language} answers." ] } From e5885ec99ae2ab3fdb0d459d8572bfe03b646e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 12:07:58 +0800 Subject: [PATCH 021/398] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/roles/role.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 47aa90197..36dfb2d36 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -282,8 +282,7 @@ class Role: @staticmethod def format_value(value, options): - """Fill parameters inside `value` with `options`. - """ + """Fill parameters inside `value` with `options`.""" if "{" not in value: return value From a4017a1eeca186ac41cec05a851f200fab8c33d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 13:08:17 +0800 Subject: [PATCH 022/398] feat: + annotation --- examples/write_teaching_plan.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index ec8ad8948..da97a5463 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -80,9 +80,9 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): """ We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea for `software` industry, such as "Creating a snake game."; lesson filename for `education` industry. + :param idea: lesson filename. :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. - :param n_round: + :param n_round: Reserved. :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` :return: @@ -91,4 +91,11 @@ def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): if __name__ == '__main__': + """ + Formats: + ``` + python write_teaching_plan.py lesson_filename --teaching_language= --language= + ``` + If `lesson_filename` is not available, a demo lesson content will be used. + """ fire.Fire(main) From b8901f2bb17cde7eb4cff9e85e1269d0664ec1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 13:23:17 +0800 Subject: [PATCH 023/398] feat: +annotation --- metagpt/actions/write_teaching_plan.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 1c9b1a86e..3718c9801 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -23,11 +23,11 @@ class WriteTeachingPlanPart(Action): def __init__(self, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): """ - Args: - name: action name - context: context - llm: object of :class:`LLM` - topic: topic part of teaching plan + :param name: action name + :param context: context + :param llm: object of :class:`LLM` + :param topic: topic part of teaching plan + :param language: A human language, such as Chinese, English, French, etc. """ super().__init__(name, context, llm) self.topic = topic From 80a189ad4a1546f8c1a9dbe00c42725868c35e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 7 Aug 2023 11:24:34 +0800 Subject: [PATCH 024/398] feat: +meta role --- .gitignore | 1 + config/pattern/write_teaching_plan.yaml | 51 +++++++++++++ examples/write_teaching_plan.py | 5 +- metagpt/roles/fork_meta_role.py | 98 +++++++++++++++++++++++++ metagpt/roles/meta_role.py | 29 ++++++++ 5 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 config/pattern/write_teaching_plan.yaml create mode 100644 metagpt/roles/fork_meta_role.py create mode 100644 metagpt/roles/meta_role.py diff --git a/.gitignore b/.gitignore index 3ec71f8b6..e326e8372 100644 --- a/.gitignore +++ b/.gitignore @@ -163,6 +163,7 @@ workspace/* *.mmd tmp output.wav +*.bak # output folder output diff --git a/config/pattern/write_teaching_plan.yaml b/config/pattern/write_teaching_plan.yaml new file mode 100644 index 000000000..fbd8b4ae9 --- /dev/null +++ b/config/pattern/write_teaching_plan.yaml @@ -0,0 +1,51 @@ +# `fork` role demo +- role_type: "fork" + name: "Lily" + profile: "{teaching_language} Teacher" + goal: "writing a {language} teaching plan part by part" + constraints: "writing in {language}" + desc: "" + actions: + - name: "" + topic: "Title" + language: "Chinese" + statements: + - "Statement: Find and return the title of the lesson only in markdown first-level header format, without anything else." + - name: "" + topic: "Teaching Content" + language: "Chinese" + statements: + - "Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar structures that appear in the textbook, as well as the listening materials and key points." + - "Statement: \"Teaching Content\" must include more examples." + - name: "" + topic: "Teaching Time Allocation" + language: "Chinese" + statements: + - "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each part of the textbook content." + - name: "" + topic: "Teaching Methods and Strategies" + language: "Chinese" + statements: + - "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, procedures, in detail." + - name: "" + topic: "Vocabulary Cloze" + language: "Chinese" + statements: + - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers. The key-related vocabulary and phrases in the textbook content must all be included in the exercises." + - name: "" + topic: "Grammar Questions" + language: "Chinese" + statements: + - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create grammar questions. 10 questions." + - name: "" + topic: "Choice Questions" + language: "Chinese" + statements: + - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create choice questions. 10 questions." + - name: "" + topic: "Translation Questions" + language: "Chinese" + statements: + - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create translation questions. The translation should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers." + + diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index da97a5463..30a8d8366 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -1,7 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Modified By: mashenquan, 2023-07-27, + `industry` concept +@Time : 2023-07-27 +@Author : mashenquan +@File : write_teaching_plan.py +@Desc: Write teaching plan demo """ import asyncio diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py new file mode 100644 index 000000000..1a69b9ca7 --- /dev/null +++ b/metagpt/roles/fork_meta_role.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : fork_meta_role.py +@Desc : 我试图将UML的一些符号概念引入到MetaGPT，使其具备通过符号拼接自由搭建flow的能力。同时我也尝试将这些符号做得配置化和标准化，让flow搭建流程更便捷。这是一个`fork` meta-role demo，实现的是write_teaching_plan功能。 +""" + + + +async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): + """Run a startup. Be a teacher in education industry.""" + + demo_lesson = """ + UNIT 1 Making New Friends + TOPIC 1 Welcome to China! + Section A + + 1a Listen and number the following names. + Jane Mari Kangkang Michael + Look, listen and understand. Then practice the conversation. + Work in groups. Introduce yourself using + I ’m ... Then practice 1a + with your own hometown or the following places. + + 1b Listen and number the following names + Jane Michael Maria Kangkang + 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. + China the USA the UK Hong Kong Beijing + + 2a Look, listen and understand. Then practice the conversation + Hello! + Hello! + Hello! + Hello! Are you Maria? + No, I’m not. I’m Jane. + Oh, nice to meet you, Jane + Nice to meet you, too. + Hi, Maria! + Hi, Kangkang! + Welcome to China! + Thanks. + + 2b Work in groups. Make up a conversation with your own name and the + following structures. + A: Hello! / Good morning! / Hi! I’m ... Are you ... ? + B: ... + + 3a Listen, say and trace + Aa Bb Cc Dd Ee Ff Gg + + 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. + Aa Bb Cc Dd Ee Ff Gg + + 3c Match the big letters with the small ones. Then write them on the lines. + """ + + lesson = "" + if lesson_file is not None and Path(lesson_file).exists(): + async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: + lesson = await reader.read() + logger.info(f"Course content: {lesson}") + if not lesson: + logger.info("No course content provided, using the demo course.") + lesson = demo_lesson + + + + company = SoftwareCompany() + company.hire([(*args, **kwargs)]) + company.invest(investment) + company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) + await company.run(n_round=1) + + +def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): + """ + We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + :param idea: lesson filename. + :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. + :param n_round: Reserved. + :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + :return: + """ + asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) + + +if __name__ == '__main__': + """ + Formats: + ``` + python write_teaching_plan.py lesson_filename --teaching_language= --language= + ``` + If `lesson_filename` is not available, a demo lesson content will be used. + """ + fire.Fire(main) diff --git a/metagpt/roles/meta_role.py b/metagpt/roles/meta_role.py new file mode 100644 index 000000000..1da180355 --- /dev/null +++ b/metagpt/roles/meta_role.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : meta_role.py +@Desc : 我试图将UML的一些符号概念引入到MetaGPT，使其具备通过符号拼接自由搭建flow的能力。同时我也尝试将这些符号做得配置化和标准化，让flow搭建流程更便捷。 + 分工参照UML 2.0 activity diagrams: `https://www.uml-diagrams.org/activity-diagrams.html` +""" +from typing import Dict, List + +from metagpt.roles import Role +from pydantic import BaseModel + +class UMLMetaRoleArgs(BaseModel): + role_type: str + name: str = "" + profile: str = "" + goal: str = "" + constraints: str = "" + desc: str = "" + actions: List + +class UMLMetaRole(Role): + """UML activity roles抽象父类""" + + def __init__(self, role_args: Dict): + """""" + self.role_args From 5702aaa5ad4e9113e954ab16e1fde1965b3f591b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 7 Aug 2023 21:12:27 +0800 Subject: [PATCH 025/398] feat: + uml fork style role demo --- config/pattern/template.yaml | 40 +++++ config/pattern/write_teaching_plan.yaml | 103 +++++++++++-- examples/fork_meta_role.py | 121 +++++++++++++++ metagpt/actions/meta_action.py | 61 ++++++++ metagpt/memory/memory.py | 10 +- metagpt/roles/fork_meta_role.py | 187 ++++++++++++++---------- metagpt/roles/meta_role.py | 29 ---- metagpt/roles/role.py | 6 +- metagpt/roles/teacher.py | 2 +- metagpt/roles/uml_meta_role_factory.py | 43 ++++++ metagpt/roles/uml_meta_role_options.py | 69 +++++++++ 11 files changed, 543 insertions(+), 128 deletions(-) create mode 100644 config/pattern/template.yaml create mode 100644 examples/fork_meta_role.py create mode 100644 metagpt/actions/meta_action.py delete mode 100644 metagpt/roles/meta_role.py create mode 100644 metagpt/roles/uml_meta_role_factory.py create mode 100644 metagpt/roles/uml_meta_role_options.py diff --git a/config/pattern/template.yaml b/config/pattern/template.yaml new file mode 100644 index 000000000..d148804f0 --- /dev/null +++ b/config/pattern/template.yaml @@ -0,0 +1,40 @@ +# Pattern Configuration Template +# Created By: mashenquan, 2023-8-7 +# File Name: template.yaml +# This template defines a set of structural standards for generating roles and action flows based on configurations. +# For more about UML 2.0 activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` + +# project settings +startup: + requirement: "TeachingPlanRequirement" # Defines project initial requirement action + role: "Teacher" # Defines project role + investment: 3.0 # Defines the max project investment + n_round: 1 # Defines the max project round count + +# roles settings +roles: # A project can involve multiple roles. +- role_type: "fork" # `fork` type role corresponds to the functional positioning of the `fork` node in UML 2.0 activity diagrams. + name: "Lily" + profile: "{teaching_language} Teacher" + goal: "writing a {language} teaching plan part by part" + constraints: "writing in {language}" + role: "You are a {teaching_language} Teacher, named Lily, your goal is ..." + desc: "" + output_filename: "teaching_plan_demo.md" + requirement: ["TeachingPlanRequirement"] + templates: # The template provides a convenient way to generate prompts. After each action selects its respective template, you only need to provide the corresponding variable values. Variable replacement is automatically handled by the framework. + - "Do ..." + - "Do ..." + # role's action settings + actions: # A role can have multiple actions. + - name: "" + topic: "Title" + language: "Chinese" + statements: # When replacing template variables, multiple statements will be joined into a single string using line breaks. + - "Statement: Find and return ..." + template_ix: 0 + rsp_begin_tag: "[..._BEGIN]" # When asking, request the LLM to include the tag in the response. It's optional. + rsp_end_tag: "[..._END]" # When asking, request the LLM to include the tag in the response. It's optional. + + + diff --git a/config/pattern/write_teaching_plan.yaml b/config/pattern/write_teaching_plan.yaml index fbd8b4ae9..6a05bad96 100644 --- a/config/pattern/write_teaching_plan.yaml +++ b/config/pattern/write_teaching_plan.yaml @@ -1,51 +1,124 @@ -# `fork` role demo -- role_type: "fork" +# The `fork` role demo implements the flow of the code in `examples/write_teaching_plan.py`. + +# project settings +startup: + requirement: "TeachingPlanRequirement" # Defines project initial requirement action + role: "Teacher" + investment: 3.0 + n_round: 1 + +# roles settings +roles: # A project can involve multiple roles. +- role_type: "fork" # `fork` type role corresponds to the functional positioning of the `fork` node in UML 2.0 activity diagrams. name: "Lily" profile: "{teaching_language} Teacher" goal: "writing a {language} teaching plan part by part" constraints: "writing in {language}" + role: "You are a {teaching_language} Teacher, named Lily, your goal is writing a {teaching_language} teaching plan part by part, and the constraint is writing in {language}." desc: "" - actions: + output_filename: "teaching_plan_demo.md" + requirement: ["TeachingPlanRequirement"] + templates: # The template provides a convenient way to generate prompts. After each action selects its respective template, you only need to provide the corresponding variable values. Variable replacement is automatically handled by the framework. + - "Do not refer to the context of the previous conversation records, start the conversation anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the format requirements for your responses;\n\t\"Constraint\" defines the conditions that your responses must comply with.\n\n{statements}\nConstraint: Writing in {language}.\nAnswer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END]" + - "Do not refer to the context of the previous conversation records, start the conversation anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the format requirements for your responses;\n\t\"Constraint\" defines the conditions that your responses must comply with.\n\nCapacity and role: {role}\nStatement: Write the \"{topic}\" part of teaching plan, WITHOUT ANY content unrelated to \"{topic}\"!!\n{statements}\nAnswer options: Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\nAnswer options: Using proper markdown format from second-level header format.\nConstraint: Writing in {language}.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END]" + actions: # 一个role可以有多个action - name: "" topic: "Title" language: "Chinese" - statements: - - "Statement: Find and return the title of the lesson only in markdown first-level header format, without anything else." + statements: # When replacing template variables, multiple statements will be joined into a single string using line breaks. + - "Statement: Find and return the title of the lesson only with \"# \" prefixed, without anything else." + template_ix: 0 + - name: "" + topic: "Teaching Hours" + language: "Chinese" + statements: [] + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" # When asking, request the LLM to include the tag in the response. It's optional. + rsp_end_tag: "[TEACHING_PLAN_END]" # When asking, request the LLM to include the tag in the response. It's optional. + - name: "" + topic: "Teaching Objectives" + language: "Chinese" + statements: [] + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Teaching Content" language: "Chinese" statements: - "Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar structures that appear in the textbook, as well as the listening materials and key points." - "Statement: \"Teaching Content\" must include more examples." - - name: "" - topic: "Teaching Time Allocation" - language: "Chinese" - statements: - - "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each part of the textbook content." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Teaching Methods and Strategies" language: "Chinese" statements: - "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, procedures, in detail." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" + - name: "" + topic: "Learning Activities" + language: "Chinese" + statements: [] + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" + - name: "" + topic: "Teaching Time Allocation" + language: "Chinese" + statements: + - "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each part of the textbook content." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" + - name: "" + topic: "Assessment and Feedback" + language: "Chinese" + statements: [] + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" + - name: "" + topic: "Teaching Summary and Improvement" + language: "Chinese" + statements: [] + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Vocabulary Cloze" language: "Chinese" statements: - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers. The key-related vocabulary and phrases in the textbook content must all be included in the exercises." - - name: "" - topic: "Grammar Questions" - language: "Chinese" - statements: - - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create grammar questions. 10 questions." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Choice Questions" language: "Chinese" statements: - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create choice questions. 10 questions." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" + - name: "" + topic: "Grammar Questions" + language: "Chinese" + statements: + - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create grammar questions. 10 questions." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Translation Questions" language: "Chinese" statements: - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create translation questions. The translation should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" diff --git a/examples/fork_meta_role.py b/examples/fork_meta_role.py new file mode 100644 index 000000000..21e3b5f7c --- /dev/null +++ b/examples/fork_meta_role.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : fork_meta_role.py +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to possess the + ability to construct flows freely by concatenating symbols. Simultaneously, I am also striving to make + these symbols configurable and standardized, making the process of building flow structures more + convenient. This is a fork meta-role demo that implements the functionality of + `examples/write_teaching_plan.py`. +""" + +import asyncio +from pathlib import Path + +import aiofiles +import fire +import yaml + +from metagpt.actions.meta_action import MetaAction +from metagpt.logs import logger +from metagpt.roles.uml_meta_role_factory import UMLMetaRoleFactory +from metagpt.roles.uml_meta_role_options import ProjectConfig +from metagpt.software_company import SoftwareCompany + + +async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): + """Run a startup. Be a teacher in education industry.""" + + demo_lesson = """ + UNIT 1 Making New Friends + TOPIC 1 Welcome to China! + Section A + + 1a Listen and number the following names. + Jane Mari Kangkang Michael + Look, listen and understand. Then practice the conversation. + Work in groups. Introduce yourself using + I ’m ... Then practice 1a + with your own hometown or the following places. + + 1b Listen and number the following names + Jane Michael Maria Kangkang + 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. + China the USA the UK Hong Kong Beijing + + 2a Look, listen and understand. Then practice the conversation + Hello! + Hello! + Hello! + Hello! Are you Maria? + No, I’m not. I’m Jane. + Oh, nice to meet you, Jane + Nice to meet you, too. + Hi, Maria! + Hi, Kangkang! + Welcome to China! + Thanks. + + 2b Work in groups. Make up a conversation with your own name and the + following structures. + A: Hello! / Good morning! / Hi! I’m ... Are you ... ? + B: ... + + 3a Listen, say and trace + Aa Bb Cc Dd Ee Ff Gg + + 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. + Aa Bb Cc Dd Ee Ff Gg + + 3c Match the big letters with the small ones. Then write them on the lines. + """ + + lesson = "" + if lesson_file is not None and Path(lesson_file).exists(): + async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: + lesson = await reader.read() + logger.info(f"Course content: {lesson}") + if not lesson: + logger.info("No course content provided, using the demo course.") + lesson = demo_lesson + + yaml_filename = kwargs["config"] + kwargs["lesson"] = lesson + + with open(yaml_filename, "r") as reader: + configs = yaml.safe_load(reader) + + startup_config = ProjectConfig(**configs) + roles = UMLMetaRoleFactory.create_roles(startup_config.roles, **kwargs) + company = SoftwareCompany() + company.hire(roles) + company.invest(startup_config.startup.investment) + company.start_project(lesson, role=startup_config.startup.role, + cause_by=MetaAction.get_action_type(startup_config.startup.requirement)) + await company.run(n_round=startup_config.startup.n_round) + + +def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): + """ + We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + :param idea: lesson filename. + :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. + :param n_round: Reserved. + :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + :return: + """ + asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) + + +if __name__ == '__main__': + """ + Formats: + ``` + python write_teaching_plan.py lesson_filename --teaching_language= --language= + ``` + If `lesson_filename` is not available, a demo lesson content will be used. + """ + fire.Fire(main) diff --git a/metagpt/actions/meta_action.py b/metagpt/actions/meta_action.py new file mode 100644 index 000000000..3f01b8c0f --- /dev/null +++ b/metagpt/actions/meta_action.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : meta_action.py +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the + ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to + make these symbols configurable and standardized, making the process of building flows more convenient. + For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` + This file defines a meta action capable of generating arbitrary actions at runtime based on a + configuration file. +""" + +from typing import Type + +from metagpt.actions import Action +from metagpt.logs import logger +from metagpt.roles.uml_meta_role_options import MetaActionOptions +from metagpt.schema import Message + + +class MetaAction(Action): + def __init__(self, options: MetaActionOptions, llm=None, **kwargs): + super(MetaAction, self).__init__(options.name, kwargs.get("context"), llm=llm) + self.prompt = options.format_prompt(**kwargs) + self.options = options + self.kwargs = kwargs + + def __str__(self): + """Return `topic` value when str()""" + return self.options.topic + + def __repr__(self): + """Show `topic` value when debug""" + return self.options.topic + + async def run(self, messages, *args, **kwargs): + if len(messages) < 1 or not isinstance(messages[0], Message): + raise ValueError("Invalid args, a tuple of List[Message] is expected") + + logger.debug(self.prompt) + rsp = await self._aask(prompt=self.prompt) + logger.debug(rsp) + self._set_result(rsp) + return self.rsp + + def _set_result(self, rsp): + if self.options.rsp_begin_tag and self.options.rsp_begin_tag in rsp: + ix = rsp.index(self.options.rsp_begin_tag) + rsp = rsp[ix + len(self.options.rsp_begin_tag):] + if self.options.rsp_end_tag and self.options.rsp_end_tag in rsp: + ix = rsp.index(self.options.rsp_end_tag) + rsp = rsp[0:ix] + self.rsp = rsp.strip() + + @staticmethod + def get_action_type(topic: str): + """Create a runtime :class:`Action` subclass""" + action_type: Type["Action"] = type(topic, (Action,), {"name": topic}) + return action_type diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index a96aaf1be..625d98675 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -4,6 +4,8 @@ @Time : 2023/5/20 12:15 @Author : alexanderwu @File : memory.py +@Modified By: mashenquan, 2023-8-7. Modified get_by_actions() to support for dynamically generated Action classes + at runtime. """ from collections import defaultdict from typing import Iterable, Type @@ -80,8 +82,12 @@ class Memory: def get_by_actions(self, actions: Iterable[Type[Action]]) -> list[Message]: """Return all messages triggered by specified Actions""" rsp = [] + # Using the `type(obj).__name__` approach to support the runtime creation of requirement classes. + # See `MetaAction.get_action_type()` for more. + class_names = {type(k).__name__: k for k in self.index.keys()} for action in actions: - if action not in self.index: + if type(action).__name__ not in class_names: continue - rsp += self.index[action] + key = class_names[type(action).__name__] + rsp += self.index[key] return rsp diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py index 1a69b9ca7..555bc8cf3 100644 --- a/metagpt/roles/fork_meta_role.py +++ b/metagpt/roles/fork_meta_role.py @@ -4,95 +4,124 @@ @Time : 2023/8/7 @Author : mashenquan @File : fork_meta_role.py -@Desc : 我试图将UML的一些符号概念引入到MetaGPT，使其具备通过符号拼接自由搭建flow的能力。同时我也尝试将这些符号做得配置化和标准化，让flow搭建流程更便捷。这是一个`fork` meta-role demo，实现的是write_teaching_plan功能。 +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the + ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to + make these symbols configurable and standardized, making the process of building flows more convenient. + For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` + This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a + configuration file. """ +import re + +import aiofiles + +from metagpt.actions.meta_action import MetaAction +from metagpt.const import WORKSPACE_ROOT +from metagpt.logs import logger +from metagpt.roles import Role +from metagpt.roles.uml_meta_role_options import MetaActionOptions, UMLMetaRoleOptions +from metagpt.schema import Message -async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): - """Run a startup. Be a teacher in education industry.""" +class ForkMetaRole(Role): + """A `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file""" + def __init__(self, options, **kwargs): + """Initialize a `fork` style meta role - demo_lesson = """ - UNIT 1 Making New Friends - TOPIC 1 Welcome to China! - Section A + :param options: pattern yaml file data + :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + """ + opts = UMLMetaRoleOptions(**options) + global_variables = { + "name": Role.format_value(opts.name, kwargs), + "profile": Role.format_value(opts.profile, kwargs), + "goal": Role.format_value(opts.goal, kwargs), + "constraints": Role.format_value(opts.constraints, kwargs), + "desc": Role.format_value(opts.desc, kwargs), + "role": Role.format_value(opts.role, kwargs) + } + for k, v in kwargs.items(): + if k not in global_variables: + global_variables[k] = v - 1a Listen and number the following names. - Jane Mari Kangkang Michael - Look, listen and understand. Then practice the conversation. - Work in groups. Introduce yourself using - I ’m ... Then practice 1a - with your own hometown or the following places. + super(ForkMetaRole, self).__init__( + name=global_variables["name"], + profile=global_variables["profile"], + goal=global_variables["goal"], + constraints=global_variables["constraints"], + desc=global_variables["desc"], + **kwargs + ) + self.options = options + actions = [] + for m in opts.actions: + for k, v in m.items(): + v = Role.format_value(v, kwargs) + m[k] = v + for k, v in global_variables.items(): + if k not in m: + m[k] = v - 1b Listen and number the following names - Jane Michael Maria Kangkang - 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. - China the USA the UK Hong Kong Beijing + o = MetaActionOptions(**m) + o.set_default_template(opts.templates[o.template_ix]) - 2a Look, listen and understand. Then practice the conversation - Hello! - Hello! - Hello! - Hello! Are you Maria? - No, I’m not. I’m Jane. - Oh, nice to meet you, Jane - Nice to meet you, too. - Hi, Maria! - Hi, Kangkang! - Welcome to China! - Thanks. + act = MetaAction(options=o, llm=self._llm, **m) + actions.append(act) + self._init_actions(actions) + requirement_types = set() + for v in opts.requirement: + requirement_types.add(MetaAction.get_action_type(v)) + self._watch(requirement_types) - 2b Work in groups. Make up a conversation with your own name and the - following structures. - A: Hello! / Good morning! / Hi! I’m ... Are you ... ? - B: ... + async def _think(self) -> None: + """Everything will be done part by part.""" + if self._rc.todo is None: + self._set_state(0) + return - 3a Listen, say and trace - Aa Bb Cc Dd Ee Ff Gg + if self._rc.state + 1 < len(self._states): + self._set_state(self._rc.state + 1) + else: + self._rc.todo = None - 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. - Aa Bb Cc Dd Ee Ff Gg + async def _react(self) -> Message: + ret = Message(content="") + while True: + await self._think() + if self._rc.todo is None: + break + logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") + msg = await self._act() + if ret.content != '': + ret.content += "\n\n\n" + ret.content += msg.content + logger.info(ret.content) + await self.save(ret.content) + return ret - 3c Match the big letters with the small ones. Then write them on the lines. - """ + async def save(self, content): + """Save teaching plan""" + output_filename = self.options.get("output_filename") + if not output_filename: + return + filename = ForkMetaRole.new_file_name(output_filename) + pathname = WORKSPACE_ROOT / "teaching_plan" + pathname.mkdir(exist_ok=True) + pathname = pathname / filename + try: + async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer: + await writer.write(content) + except Exception as e: + logger.error(f'Save failed：{e}') + logger.info(f"Save to:{pathname}") - lesson = "" - if lesson_file is not None and Path(lesson_file).exists(): - async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: - lesson = await reader.read() - logger.info(f"Course content: {lesson}") - if not lesson: - logger.info("No course content provided, using the demo course.") - lesson = demo_lesson - - - - company = SoftwareCompany() - company.hire([(*args, **kwargs)]) - company.invest(investment) - company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) - await company.run(n_round=1) - - -def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): - """ - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - :param idea: lesson filename. - :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. - :param n_round: Reserved. - :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` - :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` - :return: - """ - asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) - - -if __name__ == '__main__': - """ - Formats: - ``` - python write_teaching_plan.py lesson_filename --teaching_language= --language= - ``` - If `lesson_filename` is not available, a demo lesson content will be used. - """ - fire.Fire(main) + @staticmethod + def new_file_name(lesson_title, ext=".md"): + """Create a related file name based on `lesson_title` and `ext`.""" + # Define the special characters that need to be replaced. + illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' + # Replace the special characters with underscores. + filename = re.sub(illegal_chars, '_', lesson_title) + ext + return re.sub(r'_+', '_', filename) \ No newline at end of file diff --git a/metagpt/roles/meta_role.py b/metagpt/roles/meta_role.py deleted file mode 100644 index 1da180355..000000000 --- a/metagpt/roles/meta_role.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : meta_role.py -@Desc : 我试图将UML的一些符号概念引入到MetaGPT，使其具备通过符号拼接自由搭建flow的能力。同时我也尝试将这些符号做得配置化和标准化，让flow搭建流程更便捷。 - 分工参照UML 2.0 activity diagrams: `https://www.uml-diagrams.org/activity-diagrams.html` -""" -from typing import Dict, List - -from metagpt.roles import Role -from pydantic import BaseModel - -class UMLMetaRoleArgs(BaseModel): - role_type: str - name: str = "" - profile: str = "" - goal: str = "" - constraints: str = "" - desc: str = "" - actions: List - -class UMLMetaRole(Role): - """UML activity roles抽象父类""" - - def __init__(self, role_args: Dict): - """""" - self.role_args diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index f79764324..1d65a7f26 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,7 +4,7 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py -@Modified By: mashenquan, 2023-07-27, :class:`Role` + properties. +@Modified By: mashenquan, 2023-8-7, :class:`Role` + properties. """ from __future__ import annotations @@ -286,6 +286,8 @@ class Role: @staticmethod def format_value(value, options): """Fill parameters inside `value` with `options`.""" + if not isinstance(value, str): + return value if "{" not in value: return value @@ -295,7 +297,7 @@ class Role: except KeyError as e: logger.warning(f"Parameter is missing:{e}") for k, v in options.items(): - value = value.replace("{" + f"{k}" + "}", v) + value = value.replace("{" + f"{k}" + "}", str(v)) return value __DEFAULT_OPTIONS__ = { diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index 95d54133b..24ede7402 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -5,7 +5,7 @@ @Author : mashenquan @File : teacher.py """ -from pathlib import Path + import aiofiles diff --git a/metagpt/roles/uml_meta_role_factory.py b/metagpt/roles/uml_meta_role_factory.py new file mode 100644 index 000000000..78f9689a2 --- /dev/null +++ b/metagpt/roles/uml_meta_role_factory.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : uml_meta_role_factory.py +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the + ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to + make these symbols configurable and standardized, making the process of building flows more convenient. + For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` +""" + +from metagpt.roles.fork_meta_role import ForkMetaRole +from metagpt.roles.uml_meta_role_options import UMLMetaRoleOptions + + +class UMLMetaRoleFactory: + """Factory of UML activity role classes""" + + @classmethod + def create_roles(cls, role_configs, **kwargs): + """Generate the flow of the project based on the configuration in the format of config/pattern/template.yaml. + + :param role_configs: `roles` field of template.yaml + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + + """ + roles = [] + for m in role_configs: + opt = UMLMetaRoleOptions(**m) + constructor = cls.CONSTRUCTORS.get(opt.role_type) + if constructor is None: + raise NotImplementedError( + f"{opt.role_type} is not implemented" + ) + r = constructor(m, **kwargs) + roles.append(r) + return roles + + CONSTRUCTORS = { + "fork": ForkMetaRole, + # TODO: add more activity node constructor here.. + } diff --git a/metagpt/roles/uml_meta_role_options.py b/metagpt/roles/uml_meta_role_options.py new file mode 100644 index 000000000..1d0fb322e --- /dev/null +++ b/metagpt/roles/uml_meta_role_options.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : uml_meta_role_options.py +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the + ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to + make these symbols configurable and standardized, making the process of building flows more convenient. + For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` +""" + +from typing import List, Dict + +from pydantic import BaseModel + + +# `startup` field of config/pattern/template.yaml +class StartupConfig(BaseModel): + requirement: str + role: str + investment: float = 3.0 + n_round: int = 3 + + +# config/pattern/template.yaml +class ProjectConfig(BaseModel): + startup: StartupConfig + roles: List[Dict] + + +# element of `actions` field of config/pattern/template.yaml +class MetaActionOptions(BaseModel): + topic: str + name: str = "" + language: str = "Chinese" + template_ix: int = 0 + statements: List[str] = [] + template: str = "" + rsp_begin_tag: str = "" + rsp_end_tag: str = "" + + def set_default_template(self, v): + if not self.template: + self.template = v + + def format_prompt(self, **kwargs): + statements = "\n".join(self.statements) + opts = kwargs.copy() + opts["statements"] = statements + + from metagpt.roles import Role + prompt = Role.format_value(self.template, opts) + return prompt + + +# element of `roles` field of config/pattern/template.yaml +class UMLMetaRoleOptions(BaseModel): + role_type: str + name: str = "" + profile: str = "" + goal: str = "" + role: str = "" + constraints: str = "" + desc: str = "" + templates: List[str] = [] + output_filename: str = "" + actions: List + requirement: List From 76ea924a051232ed6e2d3e4dcb84906f35ac09bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 8 Aug 2023 15:25:26 +0800 Subject: [PATCH 026/398] feat: + unit test --- metagpt/roles/role.py | 5 ++ .../roles/test_uml_meta_role_factory.py | 61 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 tests/metagpt/roles/test_uml_meta_role_factory.py diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1d65a7f26..68baeccf5 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -168,6 +168,11 @@ class Role: """Return role `constraints`, read only""" return self._setting.constraints + @property + def action_count(self): + """Return number of action""" + return len(self._actions) + def _get_prefix(self): """获取角色前缀""" if self._setting.desc: diff --git a/tests/metagpt/roles/test_uml_meta_role_factory.py b/tests/metagpt/roles/test_uml_meta_role_factory.py new file mode 100644 index 000000000..f59a30611 --- /dev/null +++ b/tests/metagpt/roles/test_uml_meta_role_factory.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/8 +@Author : mashenquan +@File : test_uml_meta_role_factory.py +""" +from typing import List, Dict + +from pydantic import BaseModel + +from metagpt.roles.uml_meta_role_factory import UMLMetaRoleFactory + + +def test_create_roles(): + class Inputs(BaseModel): + roles: List + kwargs: Dict + + inputs = [ + { + "roles": [ + { + "role_type": "fork", + "name": "Lily", + "profile": "{teaching_language} Teacher", + "goal": "writing a {language} teaching plan part by part", + "constraints": "writing in {language}", + "role": "You are a {teaching_language} Teacher, named Lily.", + "desc": "", + "output_filename": "teaching_plan_demo.md", + "requirement": ["TeachingPlanRequirement"], + "templates": ["Do 1 {statements}", "Do 2 {statements}"], + "actions": [ + { + "name": "", + "topic": "Title", + "language": "Chinese", + "statements": ["statement 1", "statement 2"]} + ], + "template_ix": 0 + } + ], + "kwargs": { + "teaching_language": "AA", + "language": "BB", + } + } + ] + + for i in inputs: + seed = Inputs(**i) + roles = UMLMetaRoleFactory.create_roles(seed.roles, **seed.kwargs) + assert len(roles) == 1 + assert "{" not in roles[0].profile + assert "{" not in roles[0].goal + assert roles[0].action_count == 1 + + +if __name__ == '__main__': + test_create_roles() From 1526680fc4b867668a84bf3813d02aefa0004044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 8 Aug 2023 18:02:49 +0800 Subject: [PATCH 027/398] feat: + unit test --- .../roles/test_uml_meta_role_options.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tests/metagpt/roles/test_uml_meta_role_options.py diff --git a/tests/metagpt/roles/test_uml_meta_role_options.py b/tests/metagpt/roles/test_uml_meta_role_options.py new file mode 100644 index 000000000..1eb66c50e --- /dev/null +++ b/tests/metagpt/roles/test_uml_meta_role_options.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/8 +@Author : mashenquan +@File : test_uml_meta_role_options.py +""" +from typing import List + +from pydantic import BaseModel + +from metagpt.roles.uml_meta_role_options import MetaActionOptions + + +def test_set_default_template(): + class Inputs(BaseModel): + statements: List + template: str + expect_prompt: str + + inputs = [ + { + "statements": ["Statement: 1", "Statement: 2"], + "template": "{statements}", + "expect_prompt": "Statement: 1\nStatement: 2" + } + ] + + for i in inputs: + seed = Inputs(**i) + opt = MetaActionOptions(topic="", statements=seed.statements) + assert opt.template == "" + opt.set_default_template(seed.template) + assert opt.template == seed.template + kwargs = {} + assert opt.format_prompt(**kwargs) == seed.expect_prompt + + +if __name__ == '__main__': + test_set_default_template() From 25f461b6a4ae2137c463202ce39769a107bbab89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 8 Aug 2023 19:02:57 +0800 Subject: [PATCH 028/398] feat: + unit test --- tests/metagpt/roles/test_fork_meta_role.py | 90 ++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tests/metagpt/roles/test_fork_meta_role.py diff --git a/tests/metagpt/roles/test_fork_meta_role.py b/tests/metagpt/roles/test_fork_meta_role.py new file mode 100644 index 000000000..b2659330d --- /dev/null +++ b/tests/metagpt/roles/test_fork_meta_role.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/8 +@Author : mashenquan +@File : test_fork_meta_role.py +""" +from typing import Dict + +from pydantic import BaseModel + +from metagpt.roles.fork_meta_role import ForkMetaRole + + +def test_creat_role(): + class Inputs(BaseModel): + role: Dict + action_count: int + + inputs = [ + { + "role": { + "role_type": "fork", + "name": "Lily", + "profile": "{teaching_language} Teacher", + "goal": "writing a {language} teaching plan part by part", + "constraints": "writing in {language}", + "role": "You are a {teaching_language} Teacher, named Lily, your goal is writing a {" + "teaching_language} teaching plan part by part, and the constraint is writing in {language}.", + "desc": "", + "output_filename": "teaching_plan_demo.md", + "requirement": ["TeachingPlanRequirement"], + "templates": [ + "Do not refer to the context of the previous conversation records, start the conversation " + "anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[" + "LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" " + "defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the " + "format requirements for your responses;\n\t\"Constraint\" defines the conditions that your " + "responses must comply with.\n\n{statements}\nConstraint: Writing in {language}.\nAnswer options: " + "Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\n[" + "LESSON_BEGIN]\n{lesson}\n[LESSON_END]", + "Do not refer to the context of the previous conversation records, start the conversation " + "anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[" + "LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" " + "defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the " + "format requirements for your responses;\n\t\"Constraint\" defines the conditions that your " + "responses must comply with.\n\nCapacity and role: {role}\nStatement: Write the \"{topic}\" part " + "of teaching plan, WITHOUT ANY content unrelated to \"{topic}\"!!\n{statements}\nAnswer options: " + "Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" " + "tags.\nAnswer options: Using proper markdown format from second-level header " + "format.\nConstraint: Writing in {language}.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END] " + ], + "actions": [ + { + "name": "", + "topic": "Title", + "language": "Chinese", + "statements": [ + "Statement: Find and return the title of the lesson only with \"# \" prefixed, without " + "anything else."], + "template_ix": 0}, + { + "name": "", + "topic": "Teaching Hours", + "language": "Chinese", + "statements": [], + "template_ix": 1, + "rsp_begin_tag": "[TEACHING_PLAN_BEGIN]", + "rsp_end_tag": "[TEACHING_PLAN_END]"} + ] + }, + "action_count": 2 + } + ] + + for i in inputs: + seed = Inputs(**i) + kwargs = { + "teaching_language": "AA", + "language": "BB" + } + role = ForkMetaRole(seed.role, **kwargs) + assert role.action_count == 2 + assert "{" not in role.profile + assert "{" not in role.goal + assert "{" not in role.constraints + + +if __name__ == '__main__': + test_creat_role() From b0209f3d419e2feffa099902ba4f76429963e6f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 8 Aug 2023 20:10:40 +0800 Subject: [PATCH 029/398] feat: + unit test --- tests/metagpt/actions/test_meta_action.py | 51 +++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/metagpt/actions/test_meta_action.py diff --git a/tests/metagpt/actions/test_meta_action.py b/tests/metagpt/actions/test_meta_action.py new file mode 100644 index 000000000..cbaf3456c --- /dev/null +++ b/tests/metagpt/actions/test_meta_action.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/8 +@Author : mashenquan +@File : test_meta_action.py +""" +from typing import Dict + +from pydantic import BaseModel + +from metagpt.actions.meta_action import MetaAction +from metagpt.roles.uml_meta_role_options import MetaActionOptions + + +def test_meta_action_create(): + class Inputs(BaseModel): + options: Dict + kwargs: Dict + expect_class_name: str + expect_prompt: str + + inputs = [ + { + "options": { + "topic": "TOPIC_A", + "name": "A", + "language": "XX", + "template_ix": 0, + "statements": ["Statement A", "Statement B"], + "template": "{statements}", + "rsp_begin_tag": "", + "rsp_end_tag": "" + }, + "kwargs": {}, + "expect_class_name": "TOPIC_A", + "expect_prompt": "\n".join(["Statement A", "Statement B"]), + } + ] + + for i in inputs: + seed = Inputs(**i) + opt = MetaActionOptions(**seed.options) + act = MetaAction(opt, **seed.kwargs) + assert seed.expect_prompt == act.prompt + t = MetaAction.get_action_type(seed.expect_class_name) + assert t.__name__ == seed.expect_class_name + + +if __name__ == '__main__': + test_meta_action_create() From f8bc4e462575622927a2b7b022468bd95a38a8e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 10:48:40 +0800 Subject: [PATCH 030/398] feat: test pass --- config/pattern/write_teaching_plan.yaml | 2 +- ...{fork_meta_role.py => fork_meta_role_write_teaching_plan.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename examples/{fork_meta_role.py => fork_meta_role_write_teaching_plan.py} (100%) diff --git a/config/pattern/write_teaching_plan.yaml b/config/pattern/write_teaching_plan.yaml index 6a05bad96..357717908 100644 --- a/config/pattern/write_teaching_plan.yaml +++ b/config/pattern/write_teaching_plan.yaml @@ -16,7 +16,7 @@ roles: # A project can involve multiple r constraints: "writing in {language}" role: "You are a {teaching_language} Teacher, named Lily, your goal is writing a {teaching_language} teaching plan part by part, and the constraint is writing in {language}." desc: "" - output_filename: "teaching_plan_demo.md" + output_filename: "teaching_plan_demo" requirement: ["TeachingPlanRequirement"] templates: # The template provides a convenient way to generate prompts. After each action selects its respective template, you only need to provide the corresponding variable values. Variable replacement is automatically handled by the framework. - "Do not refer to the context of the previous conversation records, start the conversation anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the format requirements for your responses;\n\t\"Constraint\" defines the conditions that your responses must comply with.\n\n{statements}\nConstraint: Writing in {language}.\nAnswer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END]" diff --git a/examples/fork_meta_role.py b/examples/fork_meta_role_write_teaching_plan.py similarity index 100% rename from examples/fork_meta_role.py rename to examples/fork_meta_role_write_teaching_plan.py From 9e91142c4ef30ef08ff9552a1b57bd954540e6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 13:46:22 +0800 Subject: [PATCH 031/398] fixbug: Align parameters with the parent class --- metagpt/actions/design_api.py | 2 +- metagpt/actions/project_management.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 1447eacc3..48fa7171a 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -135,7 +135,7 @@ class WriteDesign(Action): self._save_prd(docs_path, resources_path, context[-1].content) self._save_system_design(docs_path, resources_path, content) - async def run(self, context): + async def run(self, context, **kwargs): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 89c59dcda..80b891bb8 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -115,7 +115,7 @@ class WriteTasks(Action): requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt' requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) - async def run(self, context): + async def run(self, context, **kwargs): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) self._save(context, rsp) From 9e9e0d56e56ba2206f090bc040c14b580e1254d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 13:55:07 +0800 Subject: [PATCH 032/398] fixbug: Align parameters with the parent class --- metagpt/actions/design_api.py | 1 + metagpt/actions/project_management.py | 1 + 2 files changed, 2 insertions(+) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 48fa7171a..cf23e6ad1 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : design_api.py +@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. """ import shutil from pathlib import Path diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 80b891bb8..16473ff01 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 19:12 @Author : alexanderwu @File : project_management.py +@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. """ from typing import List, Tuple From 35470dcee4d4635b798f03ccd6154467d65cb57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 14:40:43 +0800 Subject: [PATCH 033/398] fixbug: empty string causes aiofiles.open exepctition --- examples/write_teaching_plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index da97a5463..86125f090 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -62,7 +62,7 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * """ lesson = "" - if lesson_file is not None and Path(lesson_file).exists(): + if lesson_file and Path(lesson_file).exists(): async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: lesson = await reader.read() logger.info(f"Course content: {lesson}") From ed56aab79cc90ede26ca4993476e260721093fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 14:54:08 +0800 Subject: [PATCH 034/398] fixbug: empty string causes aiofiles.open exepctition --- config/pattern/write_teaching_plan.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/pattern/write_teaching_plan.yaml b/config/pattern/write_teaching_plan.yaml index 357717908..5b5f2af77 100644 --- a/config/pattern/write_teaching_plan.yaml +++ b/config/pattern/write_teaching_plan.yaml @@ -26,8 +26,10 @@ roles: # A project can involve multiple r topic: "Title" language: "Chinese" statements: # When replacing template variables, multiple statements will be joined into a single string using line breaks. - - "Statement: Find and return the title of the lesson only with \"# \" prefixed, without anything else." + - "Statement: Find and return the title of the lesson only with \"# \" string prefixed, without anything else." template_ix: 0 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Teaching Hours" language: "Chinese" From b786bce4b9daf573f8f5eb4adc78f9f453ea3e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 14:54:30 +0800 Subject: [PATCH 035/398] fixbug: empty string causes aiofiles.open exepctition --- examples/fork_meta_role_write_teaching_plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/fork_meta_role_write_teaching_plan.py b/examples/fork_meta_role_write_teaching_plan.py index 21e3b5f7c..d1a6e0070 100644 --- a/examples/fork_meta_role_write_teaching_plan.py +++ b/examples/fork_meta_role_write_teaching_plan.py @@ -73,7 +73,7 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * """ lesson = "" - if lesson_file is not None and Path(lesson_file).exists(): + if lesson_file and Path(lesson_file).exists(): async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: lesson = await reader.read() logger.info(f"Course content: {lesson}") From 13a91349f78322a7f3df87e36b266ed501398c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 16:37:29 +0800 Subject: [PATCH 036/398] fixbug: cannot find metagpt module --- examples/llm_hello_world.py | 5 ++++- examples/search_google.py | 5 ++++- examples/search_kb.py | 5 ++++- examples/search_with_specific_engine.py | 9 ++++++++- examples/write_teaching_plan.py | 6 +++++- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/examples/llm_hello_world.py b/examples/llm_hello_world.py index 3ba03eea0..329247afc 100644 --- a/examples/llm_hello_world.py +++ b/examples/llm_hello_world.py @@ -4,9 +4,12 @@ @Time : 2023/5/6 14:13 @Author : alexanderwu @File : llm_hello_world.py +@Modified By: mashenquan, 2023-8-9, fix-bug: cannot find metagpt module. """ import asyncio - +from pathlib import Path +import sys +sys.path.append(str(Path(__file__).resolve().parent.parent)) from metagpt.llm import LLM, Claude from metagpt.logs import logger diff --git a/examples/search_google.py b/examples/search_google.py index 9e9521b9c..df45c29ea 100644 --- a/examples/search_google.py +++ b/examples/search_google.py @@ -4,10 +4,13 @@ @Time : 2023/5/7 18:32 @Author : alexanderwu @File : search_google.py +@Modified By: mashenquan, 2023-8-9, fix-bug: cannot find metagpt module. """ import asyncio - +from pathlib import Path +import sys +sys.path.append(str(Path(__file__).resolve().parent.parent)) from metagpt.roles import Searcher diff --git a/examples/search_kb.py b/examples/search_kb.py index b6f7d87a0..449099380 100644 --- a/examples/search_kb.py +++ b/examples/search_kb.py @@ -2,9 +2,12 @@ # -*- coding: utf-8 -*- """ @File : search_kb.py +@Modified By: mashenquan, 2023-8-9, fix-bug: cannot find metagpt module. """ import asyncio - +from pathlib import Path +import sys +sys.path.append(str(Path(__file__).resolve().parent.parent)) from metagpt.const import DATA_PATH from metagpt.document_store import FaissStore from metagpt.logs import logger diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index 7cc431cd4..4423011e4 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -1,5 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Modified By: mashenquan, 2023-8-9, fix-bug: cannot find metagpt module. +""" import asyncio - +from pathlib import Path +import sys +sys.path.append(str(Path(__file__).resolve().parent.parent)) from metagpt.roles import Searcher from metagpt.tools import SearchEngineType diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index 86125f090..5ab7d3ab5 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -1,11 +1,15 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Modified By: mashenquan, 2023-07-27, + `industry` concept +@Modified By: mashenquan, 2023-07-27, + write teaching plan flow demo + """ import asyncio from pathlib import Path +import sys + +sys.path.append(str(Path(__file__).resolve().parent.parent)) import aiofiles import fire from metagpt.logs import logger From 63678de181096b73f6680b36bcc97e38c4dca113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 20:48:25 +0800 Subject: [PATCH 037/398] feat: add more text formatting options --- metagpt/actions/azure_tts.py | 42 +++++++++++++++++++------ tests/metagpt/actions/test_azure_tts.py | 30 +++++++++++++++--- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/metagpt/actions/azure_tts.py b/metagpt/actions/azure_tts.py index f528ba001..3520de8b4 100644 --- a/metagpt/actions/azure_tts.py +++ b/metagpt/actions/azure_tts.py @@ -4,11 +4,13 @@ @Time : 2023/6/9 22:22 @Author : Leo Xiao @File : azure_tts.py +@Modified By: mashenquan, 2023-8-9, add more text formatting options """ from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer from metagpt.actions.action import Action from metagpt.config import Config +from metagpt.const import WORKSPACE_ROOT class AzureTTS(Action): @@ -17,7 +19,7 @@ class AzureTTS(Action): self.config = Config() # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles - def synthesize_speech(self, lang, voice, role, text, output_file): + def synthesize_speech(self, lang, voice, text, output_file): subscription_key = self.config.get('AZURE_TTS_SUBSCRIPTION_KEY') region = self.config.get('AZURE_TTS_REGION') speech_config = SpeechConfig( @@ -29,25 +31,47 @@ class AzureTTS(Action): speech_config=speech_config, audio_config=audio_config) - # if voice=="zh-CN-YunxiNeural": + # More detail: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-voice ssml_string = f""" - - {text} - + {text} """ - synthesizer.speak_ssml_async(ssml_string).get() + return synthesizer.speak_ssml_async(ssml_string).get() + @staticmethod + def role_style_text(role, style, text): + return f'{text}' + + @staticmethod + def role_text(role, text): + return f'{text}' + + @staticmethod + def style_text(style, text): + return f'{text}' if __name__ == "__main__": azure_tts = AzureTTS("azure_tts") + text = """ + 女儿看见父亲走了进来，问道： + + “您来的挺快的，怎么过来的？” + + 父亲放下手提包，说： + + “刚打车过来的，路上还挺顺畅。” + + """ + path = WORKSPACE_ROOT / "tts" + path.mkdir(exist_ok=True, parents=True) + filename = path / "output.wav" azure_tts.synthesize_speech( "zh-CN", "zh-CN-YunxiNeural", - "Boy", - "你好，我是卡卡", - "output.wav") + text=AzureTTS.role_style_text(role="Boy", style="affectionate", text="你好，我是卡卡"), + output_file=str(filename) + ) diff --git a/tests/metagpt/actions/test_azure_tts.py b/tests/metagpt/actions/test_azure_tts.py index b5a333af2..2145f7133 100644 --- a/tests/metagpt/actions/test_azure_tts.py +++ b/tests/metagpt/actions/test_azure_tts.py @@ -4,18 +4,38 @@ @Time : 2023/7/1 22:50 @Author : alexanderwu @File : test_azure_tts.py +@Modified By: mashenquan, 2023-8-9, add more text formatting options """ from metagpt.actions.azure_tts import AzureTTS +from metagpt.const import WORKSPACE_ROOT def test_azure_tts(): azure_tts = AzureTTS("azure_tts") - azure_tts.synthesize_speech( + text = """ + 女儿看见父亲走了进来，问道： + + “您来的挺快的，怎么过来的？” + + 父亲放下手提包，说： + + “Writing a binary file in Python is similar to writing a regular text file, but you'll work with bytes instead of strings.” + + """ + path = WORKSPACE_ROOT / "tts" + path.mkdir(exist_ok=True, parents=True) + filename = path / "girl.wav" + result = azure_tts.synthesize_speech( "zh-CN", - "zh-CN-YunxiNeural", - "Boy", - "你好，我是卡卡", - "output.wav") + "zh-CN-XiaomoNeural", + text=text, + output_file=str(filename)) + + print(result) # 运行需要先配置 SUBSCRIPTION_KEY # TODO: 这里如果要检验，还要额外加上对应的asr，才能确保前后生成是接近一致的，但现在还没有 + + +if __name__ == '__main__': + test_azure_tts() From de610df25d310211c2f235c9a7a79fc4162c219e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 17 Aug 2023 20:41:07 +0800 Subject: [PATCH 038/398] feat: +OAS framework --- metagpt/actions/azure_tts.py | 53 -------- metagpt/tools/azure_tts.py | 114 ++++++++++++++++++ metagpt/tools/hello.py | 27 +++++ metagpt/tools/metagpt_openapi_svc.py | 20 +++ metagpt/utils/common.py | 13 ++ requirements.txt | 4 +- spec/metagpt_openapi.yaml | 64 ++++++++++ spec/openapi.yaml | 35 ++++++ .../{actions => tools}/test_azure_tts.py | 7 +- 9 files changed, 282 insertions(+), 55 deletions(-) delete mode 100644 metagpt/actions/azure_tts.py create mode 100644 metagpt/tools/azure_tts.py create mode 100644 metagpt/tools/hello.py create mode 100644 metagpt/tools/metagpt_openapi_svc.py create mode 100644 spec/metagpt_openapi.yaml create mode 100644 spec/openapi.yaml rename tests/metagpt/{actions => tools}/test_azure_tts.py (67%) diff --git a/metagpt/actions/azure_tts.py b/metagpt/actions/azure_tts.py deleted file mode 100644 index f528ba001..000000000 --- a/metagpt/actions/azure_tts.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/6/9 22:22 -@Author : Leo Xiao -@File : azure_tts.py -""" -from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer - -from metagpt.actions.action import Action -from metagpt.config import Config - - -class AzureTTS(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) - self.config = Config() - - # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles - def synthesize_speech(self, lang, voice, role, text, output_file): - subscription_key = self.config.get('AZURE_TTS_SUBSCRIPTION_KEY') - region = self.config.get('AZURE_TTS_REGION') - speech_config = SpeechConfig( - subscription=subscription_key, region=region) - - speech_config.speech_synthesis_voice_name = voice - audio_config = AudioConfig(filename=output_file) - synthesizer = SpeechSynthesizer( - speech_config=speech_config, - audio_config=audio_config) - - # if voice=="zh-CN-YunxiNeural": - ssml_string = f""" - - - - {text} - - - - """ - - synthesizer.speak_ssml_async(ssml_string).get() - - -if __name__ == "__main__": - azure_tts = AzureTTS("azure_tts") - azure_tts.synthesize_speech( - "zh-CN", - "zh-CN-YunxiNeural", - "Boy", - "你好，我是卡卡", - "output.wav") diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py new file mode 100644 index 000000000..19d7c2ab1 --- /dev/null +++ b/metagpt/tools/azure_tts.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : azure_tts.py +@Desc : azure TTS openapi, which provides text-to-speech functionality +""" +from pathlib import Path +from uuid import uuid4 +import base64 +import sys + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.utils.common import initalize_enviroment +from metagpt.logs import logger + +from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer +import os + + +class AzureTTS: + """Azure Text-to-Speech""" + + def __init__(self, subscription_key, region): + """ + :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` + :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. + """ + self.subscription_key = subscription_key if subscription_key else os.environ.get('AZURE_TTS_SUBSCRIPTION_KEY') + self.region = region if region else os.environ.get('AZURE_TTS_REGION') + + # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles + def synthesize_speech(self, lang, voice, text, output_file): + speech_config = SpeechConfig( + subscription=self.subscription_key, region=self.region) + speech_config.speech_synthesis_voice_name = voice + audio_config = AudioConfig(filename=output_file) + synthesizer = SpeechSynthesizer( + speech_config=speech_config, + audio_config=audio_config) + + # More detail: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-voice + ssml_string = "" \ + f"{text}" + + return synthesizer.speak_ssml_async(ssml_string).get() + + @staticmethod + def role_style_text(role, style, text): + return f'{text}' + + @staticmethod + def role_text(role, text): + return f'{text}' + + @staticmethod + def style_text(style, text): + return f'{text}' + + +# Export +def openapi_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): + """openapi/tts/azsure + For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + + :param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param voice: For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`, `https://speech.microsoft.com/portal/voicegallery` + :param style: Speaking style to express different emotions like cheerfulness, empathy, and calm. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param role: With roles, the same voice can act as a different age and gender. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param text: Text to convert + :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` + :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. + :return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string. + + """ + if not text: + return "" + + if not lang: + lang = "zh-CN" + if not voice: + voice = "zh-CN-XiaomoNeural" + if not role: + role = "Girl" + if not style: + style = "affectionate" + if not subscription_key: + subscription_key = os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") + if not region: + region = os.environ.get("AZURE_TTS_REGION") + + xml_value = AzureTTS.role_style_text(role=role, style=style, text=text) + tts = AzureTTS(subscription_key=subscription_key, region=region) + filename = Path(__file__).resolve().parent / (str(uuid4()).replace("-", "") + ".wav") + try: + tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename)) + with open(str(filename), mode="rb") as reader: + data = reader.read() + base64_string = base64.b64encode(data).decode('utf-8') + filename.unlink() + except Exception as e: + logger.error(f"text:{text}, error:{e}") + return "" + + return base64_string + + +if __name__ == "__main__": + initalize_enviroment() + + v = openapi_azsure_tts("测试，test") + print(v) diff --git a/metagpt/tools/hello.py b/metagpt/tools/hello.py new file mode 100644 index 000000000..686fba34b --- /dev/null +++ b/metagpt/tools/hello.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/2 16:03 +@Author : mashenquan +@File : hello.py +@Desc : Implement the OpenAPI Specification 3.0 demo and use the following command to test the HTTP service: + + curl -X 'POST' \ + 'http://localhost:8080/openapi/greeting/dave' \ + -H 'accept: text/plain' \ + -H 'Content-Type: application/json' \ + -d '{}' +""" + +import connexion + + +# openapi implement +def post_greeting(name: str) -> str: + return f"Hello {name}\n" + + +if __name__ == "__main__": + app = connexion.AioHttpApp(__name__, specification_dir='../../spec/') + app.add_api("openapi.yaml", arguments={"title": "Hello World Example"}) + app.run(port=8080) diff --git a/metagpt/tools/metagpt_openapi_svc.py b/metagpt/tools/metagpt_openapi_svc.py new file mode 100644 index 000000000..94d935625 --- /dev/null +++ b/metagpt/tools/metagpt_openapi_svc.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : metagpt_openapi_svc.py +@Desc : MetaGPT OpenAPI REST API service +""" +from pathlib import Path +import sys +import connexion +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.utils.common import initalize_enviroment + +if __name__ == "__main__": + initalize_enviroment() + + app = connexion.AioHttpApp(__name__, specification_dir='../../spec/') + app.add_api("metagpt_openapi.yaml") + app.run(port=8080) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 7f090cf63..b15c1d186 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -4,14 +4,18 @@ @Time : 2023/4/29 16:07 @Author : alexanderwu @File : common.py +@Modified By: mashenquan, 2023-8-17, add `initalize_enviroment()` to load `config/config.yaml` to `os.environ` """ import ast import contextlib import inspect import os import re +from pathlib import Path from typing import List, Tuple +import yaml + from metagpt.logs import logger @@ -254,3 +258,12 @@ def parse_recipient(text): pattern = r"## Send To:\s*([A-Za-z]+)\s*?" # hard code for now recipient = re.search(pattern, text) return recipient.group(1) if recipient else "" + + +def initalize_enviroment(): + """Load `config/config.yaml` to `os.environ`""" + yaml_file_path = Path(__file__).resolve().parent.parent.parent / "config/config.yaml" + with open(str(yaml_file_path), "r") as yaml_file: + data = yaml.safe_load(yaml_file) + for k, v in data.items(): + os.environ[k] = str(v) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c18145b98..eef7464ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,4 +36,6 @@ anthropic==0.3.6 typing-inspect==0.8.0 typing_extensions==4.5.0 libcst==1.0.1 -qdrant-client==1.4.0 \ No newline at end of file +qdrant-client==1.4.0 +connexion[swagger-ui] +aiohttp_jinja2 \ No newline at end of file diff --git a/spec/metagpt_openapi.yaml b/spec/metagpt_openapi.yaml new file mode 100644 index 000000000..0bb6ae7bf --- /dev/null +++ b/spec/metagpt_openapi.yaml @@ -0,0 +1,64 @@ +openapi: "3.0.0" + +info: + title: "MetaGPT Export OpenAPIs" + version: "1.0" +servers: + - url: "/openapi" + +paths: + /tts/azsure: + post: + summary: "Convert Text to Base64-encoded .wav File Stream" + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + operationId: azure_tts.openapi_azsure_tts + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - text + properties: + text: + type: string + description: Text to convert + lang: + type: string + description: The language code or locale, e.g., en-US (English - United States) + default: "zh-CN" + voice: + type: string + description: "Voice style, see: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts), [Voice Gallery](https://speech.microsoft.com/portal/voicegallery)" + default: "zh-CN-XiaomoNeural" + style: + type: string + description: "Speaking style to express different emotions. For more details, checkout: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + default: "affectionate" + role: + type: string + description: "Role to specify age and gender. For more details, checkout: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + default: "Girl" + subscription_key: + type: string + description: "Key used to access Azure AI service API, see: [Azure Portal](https://portal.azure.com/) > `Resource Management` > `Keys and Endpoint`" + default: "" + region: + type: string + description: "Location (or region) of your resource, see: [Azure Portal](https://portal.azure.com/) > `Resource Management` > `Keys and Endpoint`" + default: "" + responses: + '200': + description: "Base64-encoded .wav file data if successful, otherwise an empty string." + content: + application/json: + schema: + type: object + properties: + result: + type: string + '400': + description: Bad Request + '500': + description: Bad Request \ No newline at end of file diff --git a/spec/openapi.yaml b/spec/openapi.yaml new file mode 100644 index 000000000..bc291b7db --- /dev/null +++ b/spec/openapi.yaml @@ -0,0 +1,35 @@ +openapi: "3.0.0" + +info: + title: Hello World + version: "1.0" +servers: + - url: /openapi + +paths: + /greeting/{name}: + post: + summary: Generate greeting + description: Generates a greeting message. + operationId: hello.post_greeting + responses: + 200: + description: greeting response + content: + text/plain: + schema: + type: string + example: "hello dave!" + parameters: + - name: name + in: path + description: Name of the person to greet. + required: true + schema: + type: string + example: "dave" + requestBody: + content: + application/json: + schema: + type: object \ No newline at end of file diff --git a/tests/metagpt/actions/test_azure_tts.py b/tests/metagpt/tools/test_azure_tts.py similarity index 67% rename from tests/metagpt/actions/test_azure_tts.py rename to tests/metagpt/tools/test_azure_tts.py index b5a333af2..667e32d01 100644 --- a/tests/metagpt/actions/test_azure_tts.py +++ b/tests/metagpt/tools/test_azure_tts.py @@ -4,8 +4,13 @@ @Time : 2023/7/1 22:50 @Author : alexanderwu @File : test_azure_tts.py +@Modified By: mashenquan, 2023-8-17, move to `tools` folder. """ -from metagpt.actions.azure_tts import AzureTTS +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.tools.azure_tts import AzureTTS def test_azure_tts(): From eb232efdfc438c0a4425fca9f6ad48f23f9825ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 17 Aug 2023 20:55:53 +0800 Subject: [PATCH 039/398] feat: rename --- metagpt/tools/azure_tts.py | 4 ++-- .../{metagpt_openapi_svc.py => metagpt_oas3_api_svc.py} | 6 +++--- spec/{metagpt_openapi.yaml => metagpt_oas3_api.yaml} | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) rename metagpt/tools/{metagpt_openapi_svc.py => metagpt_oas3_api_svc.py} (77%) rename spec/{metagpt_openapi.yaml => metagpt_oas3_api.yaml} (96%) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 19d7c2ab1..035a85108 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -61,8 +61,8 @@ class AzureTTS: # Export -def openapi_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): - """openapi/tts/azsure +def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): + """oas3/tts/azsure For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` :param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` diff --git a/metagpt/tools/metagpt_openapi_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py similarity index 77% rename from metagpt/tools/metagpt_openapi_svc.py rename to metagpt/tools/metagpt_oas3_api_svc.py index 94d935625..921629d8c 100644 --- a/metagpt/tools/metagpt_openapi_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -3,8 +3,8 @@ """ @Time : 2023/8/17 @Author : mashenquan -@File : metagpt_openapi_svc.py -@Desc : MetaGPT OpenAPI REST API service +@File : metagpt_oas3_api_svc.py +@Desc : MetaGPT OpenAPI Specification 3.0 REST API service """ from pathlib import Path import sys @@ -16,5 +16,5 @@ if __name__ == "__main__": initalize_enviroment() app = connexion.AioHttpApp(__name__, specification_dir='../../spec/') - app.add_api("metagpt_openapi.yaml") + app.add_api("metagpt_oas3_api.yaml") app.run(port=8080) diff --git a/spec/metagpt_openapi.yaml b/spec/metagpt_oas3_api.yaml similarity index 96% rename from spec/metagpt_openapi.yaml rename to spec/metagpt_oas3_api.yaml index 0bb6ae7bf..5a3e6923b 100644 --- a/spec/metagpt_openapi.yaml +++ b/spec/metagpt_oas3_api.yaml @@ -4,14 +4,14 @@ info: title: "MetaGPT Export OpenAPIs" version: "1.0" servers: - - url: "/openapi" + - url: "/oas3" paths: /tts/azsure: post: summary: "Convert Text to Base64-encoded .wav File Stream" description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - operationId: azure_tts.openapi_azsure_tts + operationId: azure_tts.oas3_azsure_tts requestBody: required: true content: From 60245fbe902287cc40ea0643d7764da0f50da29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 17 Aug 2023 21:51:50 +0800 Subject: [PATCH 040/398] feat: +openai text-to-image --- metagpt/tools/azure_tts.py | 6 +- metagpt/tools/openai_text_2_image.py | 100 +++++++++++++++++++++++++++ spec/metagpt_oas3_api.yaml | 42 ++++++++++- 3 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 metagpt/tools/openai_text_2_image.py diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 035a85108..5d0001b27 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -4,7 +4,7 @@ @Time : 2023/8/17 @Author : mashenquan @File : azure_tts.py -@Desc : azure TTS openapi, which provides text-to-speech functionality +@Desc : azure TTS OAS3 api, which provides text-to-speech functionality """ from pathlib import Path from uuid import uuid4 @@ -69,7 +69,7 @@ def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key :param voice: For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`, `https://speech.microsoft.com/portal/voicegallery` :param style: Speaking style to express different emotions like cheerfulness, empathy, and calm. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` :param role: With roles, the same voice can act as a different age and gender. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` - :param text: Text to convert + :param text: The text used for voice conversion. :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. :return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string. @@ -110,5 +110,5 @@ def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key if __name__ == "__main__": initalize_enviroment() - v = openapi_azsure_tts("测试，test") + v = oas3_azsure_tts("测试，test") print(v) diff --git a/metagpt/tools/openai_text_2_image.py b/metagpt/tools/openai_text_2_image.py new file mode 100644 index 000000000..3d2a2bbfc --- /dev/null +++ b/metagpt/tools/openai_text_2_image.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : openai_text_2_image.py +@Desc : OpenAI Text-to-Image OAS3 api, which provides text-to-image functionality. +""" +import base64 +import os +import sys +from pathlib import Path +from typing import List + +import requests +from pydantic import BaseModel + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.utils.common import initalize_enviroment +from metagpt.logs import logger + + +class OpenAIText2Image: + def __init__(self, openai_api_key): + """ + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + """ + self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') + + def text_2_image(self, text, size_type="1024x1024"): + """Text to image + + :param text: The text used for image conversion. + :param size_type: One of ['256x256', '512x512', '1024x1024'] + :return: The image data is returned in Base64 encoding. + """ + + class ImageUrl(BaseModel): + url: str + + class ImageResult(BaseModel): + data: List[ImageUrl] + created: int + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.openai_api_key}" + } + data = {"prompt": text, "n": 1, "size": size_type} + try: + response = requests.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) + response.raise_for_status() # Raise an exception for 4xx or 5xx responses + result = ImageResult(**response.json()) + except requests.exceptions.RequestException as e: + logger.error(f"An error occurred:{e}") + return "" + if len(result.data) > 0: + return OpenAIText2Image.get_image_data(result.data[0].url) + return "" + + @staticmethod + def get_image_data(url): + """Fetch image data from a URL and encode it as Base64 + + :param url: Image url + :return: Base64-encoded image data. + """ + try: + response = requests.get(url) + response.raise_for_status() # Raise an exception for 4xx or 5xx responses + image_data = response.content + base64_image = base64.b64encode(image_data).decode("utf-8") + return base64_image + + except requests.exceptions.RequestException as e: + logger.error(f"An error occurred:{e}") + return "" + + +# Export +def oas3_openai_text_2_image(text, size_type: str = "1024x1024", openai_api_key=""): + """Text to image + + :param text: The text used for image conversion. + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + :param size_type: One of ['256x256', '512x512', '1024x1024'] + :return: The image data is returned in Base64 encoding. + """ + if not text: + return "" + if not openai_api_key: + openai_api_key = os.environ.get("OPENAI_API_KEY") + return OpenAIText2Image(openai_api_key).text_2_image(text, size_type=size_type) + + +if __name__ == "__main__": + initalize_enviroment() + + v = oas3_openai_text_2_image("Panda emoji") + print(v) diff --git a/spec/metagpt_oas3_api.yaml b/spec/metagpt_oas3_api.yaml index 5a3e6923b..70c15d590 100644 --- a/spec/metagpt_oas3_api.yaml +++ b/spec/metagpt_oas3_api.yaml @@ -59,6 +59,44 @@ paths: result: type: string '400': - description: Bad Request + description: "Bad Request" '500': - description: Bad Request \ No newline at end of file + description: "Internal Server Error" + + /txt2img/openai: + post: + summary: "Convert Text to Base64-encoded Image Data Stream" + operationId: openai_text_2_image.oas3_openai_text_2_image + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + text: + type: string + description: "The text used for image conversion." + size_type: + type: string + enum: ["256x256", "512x512", "1024x1024"] + default: "1024x1024" + description: "Size of the generated image." + openai_api_key: + type: string + default: "" + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + responses: + '200': + description: "Base64-encoded image data." + content: + application/json: + schema: + type: object + properties: + image_data: + type: string + '400': + description: "Bad Request" + '500': + description: "Internal Server Error" \ No newline at end of file From 2513cca46b4c51d0f4adb9a1a5dce637c0087a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 10:57:52 +0800 Subject: [PATCH 041/398] feat: +ai-plugin --- .well-known/MetaGPT-logo.png | Bin 0 -> 50622 bytes .well-known/ai-plugin.json | 18 ++++++++++++++++++ {spec => .well-known}/metagpt_oas3_api.yaml | 4 +++- {spec => .well-known}/openapi.yaml | 0 metagpt/tools/azure_tts.py | 2 +- metagpt/tools/hello.py | 2 +- metagpt/tools/metagpt_oas3_api_svc.py | 6 +++--- metagpt/tools/openai_text_2_image.py | 4 ++-- metagpt/utils/common.py | 4 ++-- 9 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 .well-known/MetaGPT-logo.png create mode 100644 .well-known/ai-plugin.json rename {spec => .well-known}/metagpt_oas3_api.yaml (96%) rename {spec => .well-known}/openapi.yaml (100%) diff --git a/.well-known/MetaGPT-logo.png b/.well-known/MetaGPT-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..159517fcd4f62049f43eec4db62e1770d189ae89 GIT binary patch literal 50622 zcmeEt^;eVs`~Of0QNkdUvJh!SNeKl(r9@%W2x(!A(MSyy0Ra&N0g)0VH^#_~+@!l> zz!=?QG$Y2~v)A{}_@4I<_qp%gXXk9s>v_fFdR&iZq=Ei(HrDH`004mP<%?%;0D!aq z{<~P1=x>fjI8*7j^X@OqyZ`_Ro&PR|9F?R3`b!3{H_z1pMLqm0^bhBpp6WdX07_yn z9NRDg0L_LkpFK5t&#*>h{%rvqrfm5v{<#tNDJGsN$t0VJ?}qSOD9dZ5cbA_Sm|u>1 zbo0i5&gB5ZFE_qK-FaKj%5Y91^TX9(NSs#56V`jnsbf=XUcP43F3tAr?zElLnT5&6 z3tcIZ^OH^^<6x&;P5RFR0Qr^S|L<;a_T1b59-c^Yeqi|Tq2dYPzm))n&M@%)cQ?53 zzc2jH4*&Cq|HZ}s@&i2&{4Y=buN?fZ9Q>~w{I4ARe^m~OqW}Qoe6upg_PJQKjF7HO zHMP@XYtWI*cZt)btyU*4YbUdlG3&|d-+MUKvA0c3wz?#OR_0gU2Y>c@9PfsXT$cc| zS?Pqc35lVsz;PFa9#}B33635qXZxgPo%y}FeH!?~eJrH(ZmuQ41ca(hJRR;nvS?Tk_Po74Tuw6Pm&YFgj; zuVE*HcDfk>#jck=FuWBV?TDi-9DRGd-WF0q!C8=^`s0fg@jrQCWt`q+x2EB5U2zx; znFJC(O=Gs4+Zb#^LCeoTVYMSZhYKeGfz%ON_TH7>-;uXe?-4Rt2#{0#Yf3=^WA;0k zw%pV@v&lN*3xv|i ztYe!7yz)Lt12x=+sMq90V?@>6E`UO=DPnfrhr$_LY-hthaTvMaTumh0mPazheH+neG6e~4Z!=EzDBRR>#7 zcC0m^z=3E6MUGyn(M1MDari&~nsUT-s@!V$g<2b;``=!bB3y_@x`t~#1v0oX343_> z@A=diYh!AgR;tVpb7xN!-R7*W4x`~V);E~|&*QJIkVh%Td9&H3id*&A5 zxokY@2TcMY2jntx$7S(Ge?-~$OKOZgJ{X-Wi5WBaAS~;(n0^=%{V=yTuGLB9?lF8okst^O-5f2kGgG?= zPzNv>*s1PXd3DQ*JHPnGT49#UWLi+r>vqOlP?>gp(y?UW^=&BBm;}?~2BnpavI^v? zMKG9ZF0+M}9nb9^!rfmVcH^%S`oP!Mk0+F7x_}YG0W-&A`C^`p={QUA+F>*_(Ib=k zq1NXL+PXm}RdJ_Cc~ie;(L4qB7DhTHAW#2dP9PzAqDd@|7C5iP=Vpjbd3!2;_?TA@k00R^|#jNeaXFJVmE8KA>AbnBv9%O#R)elua*Ih=|R-S za0bj7En^3&ZE@_QNO52CaE$R2Hz~QS1*HtH+y3lXw)`VEc_8>kty+C{HAj&#PXeYb z&IZN(_T-q+bA~aw{L=;!c`7JWj!PWP2HT-Vjn5`*6<7nKx%~Hl}6-&WbChIcX6EX zo1e>t`;6#7W_b%j*~`Wvj;=pq_1L2{^edfF^Y(+XPTS3%XrGkKXO@#Sl{bYBmS6a9 zJeN!E55(#isQL9Ar=%StpyVqQMls7GOht;Mm2>@Bw}s za5i7Unw&1XhHHfC9YA`O*QIOYaMoDGcT{8E z$#Niz-YlN!a91kOg zv-aH_THKH38n;Kiz3o9PX03g&M!E8->_AaBkv*Z1uY^pf0LmNfjapG6>o7kwzQHmJ zpK})dsBxSW$HVrt4p+DooH7`9T;zv)&3H%GT^`uz`9&jCz*Wz{MN<|VQUUdT30Kv$?+E}pRJ^H1!alLz?E{dqC# zv-}a}4mR%+!C}5OW>7C9-H+>K)azVMiG^HAw?^eAZb&9H14w78ISADbO2)7`yZK(l zYki6O3Q74?}a0nB6mvEAp0q;=&3t3K9$``TrY;>4?&!KBjb z-iha|h=NE$W(-`AWMiiOcsELmlc|ry@(&Q8FMZBhhq*1AQDG>=r=}RIk71;$z=Ao4 z3H(*$=wlFjAd;t6v zCt-}68-F`|E1m16LZPfL=E3^tp`@ilK-~=TM$o+P_Lhnqmv?N<#<8aOfz?lpvJMNj zh4xphVj*d9yW9ywkzYSdWttroBY%Vb%7M!yPIE=xxcJ+I4hwd_TAjy+@xJc_PNIJ! z&SasuN#vL&@(k$nsO^(P0i#4zR)u7juW2Iu>eBF(ymO>-Z!I3(U9&DnLct%1|0JR>}toY?|QPc0W9+D-Y9JZCQ zeaQ?u`lP>(qd zXcRiQ(rp7lUZrjxo$o$uatx{CsIp-#cZ+drqF41^tuPO3Ms~s#2bbT8d>J|TmFj8B zLJcX5G{WmOqz{Ih4qsLY8wM0BxADZzh-hH+z9pF|Vv-5kn@^v+Ns=yqHpw3=%ZQu% zrmBN|IM>?DSY%nnQYDr4%lm!euE}ts0@-YX(N(+N`1sNCz|q^_8e2@UckkOmQ{5P* zK5@FER2%!P&rzq!Up;n%dL>m-j7^?51|M16`$|3F22R+1Dr5V1*C?5){-na1s_pvP znH)WAACmH8alS!gaW65krLrz7@OJL1SrE-y8>w%H=c{ada-4CYF74%A<3$e}i z-Tuug-UP`^l~tnW^mowtBoyNgPrCz8u%1-ivXWWxTN~&5z_h-J-J3&KL(28aMk{3V zz`lBlcWP+C(bQ}%t2y|;=Gz4~){1l*WCycM#0WWdq4xpa;%DxED6&V?p;y5OT-_}r zalJsmF6o_JUrb|>dLWAZJ6gC(n!Q}C;m?I~^qX;S$J&bB54;p<0URu zMS)?Kpz`s_dpm}|_^>i-Bm4;Ta`?D!`!XC&cf}5Omhu(8Gbg_OR_6^1?fyx+B^21L zxJzD+aFeyKvk9I5cN_g9mRyqcc8offe)@`C(ftkL=&|(qA~Tlacvq((*J^T9d zoFJhG>-jZP>M4x>0k3d{qu~!eMrbERiCM6&JCMrEtN_*s^G!V1Z z!v@_Fz-OJn0Zyizp@21b4OX&hX5^HcWFP&O55I6kS(Y%0%P9|jqha!S*!MYGg_da( zsyAk;^=(?%2uVUQM4m;5ID+H8`J(3jMBC836Yc0g+nr^pqyuEssQ199I;8+#oqs+& z%9~UgOQwt%%O0ect`{?W%8_>(x#HROG!K1d6J?a@dHsb~(Kbk;UlQW*qg))}0wp-) zs`P0W%0x&eZhsJna1iF2KMwEatt@%d&&~Fmj~>_FY=m`J1XJDM`dagopUg4|RmS#H zGHF%hPrE-Bb>o9SYJl9W3%eY#s{|HE=jDb%!dV4zv2scP^=EJlaOh}46=?RYv3UDF z5pQx48}Audic9}2+cq=7R}!OC$i_~%CA$PrHl;`Xmm%~ zK1ud)S^{P3t>}f>ni|(;s;qx_iuzHfT2>Xb_yu!8$?s58zGE!UTKK}E^p6wsq|W+b zRC5f*nG%?)9!M&~S(ZIIKQ1JWGjWbD7ph)Bbe+^jy`2~fn63|HUJ{ZkKb&5a-qHHX z+wY5`_iw^e1o@q{DTkxK$x7x)T-0j4pZ7na-vKN`0T ztSEc@;7<4?+Q!t4wVVSp6BwB|VY&kOOqfqx)#5gZWFU+VTn2KoIRuhLP+_*ZX7{rKi%yUbWjWXWaFhIy}9DoQ*;p9OW*Za%v+biY|Oh zG{01bgO;+rsfrm}-UP?PJ%7p(AI0l_mIB6|8nU2geCOM~A1-yvPbmi>L)W+8o?>@q z4(geOelQEZzj0bEe|i#lxCRcSk}-yCOvM9qM<_I^1MThE@^)$4$c)Gi-zXd~DBLEO z-QTN{E*FUwG8i@$wlF@|fVJiO%UMlTcNpfV+ol&88e})@#@M_3$X>T%W9&E6p#aM% zTO^aWaPOEv8QDLo$d$B*^Pm-FoJoJU0v>9!%^9^ui8lqPduss4i0W^)J|2897C!iJ zTZ{Firc?SPH!bjJn2vyXwC{DN4wz5adzPJYHS7|OER4ME7)^w3pD(w~4$ho*d^0>8 z$dK1qI&^LYre0RpOW*SC4`4N|nCpp9Vp(mdn}{nfdS=w^&=FT;d(ZI&_lLxo|F(IV z3`gaGBFfMkgOoYmiEM8(3+ahui$xJNC)kD+&$k^h#4>}4=5I178g#4q{@4=y4-99m zW;V{;k3BeuQRT9*tEiX}Y{cj+J*`5$0$TF9Si1M43qg6o2*1lGorZsiCC5tB;nwnn z=FL7pA~bJ4_I2$@GdX(XU4n8;syr+vZEr!KfC=|mu`DyGkwe)K=8H!~cZugD+54n; zpn??3!+*o!m50A9eFzd3R`A7DxZBBT*Z`+QopAgkjl{8K(Y};%>B>F-gD*3sol=k6 zdme`M{YLB zBA7+u(Ev&NrsDYqHvyEOKCt{L>N31808-3R9_&Buny2AH;>xjcSca9D8rs z@3z*XQaIx?OuWz183we@Vark!uXt61wl}dv%R5)P^NS6d^-B?5TkvJP-ojbkiV2k& z<`_jP9D1ouGd;N`8I)@Ad8<8baekJ^tbv2~{0J=XJL$f9Zz&X;<0 zbpLwzt7}5xG@m1M_b&RDO-ymoRSe@Y3{mB5x{hKcBd`Q04(W<(M9VmK<(bd8e3sCm zY>3zN;4u;;S^6SSC)KsSqs*#PE=aX~Oi?AytN7;omao?im#nM|T{AJn*>5lV0mhH$ z#fhusWZSnxG;bcayNlM{MXOb_{$e7IGPxRcdiV=KxZ#y-CT11&zFa&Ob~@$fCHs+} zRBX`W>@MOfi~$L~e7|lC7~QBCwpXiJ+!f(CR!neTa)=_V$~LM7$?r(a!b=Znvqw(t z-eB(%sVdV(1pY}Ll=TjI2#YiWhBf}a|NYiQDv5~5xAuClwPmsIR+401_q3#yfv*F0 zMLtoM8T)3{b>`F3!o(m>%*4vG>@y@xo!~WCm&qBS{qkMJNZ=_2b0WtyHShU4_H}iv z)dm8JpIls1chBceHIBP}FGBFd?Ak-$jbyG{r&omBA;r6yd-o9w-6##T5L%CbdIb}e zJ4cYe8mM@Y%3xM!p2gf3NXHz;EY!|1IuB&}T{Pb#ADFg2(R+Q0(@pcpn%SagjS_Z0 zEuvp}=n$}$D^;`7Tk>WN<&}k*X+&}qvGot;(H3mTP(fwt(2as{khH{Q{9TsV<-ZZL2vwv}x|gnKEdk+rhMynF7}BFPiD11(2dOw}a{_dm2S-npR_F_ynuNlgN)1m$}sB7XkQt+P9;-5GR_RfXK zfo}BNSnS8p5m#WK+TfAGJ=C$3DAg|xB80BT7d<5p;}TpehQe)5v(A+vV@tra1!t3=(pxUQ^^rT z;2p!K#zW7RRpl9#ismHYO)0NIypg}g*YIm-?7DdHhDPC(cxAsqZ)X$Q*;q`% z`l+Iu_|O-U4J29RO732v|t!Mw@xT?LehSDs&7cc`baqN(ol z@aN^tysHhRYJRVmW~bufHl3F8G;#ia>phQvpiU(XlVmaT%4wm}Q>(nrU_Uhr)-*D3 zB%965@-!aOlaplvbMbz7&3)Jv!A}YfX`aHC@AabzI30Zjk58`hE?~1a51l53ALdSp zS*QD2k~aQ2?Rb49PaaLcTf=8c>{=IXM#JZYz>=&#Ufk*HmM{f#9o*$mx`~+$6QqDu5m}9u) zdTRKrgt9bHRtgwc_U5;ALk^0cWK7c9K02g$)CtP3Kd`7(onbqN$%F<4efJi6i(`6x zMdf|ryFA}|v)S@@9+y`>GpAG`p%+QNO#H}OKfm*XkW_jLKCjNi!M_8S}d5Px0K z!s>tGS{09%Xq|47ZdJnOww!K4Zo{fC#C5~;8$7mtQiEK)Npn|_(=~|_1t7;D6REU- z$gbKQsV&fh3i4yu*i{PoH`~5)a)e96E}SWsvv7rW>T@q{Z@W#CRBA z$@n$%WZ>H&1|>rRZ*ycfqDeCP8AgVOjvOu_{SPvYB=7E*m@9E z@%@p6j;qf%sLVlMT9w4&XGxmCTWS(_35U2VmaT1-1y$n-gU8e1cjc$T6no_W9@ z>f-9bUO(h%BbQL=i|5E4F#Pk7Cn@16FvTszxYZcaa}$s(@-&qs1B8H&FB0Kbm&BUL;wWIkIe^ISlN(<1RVb=PJl{+&DV_dacK`5&aKg*;VoODgCqI^+H(rzg)`UL(~n{Q#M?#+T@8 z>5Z|oqZ7~H5;Y)#amva8l(5XCC3PJL@frY1z3e7V#YCK(Uav*z;Vwi_#`Nel{RhOns z<<9ivt$YLHdV8x5um>f}8tzd2FBzVT7?9A^sz`or79~flA!`7k4yk%$QTtQ1&D`aa zx1#QTjpZ9d^w>HXIZdrH-VDx@IL{X(U9~Az<-W?LiHU^bjTtccn^Npt%L<9r{7rLS z&-ZVgJ8No8M;A%v!*^A9WVl>uad`@TqUc9?B+CRqnh`hgI-yXyTrWbbm<{Q+YPgX&-!4;;v_r84VFZmTo-bqnS=h`eZ18Bkf}k} z)e$puRwdUZ%7V+O0--h-k^ip!$_kv3h3kun$l{N}csUQuebgp@&R>(?0|>Shx9H-QIE z@AQ%D(j{-wRh?AU#1p)yO_Y}5zn=dJK-A zY7+!M`nA1Tl$$L%w(RK3R%Xd_5s-chaCZ3%S*$2TLg!TsJ1M>qS9XUt!hpEFsK?ZPh{tJ+@K;88|JA&3ax;+Z zous^-YBt*l83^YWL*FX7E`IB!Y-OfTU5-0K!_GseA{m-X$gylQmh2_C%D<7SoxO8S z)z;GNa+y0{oLgD@V-B(kp*x zuvS3vo&RFUbhr^!G`s4O$8PORDt1Xodc;XJurLZSIlsRJ{zyY>A$IHj+(SnadfJ#5 z9l6b8VsNi=bJY>S?}JHR4}NPy!1MT4Iotc+uH?=ISP9*oo2)krx^iO&oUB=)q4OR# zbD{;^%g(^L_(^RfN(9h~I1tq?X==LueWqKZG2zyognL*iB-;o*@72vW_$q5`Jur<& z4R-ACO({L;!Of5@c|Ji7h=F-t^ZgNy(J!d~5bA{;tJ-e7<_>q*y2M8Bk_TT315wD~ zR{gWP#g|BJVaY<$1lLnJvp2~WkMw$u2Q;_W9_}H5qdF{&f9VR6rPd40BNxTk ztgIa1tipots(6701izyi*S~9-Ia=Uiut%T554JtZB z|72UMpg|IQOAz#DF53a;R&^8BOVlEO)6fd7b~T;H)z;2mpw9ZBLCv~LGpwmG;k}P$ zN{(sAkW{_f0noR9zg;;!jjPHsVDq2Tk~@b(g4H+wgq_9(`zoQhU`zSGMgE9qRMV<{ z!?SZ(QS2XJNOud3!jTIiiNoc4qf>Y**!0bsj{eqI(5&y$e@t?ynpZ7-EF#^Yn1k#L zKtdlXVt8F?+dPBSh!q?WVFOs52A$G+SNK?%HLhX3`izevs)ds}tr(>`DePe;j*rEwI zYmrb>j*y7qFi4hPhuLbbh$O!pF57gur@!g>#n#}t79{8Ko}_&@gSh-TLdOPHNX=T% zIc(}rx;z9(2|PeEPd{2~VMWy#F;5(U zdPkN+kw(f|hUs2M$`(M35mFz&pS%BZ9NqjyD4#1-u3F$G@4t9@dU z&3s?TZSbQ>$R6IbCebh zAn?Qvx@#b}_ZmH7yiYty_>lFg-p$$z+j;pk4}wFfR3Puy9|-X@KZwADb@ z3!*$MKm@Bjz0lmNfopJrYv~xD{kFq%v?}g(($E4A@rF}|uRj)tTCO*TkMpA+XiMoZ zTO=~uC5ouDF7Y*%1$Rf`pFvW7G|Myf@wu{KPfw_ffRog4z`qb1j1t^;!Q{?ZxQGt! zTjrekph2@YSCfH5w+!$mkjqF-L9CDpAD=Q%|$s%MO^Hy_e{RS~SVS8?FcMIX>@SUB~Juv0UHk z9Z>lei85MI{F^+Q`I_Ry1vr)`R%M?3Q1pP#8bqz{AS&IBingO%5R$N$NxTQyS4U(7 z;(eS(lM-Rw2yaz}Ti)AmP}6fb}BRgnJ9X>4=>%5omAS9M^e4(Q5!^_L~O zTFjzcMZO`f;Fb|$#&lgSo*Ai-I9A#BGcy_NrypALdGi-;H*3$r=msTmRI4)hJ(ADt z@~0ipAz=8+Q^Jb?)vW5&0_K5c>rL`)QSL|Q2NxFk zU}k9_19x)r9jiBXDtb#iuZvg4)c!F$zV_?jtIYWjY)hSkVRrnbY=xe5Yu|r~T`Hn36o+E(lnG{v{1N#OA^EFGC26iGTILkDsTHGUGT1dl z4sN1jzW3yeN~(0j=hFtw^yt&mCmu}bI$zozzE?bkLi&&Nw^#&0$#|Oo@m6c2Q%#!v zNTJ!G6W{onLt;p%2V;pEVlA_IzCD2vV8trg2e29t%}cu_k{MKLWPP_9sbCyzm+GE$ zuEm=M&2I>!c}A~v{ndgHkA(dblsDaKXh|})T;9ndHGufC`Gm>nw(+El$rFWoq}w7s&` zXzyQruqM$+1Nbo|$< zpuCZ=9k3^YuuXIQvISh0sG4vZ4ZmKvu(9FkCAb{DYV*LplXwf9$wWTuudqL|pGj?C zk_nhlDpelc4BS#w^Grw?l&w>`5q>|7M`3UJ-uLZn$B-(H8k3C}sy zf?upM@>7r!C*s}Xn9$pYX4D0^s)2|hnxFyoOcy2g603tw zXQL#Z8NF0(Me)s4V!2F*=BiUy$qB~~WWL)UBsIZHQ7k;6mCt(WV0L4!oetEN)>qim ze11xS^C3R>s<$3Xkr*$xvsIXKun-OdbxzHG($k5ULZFhPAxYhght`JjBF282h~)NV{wGhiJyAghTt#^IChPh=5_0by{b}JK!hj z$NO$odo2+=`FCQ4aTk)q{FO+)sZ+JAcIuI6sNsLbOlE-e8J4`XR zQ<>>e2_HVDzIiWsSBURBCx5CY*w?EhkhS`jSiErnkj2MZo~SgzaMmQ@O7EzNFEQ}( zN>jESs_m>hud2grNz8TMVbyZ&rU?9Otu}H%m6t97$z`%Euye!El01(iuYgWzSP;C+ zBureb0(Tb8HjdUf`(c<95NEp#OmBc*?;2qJE3~$|QbKJLIywG*yext-Qav^&vGGY* zryV^wT$ff85t<4%tXP+28vLVKEuDEa9rv>DT{d#IXjOc@`|Q-J>+Z!Bi?!RNmx2v@W*yjo58RgLs#c5pW9P*PN2fw!UU1X3(~w!zx0?hZ zay^y1!y&KxyS}9-MPR!oA~U`iQ$8+K{r-mh7hNRi)l}_Dg2Kng#(R5x4^Nps9z^*& zL#J;xT8YR9Eq(shu+hLspQZHbB_X6TE{*8(gs7So*b441bTu?h5M*UN4f`a0V!4Y#$#DugT%CA&?skLE4{b)!& z6$5{c?C~JGr&0NFV(UDyd9m3&8pogCeZSp1`DF669Zx5Y)}t~z$E*)NHgV6)O2ngQ z2I9o687nxT5l;?9JN(xlEy&qkdd-D5)r*nr&YV~%_;(_de5bFz+p zVAsO(A%HGtl{LxBGBI8}W1>M`Af4S@ye*%+)2sFv>aXS9x_^3SNv|4|kye(Ew5**t zu_&56;^fD?aH(wi8D>BZEi9r4a|Q6g2#AJ1>K=F7(K@rJk^L+E_3|yI0#?GVm?4(d z_F0V-34Zxr+xAOyRY&u6i~m%s4j%Khk84Pw0MaY-k; zdB{QNWI<_#TQ97D0IePa&42ZZ?F4098Y?(KAGH@tgxfOQG(6VWp%0%li|*>IrAPPR&dQ^FqfCOgNhRqk?mb5xZC;q{%=qTU$EZWzi^ z4IqvSe#kVVprB(L4W$e)RI2dT$Sdo*ntWTjF;n~JeOEaagpE1R;3 zu>i&h_V;K1*Bgk{gzVm-cM8rgBiv;h?S#SJ!8s-nqWw6$)_Iey*MyPwF%%zX_e#N( zU3G2nQ?lcJa|PE-b?w;U8{R0!-y8GX3mOD2)krb8`Te1raZ;^4;M0@a_r;!85yNb`mAx_Lo$qEMZ48OYjxRrJrn1{#|Q2e zFzH-uRk}y&@Y+IhEW=CJFxK(jtKO#1#o@MTIl#Gt*b9|<=P}Mr-s#5QiFV2u_mX3h zwbo<_Eo4&Y;FIA-T%g;j*TJdtJv&0{yX7QENP>hTWdNML&YzK72dLH$s3*b`z;rwqDQHoZG76%G5pHFue1bb}ZP^qz~X(r-6&R zDI0OAZdJOI`wZ1`C#gvExHe_N>lIz*_Y}E*2@M6*9?V8>XK@6-01RE>diy(f%{<>@ zuH|K&y^R0@d8Dtj@=P(PA?uUyemY->v~|Yu2W7>5|MN40P_Jxw$%zs%LjuNO%{BU! zHSbSj!2u~cI3U;{2GY-FvbD=Fa#C|?{AY9eQ$jVdRuc4VPa9E8UQcYk!FNbmEbl-d zu+9IZBR`H-Xbbs|xcg6uSWD8X7Fr@41`amDg8d-CM`#6|(kV!-vT=k@=cW#l;~NG~ zo#(~~)GZ}>jx;X=Z40l^Oq`Fl9bQlCZGzf{@lp9`}gy_Q<49#N+I+ z?r}lueZl8+dQNCiM7@t&w4=&>1M72de}}K4*YP_VslL1wp%%g2$qr3lE`91zQCw0y zJU&rNR8u`F`~p7;_gzi{NA_HYLljQT?ANJUf4`S2@(7f`WyiBWDug?Pq%>)xY9u(i zW3(=N4*p9mSp6f7_~jln7ai|HL`?~c?vO&tDDD>CTuvoVNN#02qxg=eA{U(l1u}n@ z8Bl^Y9+_26Gy3uRxz`b2>4(iogwMk8022A5(eJP1p=RG_6e?YxZ`iY@aD1XiIbJS6 z_J(}lwrbF-5RI-Kx;~Zb5;n3lJ}E}tsOhFa6QYY0%)(<$P5Kn6UT=4o`zw%4WrDo< z(&(pWT-r_s3~@ehHCe8y!D^3pDMD9Z$sc_pkfQjL%{kA1mu~NDJbcLvx|R?o+n|Zw z0?ue%L}20Ifv6IiDhDd7bWQLk_{(Zh%>GtFmgdkedt6Mymq|Yb_Ov3FiMM(eNN}P*A5psqt@dUqQ_i84+p}(6Nj%*y zH^ao^gzfMdV!P(ij2|jCJtKr21>Rn|ciFr>=A^7+h=gM5Pn^{41H~VzC4{mQb=(f? z7d}rp{6mGM)BCca@3d>CIH`L8ZU)}`lmPNRAF^V^oIbHKOi?~mt}m@Py_DK%W`3o>2E1Ro_zMoVk`fWr_bL{p7{HpQ%u6RZ^lsjao60iStQa-x;={FA#si z+fcS_w-Kny5+UTs(43f5CbFa~9YNkQ^mpk|qbpE{XT+P*`&cb4-b_9=ogjB)4gTCc znS~e1DaguH?0s({N5{8HtQ(w@;WH9$F~;?}uYKwXlK1A#$O&uRgbcKABHo8k_pBmN zl2o*6hbw8PbxKyRgDyx;h^KwD?eq+x0`ee@SZNSM{s({Wyb}bJ={L4R_H5A%d?B*E zSu)=0d%;97lGNKhJ!89RsonawEc{&R*Ygx zxv97=x`wD`!w;J?8*zCZ%ig&5#w@Jh>9! zzPfaQh$(;MW=*gDQn{*Hf)Uv#x+>(ay2-Jot&ggr{Jv8%mSOf}&AfMoNQn7C*r2h_ z_%Uuu_SdYHMh~4w_C2A|>+i4pxu#zFQKU99W~v7kE8h3(thK0$amy*hT@=+JUKVRF0h-4{ zz7578bJX%ee{<|=y)iCaNznNAQ_Qu$KZBlm)yT-Fz+{KHUxq?2h_pM#8D#@^oYahTLM+Ah=p5TaN`>|o zg9^LPw!BZP{K-`r#o5A}+t^n{!VXhaTOkWS&SPpQ^KVFH(TT7Nzh{Hyek^rWT$0}* zJ<|(B8)+;ZS^4atgrky)3FLl4W%Z=t_-Z-@dHfY(z( zT=(q;*r4&c9KSC2*N?r*awEQ+JdLcUg-?krDvKnD5R;9Z?0`dbJ!KT~`jK#nC&a=B ze-ND9jB(wMlz!!eJzCEo#Myv-_+dyp_9|o8e%qBTJ*NHY&pu=A;N{cWD3oE*n>5{0 zx6kk@6+gb9^EmNL+T(KlD}S8nyYqL$9;S0njQk5+R4`m~@Of&zw0Q*8t)}*V>Y?U4 z?QlB>7o`$7Q*<;6=IQNBe0M2>Pm{?I97_8G*+8$J7l9QgJk7`=}VdpDM%Qvk(|LJ4ercSqG-Sv+yZKE3IR%wxT3rbi) zvr5prI?BEuW-qjUgiqBJ>$zj1T%c zRVL-d#YO_itEv%o`c`VYi4asHa7iWZRR{WPq3wba@?5K39?4JBX!w!Y93NCBD{daE$vA7uSb^*f6goLV zFNF*ra&%Pis2TCUH2FrmZNEb8cOD*7c{L|i9B=$4;l9*^29&&OoW&kFIQiD`$rCAM z68e3YE>7j`L@IAHzUAkx1(75a{+0Lk5$)U@w>4F5d@|rtAb5 zZse5s{b#j&NqPq3XYp|1l?-w(3-Kf6Wg~d-06qJrZ+la5!86Q z#pboYcTiziZV~ERli7a8~FdvvL*p+u;!s#;~J`uvAr&1xmKuGsT?cdl~*T z?Bzg}LYt_L`EMU_Pi}g`gV*7+KaN|W{U(pn!l&{Um8DzPL!9#8iv%7h&`=xnF&d+%P>)WS4TvLK- zTLq+kR+hcr4n-JQNaa9&7?l;3wEl3n^c32nA9nr356WwljGlWj&Pv>c;E<_x8&^W@ z8_Y-*D!m$h)X3wb$vmR}Bigkv887pG$s!#@!8)vHcyy{28aa=;z=a>#dmik}Jht=@ znyjz%v_*U}Wtz@kaO&AI5om*a}l;=-9yhf-*66+V9r}#0%YC zR(ZT$X@~W58y*Xurzb0T1nYf+r#_<{_2^v7;Jx8ble^8G@(DiSMS}1&80jx|8?8kb zf$thR8v$Q@A7zzX6;wJlhmHQ23>Rerhg?06c_8&O!mftX@|@Fc@WprY!&9XwUD&gA z+&-$rGqnJY+~RUjYlDB!h~{$#8hO)?nQKFNo2Xau0^a|}(^oh&^?zX#B8r5HfJ%K0 zP)QN#QV|tNg;65~M1g^H4ln>|m68VOjWKGB9@5exqee)_7(K=q8}G&6`@Vm`?!EiO z`JD5d=XpMR^~oYfu~TFbKbP644RXi4T*aPSWMpaPQbw^H%be)SX1w7iu-s+EBQJNF zP&kW!JaoRCo$%A^>h-fHqTQm)^@JDLnG`g}cid-*peVjqAj4dGeR=Bc%aJ!wcJXHN zG)l@hY;Xab0FA2R=%bz%v(QQMRl*#*)=bWbSQdWt@ee*LKGA+w*6@r=SHG>*pOM4B zV`y+9GX1#d6ZhN`)~bsW5xn1Y;kvDu89{mN4@S6$G8LZps!#*(U6Tg{P$=LmZ{e{! zK_dM;%t5MtD)V@Ofg1*-kzn=>1v|$HpH;6DJ$LPPePLJgPp32f@zi=WdoVLDE8nZg zlhI3^?zBh&`=$x5Te4Nte+FQiI)OgOy|6JhKND+X9JrNW;)Q)L+w z&*;DJ4by+i>&^Aiq5v&46cu_jubITrG(p)K6&~m*rKs4>A#Y}QPTdw6a1nQ%+$qj^ zRDvv96M+XwKQR76y!uz$`)Dw2p-a+5GDC58FwK5?ooTx^iDYbqSRvz+8Z5Wj7;SGt z`zFn-Ui@hosNeJOW4hefN8>jS7rJfu{{7lDI!3NnaEv$AHE}%0xa1cswhw((EGzTw zqsJf~UWwZ>j}{F;Pb&|;7O&YwTevlUL(;+;gQ1Hr32LJG-EP0>2@F1G8l^rj#o<1K zW~EDcr}xuKGmn(~+&9#Oj65iletDp!yg594%%%%e!cvGBs1vUp8y$1;x&L;Jl4p>d z?h!soqaK6A`^tZmlKA%H%cJBj595oc4F`iy|0wXKP_*Tj;w!M#yk}uQX%^;0l@G-X zjQ;}XBiS+QkI$K{!z@9gF9OuWFl6$hZK>z34ucgLfy;ML z&HbsRuKV+JC4prLby-s{-%RB=b=%L-eDS+Tr#3zv=zN>h#~gKH(=1h0H!^5(xpDMoj>|P`FGFW=u9QY| z@9C-bG!~k2g-ObO;@@|rre@=82pnw6d`K;(EOw$=Im@%7>4PKeP+ttz`~U@{-jPLz zibD0rc8S41VMu+?NEM*F=QFO5CVg))1W zXuEbE32S@P150eo6fveM)QrhCxIye}#`RxUZ6dHX7FR6mLDVTfZAI#SwQGN(V4zdx zispdgnZs^amM8y6XX>1C6gNcfilX*sBYb8I>^sKq$n_ z;d4JYmmIB8ld%u({!ZL~QlVR;%s&yNv{AX4Y(bhf(uY**jL|GCKqtG=n(eMWJ;*ZH z9^OQn304COVB0*-0n$v`*o~cJzheoBl|mT>-J9Nv6R){X8!Ymmn);Lbg$cheHpOPm zF<4=$MxSW8n2ZgMVf=T3yGC!w1EEq4EO zszz@2=5w4E=F`k!8DcPABB1gB>Qgy%b?T`8iSq!&6%*t1vGSc-1U?vhYW2q)?l#l9 z7{EL3j7GgO4QMnLo*hk}kGi=SDHVGhr+lW#AF-YXT{1^-Dn+t%sSz zeSl2AXQ0+N^ScEr@9r|!wB2pXFD4HMhEnT_j32i^=6KmO(rS_BJfl|rQ`bJtTrKt& zy$`a^z6vaxIQULo-U*REKI?^~>*O0Rg_Tb>eh0*Wo{k3-Z+Ok~3m->C!WC35^|XOQ zlV*lUDeS{G8SercR5${>+)HH-ruPxIgsN5LTJ(#=PFYRqB1Y~yI6{z^VI;3a_|8ap z#=%kO9o)0Re+>g#HwC1H@^3fnshu{0`O;`ci|;MV`OP@a?!KpO*?NDAk>wXI;sFbx zf_;p!Rd4^Cdf=MNballBs2P5q=6%LBKH`9L?Dz{mR6PD}x|5rhdEv!-ZN1mAQ{9fZ zo5qRvg`dc~;RMNz&gwtMP+It`4=GNv`#%*hDHd6{FM%uAL^b}e>GRXLX&lSXV=-8- zAFiw?Su+3-Q$@8|l;Tp=>nnc~I^5tlC{)E(qvWWja$A89N^Y%O99^~X8_xV9)wg-# z%1w^mIK`V6PNcW`$zeyHE!vuF9FA}bc>KMsFdIE}if9MUAq)spY9lkD`=*NVAp%dB zF0a^tf^MZ256G{4$^qgY2se=c%^wPeq?#%gJ%_~E-8AxuQt=V8-2C8?WT+g4TY`+( z0GdLh#^s#1L=xOUz%_J+2U`Yza%Mc|G+&rytw+BnSIq^H_PRR9VUI0BmxE<3q;r-O zp>S?-XE*3VCipPy(eEU}fRtfAwKkOA^x_7O$o!LTSB(Wh_3%bBcBTE?4~G@A?}qX; zUxENyJ{MhM6wBSg)^(+I5TER;kDIn}D#FcwJdTCJgN`_Mj#}EUErda0%%^0seuzm$ zb16tyAEcg7jbZuG?xp&d92FhuLEG9FMmm9l3;QpN0O?fCuKz>!!xiLi4secw0^5GP z6tT%R6!?K%%o%b@m3UL%qr<1XDB2vZH=*sV_BS|>Q?J+bCNp$r$w=QSffe;!w6^bV zc8_r&k*MKi#?cnpS{m8y-!OBP(I^r8I>-{(gE4g8|BWzmDE|ktWo$w)Fs_`Ww*p>$ zGNXpxLj?#L!^niX)t#s5GQ&*ZMsHb&Zg{5xt=Hlap0hx)`ej{4|y+g`PCOsc; zK#9%_5q3TZfz-{IV!=d3Q7Da{g{rl~0ZQ>xtIH+%;Z}*G*f1NDoO@}%0Kw6k{E^wy zO`kMnndRCVE7!nRYaYs^n*6V&MNun?uD*yJ7N0;J0zp@v%K6Fal=o*FVD-9$+0Zve z(Xs9o0rp0~a;6tqC!d4hNgUrYI-5q=$|LFJSS@!5{e8Y!y`kgh=e}{AC3Llmo7e2( z!>~<1X%&s^U{O9w_{^aee%^JCv_@=2-sATN745yh$6mif7>K$1MP!5D{F61M1{eJ9 z^SV#llcC~Y@f(S1$~)R;fOctz*I=Dr?cU)^4KL97C*A!qt^7t|?>on@Tl*6)=>}*} z7d2Q>H{ph;=_pGW zGNIbrB7)FACz{sef&PlZDnQS9UPzJ#iSHA~dqaQEwUA;3&iBtw6z?vq)*dJQV95ww zLmuC?p))RF=tT9jFS=#f%((EOwJNNTO3q%je|a}I$(4}PDAXhv0?D`%n|uF&q7&-puwz8 z%T?^rOiI>|^@+gBmpzr|s%Lxhu*SpNk&fG`AFsMoCBv6zaPQ4+pHQDb+%CDAYk%KZ z-g)VTu7OA!euiqua-LN$C_S96U)15{_@=82v?mmNM{zs92KLF903(je9gCtrg|9er zq2+h5))*ZF15hB;OCBJrSl|^VJt?z0BRZp;fAbAdEXwW+&uUL-jt^XO7+Ci%E4`?U zoK0Snw2YN=u1e+eZBg7%vTBSLR1M;ud96BwVrw44Tg*4VftJjDLrkOneb@04zC)mg zQEM(YqHvdGCcPiIQL;@SF*rW*dkZ)5cXgKX@#UN|lWqqVRa-NNEsDCIp-c0k7Dg8+KGJuKOiaLhDK-QGq?=~h6Hi1N5r2r-l z7!I`bpmxVlW`f>&mbv?rF@zok!KF!}Ft?J{J#4i(>k!M=GOzJ~Cp-b0w3Pyf9gY_{ za#i>7Mz(6}-JA1x(V=Tw`-u|~LX*Lx)F*%h$n~3UhU|8x^3mZkycH45+01MOZ8}7C zd5U_HQTM(nk4j}t$#OI|_q@PD-@_HZTr2{JOM|ws0~e3P(a$#IF-qBJ7q6_jc$jwX zDEUwZIY-F`<$Bz3!E=hFD&FKC9YAJ?wXRM`iXjNs$l^9@aJ9Me4<(bX)jU6PHca!_ zMp3>lgVVi2$1FC`XDh5+$xYXff(RclT~^D~tTuoT?WItqM)7t)2fMm8_y$nFYL@Wf zg4f%Y4Ul8niuNBjvCC>hBNP0Bq7X`~daWn6q0yq@OyjoO5h0A#N~<*Uf1oOz){jV# z_5P{T{v7m|SFV8q3&%?-`P3^JCIMZd0C?bZOTFw7W2II7RmC45)F{N`60DhK>Bv?h zb$m-H&QvQiljK@U`+$jLN7@`)MZ{-F%e9Yx7t=OpM^{4z{23~dvQhjML>7f(Jb2++ z4X0k8pCwS)@N7?7Q-VXWr=HqGq zbN~CWWWZNTk2~7{+~~?SzB_1q3vWObcxIY=rp~nZFcrOBN8!n`WvV(8s&i#KN!5=! zXj~1$NE{`>FzwvWs}pPWy0KI8W3U?N*J9p$5obK*iF!x06^1KF37)5l-kNbA+wiGY zJkxujG63|BhgdE6|8aJTZJdNBcfx@b*Z!x(70{Je_ZICvHVZ1 z@omm07Nm=XH=e=5G zEoR%4%gP#%hkF|&jFGBLP>ttW>Mu-ueeglT#QvRgPG6&z^_7#{tiqSGKOjpPw2Cgl zC(8X}y{9&lB_=gP^6kOdaCz1!**DofK+z`fqMaLV;XSSqVs8`ko=u^Y>qDb*Tdn7J z`+QUxtnErm)@HhF)MAV)6a+u4*QY7Ey5R%Ve%GG)$TrhmGD>@{c-CzA(2Bnf8e-|N z0(0Ne#A_99&}RSaB>uwqEN2-dQY$4TlJgpC8%EHRBK)nMhw%4}$H}hKPe*t=>a4_g zL40BD|A}S$*mYdKL06ZWaj$m7%t2oJ1?O{TsVUbuet@$+Y6h!w482nNvM!&C*-Ye81Om8(6iMp zoX|oL=O-fymuLnZn?Z5PZ%LFs!Cd)LmE_ldGSw^Plkd*erJy}Kdr~(VA^~w*)UKrJ za_9ZNceJO$tBn-{4%638>?8}uvml_I^~wPw_bwiNnUmZ}zkT0fpW42&wJpU67W{a# zYHz7K3wvJ7!1=2kI2#huTcPT=!Xw$ed#;^7_kyrWR347s-VyT7Inf2Me9>GSM?DJO zZ=2;0w^CYnY!Of!P%q1R#}BK2f;{$WMF;kv)R2Rv8O7Z`6Wpb^JOq^8|JE!|j|LYS z1D_jk%#J*XTSc!Z(qKUwKVSM-p<*?ay4xJhv;);5{MkW&(Ol2ZAnt+(luea*moTw9k)YvXrSn1MlOP1! z1g)WTqVDy-QBMxo`pKN{_$Ig`vTTuJDHI@Z?=Guy|5d5mhg%Let5taf-!}KD$?YX9 z@9#U@a8p5WG=*x9TrP`DC!&bL+F}JPlX*nvaW!Dui<)^?u}4) zuZde@QrhYgw-4uNd=avUh=dF}hoABUtS<)_VKoJxS?(GosBrM!=EXQ+%Qmpfqt}@a z8(p=ytW*}d35#h$V1ahrYnX_$q-#-`v!v3>IeZshi4`-F$0(RoxYYPp*vLVoBuYt= z`JB{pEUjNpw+yEaCTx!Enl!SIwF=SVC6s0%%^!3J5BNs<^tYUSD2%rM8h~0Ov zUgh)o?PPY%lk05#rP@>GpITJq*1N0kkx)#t=fher*{cYdqF&Z_J2$meo0y0J4HGlJ z&09*YH<}iP)qV*ILUjYPW8*W0R5K4cR!kDPa~XnOlDQ%v6G1v2+8qF$@OGg#6hkK( z$%c_VWqkHyS;z)pO;=}*4f?(Tc(*W=Jab}ou6ncu$H59?^lQB!J*H->*u<%=*%3~; ziXf2|zX#$?(}z?>Gy0m{h=CbeHzzs#S<+(&f5YxW3m5TsV?FY$2ky))9`$?Fn`}E9 zUbROBzfatufJcKUDb_Ppuler|P62z12R@hiA@@nlMQaV*Uj#CN8pnIH{mdITE^8|S zq)0qAP^rq5V?VimqWJ2UMEk{`1{&Gsz$v{beXBuNk+?U>vj8`6C~Pf$jNI-s6jE|o zOxQF|Z^uQk9J%3g51`6{!cT+Tc!BO(qzFyBthJ2JQxcfrcGw8byxm)FFE*p2^o~-D zY2`sj$whcWsm|ExE-_KI=5J4G#1PE|>Xl*vVql0Wwa#ld39qZHLefxh+%USK!bF+$S%6T;7P-j_OY}!F$|eH7Q7Lqy>RhAr#@l^P z+0Re@vn^G!%$eZUMK^BU3IeMgMFQ16TB<(jjTM>TI#*ewzYflc_xg$G*;foi#I_>6 zR*NeaiWMZ6I)^;q6$|(;&QY7Kd#fKZ9Wz#OMa92^+=+FqBMAe1L~nYd0vOfV`aJjl z9bGeCz*kZ<4KXMXux)J$wU#=9o%vjfFx3m%C-mD~tor2mPT@Zasw#kA^_i+_!LIC_ zQ(=F6K+s+ZrT>}v5ZI+QX~9Wq_3Esh(+!zJMJ<88g;`QXnyN74=~9vA?#F;8`1*yA z_Yy`ZdXp2z2c0MmAHMg?#-Zfe9IV>S-(4-y(kD#}A77K~ay1arQRNg(NYcuvBR$<+ ztSi>5=^RR2z6_2rBk|%gyfoyhRgD4_>DsBd`11JJHI3y`bG5DOeH7=d%sZH_#iPB%AxM zP0sPEv~}-D*H^gP8>8+!z^JvJf{h!-n@mS3H_%^R2=ZNfAXhXoa9WAUA5f(*WGkKg zB@jw56U68FVY-@<1MlwGG#C10J+)(Q7(!6AncC-LEK8gz{rW4pGyv5Q^d`%5Z8zZf zA#I4|XPu7erjj{poyd?=6Y)iD3-A7e?r%vBw%YF67*AgSAM2q<(6M@+qg2`e@|a|i z9JZ?~mH@X)3xo)TIbtTWuf{gLK{W^|uG6O%k2w4?T$Irb96=0F_%W~Dj@d0}`L`A% zJ&rS~q$$nd^JkLP7lvdwcv*fDrxeoxU zoP2Hn>z{jEl9yQfdkY<*UfI%E*$axU)Sqoy01I}h-yMX&_N6}nfK4N?!joYW^_{1z zJ>f*TUOTk<;mvg_{`4zNzqv}YEXu^SlxQUvuhro{yEO4piv!OyMNJIr4kn?qMvnD2 z{tUQzUXDfmy&Os{CrtmG*9)pIQA*f~+Au^(Tmz0I(n*YjIjSi?jkf;FUN`zwb?TNN z&9j7oLTrl3n=@eNFyGWsno2{U6#V zXXx)cZc;B077Hh{DIJ z94-3|+HNj1??c&kDW2t-@g9QIh`^yp?do*KsTJ>|Nyf74KsTc9JY!i1mut-X>rXAe z{90o)mo9fxVy|6;Qzvv|EmOAws7%4}S(8=&vDS6ZjVhc6w0L ze@Bs)E_N0BWikstT$`STSn8v)xST^zdENr{O(<+%J)SPwa=Y#t;(TCEOq_-cHt2w> zMv%HEf}lHkzB-&0O!*Lq;0kFB%y;4YEv8xM^&MhANo#F~?7*c-DBWJooMQPA0PFfH zjhuWk!e19g*O=|7CMpoHE=4~GJ&9|n0PJiC^b#BGa((P8@IN&cu^eCgr@m-=gCn<` z!Y!ez9(pjX3Gj{h>!oIlCEib;k5FGjaMv{5kEp^fH(jT{H==(l6#>3r7fv4cgXyYC zt+z^*o~*w^Iro@R=q(*E0`!6q2%US2*m5FKKuY^{ZM=mhiLW7{5ra*#>9zOy7|SEj zl8_9$*3Av(qd2<8Jm3>y`>$p7xqvNp4nzoF`uo!`XD}ePZVPb7;P;pmT_CC}a$rWS zwxUlE5Nhdj1wqSJv+46;IxA7SqvX`B3$B7l7M&pketH;oxB(zo411|CY6;+9HuaW^ ze$Wf}U{L@BlvbcD<-?C?2e@;KVdt`bVO*{XTBo%qK0(RhcU+r@FUwCW^k8IZOWU$8 zkc!yO{r~XnU<+LgkOlbv&VKkY@A(+Z*%e(N>6I`<`NY8Ywb<5d-?ktvNCDWl8V6V> zzxg%?t5uoyB(1aa%r9<5eT<0!&AqznDJ@V)bel*7ch{*K^JMct%6^^(9Fll}a_CVN zpcf&KHFOk$W^OH#L!;E+U7ei4r~sivDqF8$vIfO^`GAGqpI_-T-O0kCOHv9de#n26&lRLMM*7w82#yxNh|c($ zHEdobU2n8f9U}UJ=XfpBEvdF1o?{ZOJMlTkxQI2#=-;pZExQ#J@3~z7Y8KCFS0IaR z?-a0f*+k{eQC?SUx*mdRLH_HX$ z72HWpZfZnB*z;~DsFCoo!gCQ_^9*s;1zJl^a{4tE;mS=%S7rx%FP&H@1%R|`X)O+| z(@hX{#D|y7I%fIpOtvT_lKp3?%ml4+e-OJ+vn$5~{AA%f0~_6g@P8=8S``M>;wXK3 zL>po?Ji@~7YHLXGXv%o8G{P*`6Q6BcRIPBJW$b(n&k9YXqYDvw%5<47gdJ!!>kR-$ z1UJ1b8>R_0UgPL`@HBKL&wV{iIc-SxS{b!qA!3oV9nSvv(4+P5`K{ zseJxJV5Sp5>)*NphKjy^gLHWAxKyMn>CQc*t9}0-_fws-U=m=8gQQQ=XJaRwMc zO=YCm?Zj+4LzkY)($Q(1bP_^Gtb(D%-yE`_Mj^28JLJN}TBa!MC5IUXLbXbFB;an- z<>xjE3AM|s;;tb=D{JMA!nHW_2&8ar|8wd{{d1MC`@V@3PYk$ci})&CX?A}K=7r52 zo}&qKP>;xVz92t*u2@tuZs7!a-1cOb6VWaN?sf z2_gV{>rXU1>lb60tFBAfzCHFdf!Qj>v*(5zTC@5Q%WQ25lGqpN|4x2{SI*3!L)UJF zYwe?)7JbQ?lR(BMplkyBWaT)Bryb#V79#&-0!F}#plPv;BfgQH=jM#*#vU=#8%1Y& zQ>NFtK+!SONwL^^pwF{7EGoy7tt$|GV}=hK5gWyB-!rl zO$LX;AX3BK%*JPD?1f{iyYq$_5G@eDff+j$x26pe+dX%<@i~1t5XJNbO0M!f%$nss zu<3!5$*Vm%(;4U~0k++46CSv$mR9X-R>_Y_oBFYlR4TC?P5$d-zuqSIyf8C;w?SKl zY|OBC{olGcncIIccn^7`a+{IJh$K#fgqe_6F zqp*;NepbjWeuU<$z;P`8svY=smC^VsW9;QS4}24CFS`!FLK64i4E|SqwffN`q_6)pW+951dgpAN4v%x|bffB-`jOYUBa{h> zU5=SE(yGJ-$cm}Nef&qgz#f1;0PjDJ={@@Ce1Rx$pj)R$+v=KOT4CZ)-U=~f1BiJ4 zvev4Tz9iOgJDJxoj^CTu{*iVY==DEj0$EWl_+USuKtCUDjmlYE?hhmyYp>M{7iN zqeK8}znM{?jx5kVOjMLab-u&TpR@>z+|a*%7+pSHXUa7LILoGuKj1nUEW`HdJ0i_1 z-lMuHq5zEglhBFNE>NYY_@^~3j*D@-q0fP)?7BTlwq$jr-@e~}?@P#m|4QJUx#PsX z?kr%hy5~45h=O4ULaCub>F+rft^*1gS9hAG8fSJIXxrg2;BR;u2Drz27Q;gS0%R`r z5EmlW=zzM9)vNn&+%rBU%?5CWZ*-%&IQw}!g;rJ#rJ*0C&FyGM+L$P&Pecr8Rc?@a zmXDugrq_2NL}LlqS3}RMs5GUSgA8-GX~|P7M%p2#%YK~v7f?Im%hEuWY9q9t3u*Pa2FQ zj0$vmNV~V!V!mun9)RiVVyNT#D3-gx{{2@%{65Vl7OXb_eb}#hNOJt1_zQjSB5DKu zea`$7t^l{z72a*aKjIWlKaGoD#6P#+cs4An^ioe$j5atCD4Zsk5$?|h*0YJeFCrpHM&ovwdC zPEjMvB^TiJzor|5*vh^S9y&jND7Sv0WTU=+5VSb9D;KW|96+7xa?)oTpP&?%<9_tF zXBE0FSG{?NSBPsA5{zZ?j1p;5Fl*16eE|C3$oa&XGDo|(4?T>#gySc#A#p8T?4>(+ zuQVMs)A3LEA;3i;D21abDC6Da#XHN7R+CG|UI9B%pz^pka0;2FnvoUXhHC$@{~x>Kd zUSh3$?lJU0(tRUBjVqh?mINpXxK!uoOQC-irZl8GYq0>?>kS%~KM=Wc zHNVBFWFz$z1ld-$u9n)g15Q0fh-<{rc)P|Tj&EetZNp|3)sc^x+&HU^hjK=5&dOIi z^MZsVM$QBix~+cMFHeg%ciROE8@fHX4v<3s)l4Ok;)c=={grEOoBHnn4-5s# zYrZdbF1z(rz)s(KKbB>?>i>1Y>DUht7<32a7A|uBc+Z6ZtcBhNpRPN}u3rJT)i;<8 z|6QXnPSss_L!x{s31m}#DG(>p5cH#xrQUQ2{xc}pI#_VjagwEYUYHga=!>2mDu(w5 z>jxcg&4Av>iN-t8MBU0Rv$Xh zOO(#_c18swvQ%;N{h%3txL5Icr=c&I>B+8MMxYH7a^eqhGoASA@mNnzm8 z09)Z1SNZA7D=t8$5jB2i5agT>Ta#2-Q8m`10Zy+y!D81f|7^~KS-w0WFs5@fnenye znprtL*84odSoun;LWKsf9E3VEj%8O9EJ#j(i4Q?%WIHwQ5IGUmN_FFcwZ;j|KFhNb1iZBev>jdy`S@DTTFjL+Ly;qQlQ9tZkB*VW>+2ne z52>2ZoRLQb0jI*>@d>K}Jff`_g54DRD0N~hV5<~3NkS#VH2dACZh>O)!|?fRc!8tS zNaE~*yl|7icFnf0L(pvfPE;1vG-UBtwVE33VE&~{$uU!%ATX=$APcR>y!Vp|Pl(Tj zg4^dVphjeRJj=YYyvlm#9o;n}$RZ{d}hfERb?Rwps}3P$ps|4geI6w#ZuTSC`!jxa!Hl+@dn6-!t5 z1(8$=2*6&-R5wz_9mq_)J<|a3Ox#v&3+P8rtL=8bNs@U~=sI#c^@IB9Ui0{}Z9S@`X$KyD?$f(34ZSp?aXoU*f_B5)&&d8(##|sAUn1@U^xfX9w4gx#{;-9^ z7(7YjXxbN>>%QoR@qX&#^d;sF+wu+pxB3A7uF22ac=^_l9=2}y0fG^=jsFle- z<~v+Wu3)5AOTWm6>EST{#8VAF~Q)JNm5fFP2C+xRF<3q(ypzWNj+GT zDc5NfFdFu^jNr^5LR+LjpQ~p{Ew1bTWmMpUkjkXdc7-~6a~fYHgBR<@>UoKHRY9(< zc{2yWPdd2W-N3uCPBB#5tK{|p+HHqGn>xClp}ke?(eWjG@cU`tc47YqpwUa?EMbbXs>LHP-Hba$`I9#2*NBQ@c&!WX!^JS_pGA^E7#%P^jB%Uo{ zGXsP}71OGf_L^`cON+1qv&>d^TQjeEx#hh1QRQ^m`(K_bd41!8;7xJox!nQt_P_~y z=|y<5u?NPe#J73}j3r_~(8{YtSAm0<5%7{10?Bx-8PX?>oahS*GLPVS7UK!9dIG(y z?!$e&jRwdUaF=!~Iqk5~3JM_s7R`RR5hty}CV964Whn<6@Z~tpv_C9W7qGCYf!!fN zJUjpGOz(uo2@)Ao35f zU^d5!Y$eE#{|;0p4DRwYd$Aw*N_;CB4gOv~o z57^q6{q04I;+Ksk1G=lG>^nvUKfjmrKy#3ckPm~hYIMyE;hvQ#r*~>AkA3u_WL=*eSc@xl>unP(N4_{y->anB(_7 zk!)G}qdS|Tx6pG>Ht2qV@uCwBhe zf_TgtqxRoYmjIBV@A~fZA!mL8V{O)v{Vc)Y&7j)9CE!yg7lACKWPJv>qf8pehNgjX z*#B7~xTJ`|R&$JdPPCtz_!+iw2rF{&(H&vb@}p$GzAB7ks7zz$Z+U9~P<%bE5Pt%P zXfCN90iw6O+%I!B=hgE+-~Wh|S@onFz6aHy5rfqt)Y{yqzm8Rp5_;2!+R9B408IeX z;Gb@oAOsPV)QgoZhx)IjNH2g#kF`UR;js8D1We)u11u8Y!hI1zKLiG-he!!O7C!M& zX!*2U(&)$w`|kfB;cXbIOlpk0+W#cNI^l=A!y`np2v;Dp9MR0I_ z6}!Aq;1s0N7y1VYCte$xQWa!LB;z_^J3vS1%h^KbI~jFz4OZQ3#;TWF(GieeZ{g*Z zqiLmjy0CKC903XT8EbAhbfWG!#Ay~?hI=($UmQe?Dv+$aB&zjAIOZ>!-aS@Oc#~C3 zuCxBDR=hJqsHUa>webm`C+>=pcZnev$olA36nE2YX6PJlr){0-n_-|Xi!g^zJP!IVKU@|@c! z%V5@tO+jLepqeG!9ybshY}f1iK$)_dT$ib}N0R{yPR2CF(inonMR(@_OTG`U@BMw= z9YBCp&mX-QCRAL17qrxc?!nD8W_q?5m2ey_=XJkuy z&@x&VtfPJ!D=oy4+tMBE4`VRgDhu$L_hwH(UC?GW(8~Swq)~dxZvPGs;!O9<(FFfJ za&9y}^_@XBdKAQ(kH)`lR?Mv@8uD?Kp?eLmkZo5}1f!Yn}4T^;@vetb9UAl)!2pYK^3Y)YOj)AWvS#~Sjp zCBOxG2wwb%EU`LTm-6EKT7^(RJi6r+0=OWDPL4@rPGbUj1wR$0lXb{TM4vN%L5;b~ zCj=Z0Jgzwi5^Mn6U9*5rNOluo_Yk~Cta$VDoMs~W?Q{AUq0cb|#^2V;*!$fE z4d2}2Ha7L{H1!!vb*CAtUeBnbMOyU$wHxP0@1i7G^|D?;r9X*lH67?;+_&A3tE0C& zJtP?R-txq<$58-Zo`hrQg60LPugzHUGjiX@y9&6X(Kbv(c6I2*_%J(L&qcZFSGc&) zb6Lp)>mCJwDIw28==fhSaScgms{kDK{;}U)#0%?u6RDg{Hd^sdySoqGSx6^q606Tx z!u(SM_X+KNc|&O>jIu#gM`y-;H1X$dWSZ(%*E_>!mG(qFSv5AvWr<9QG5vY<&HZJu zOy6Z$s)8KW%wEtAoV?kv%)QeOkv$Vf#rAAUnAK`w&2Qx2<$VM%Jh;h9GOnHDQI;&a#Tx{-q#_Pq-pua4Hy@Brx0ep2$M7A{5OBpx z21M9#WnJ<}zsY-1a>_@;uY7FhmWQACxlyj1yHP!sQRj97zQ61F!JZaMA=_T|;{BSd zJN0m%){Dynlp#S&oK}*;pn4Cf*}S2FoH||sWwsKClY2}PW@^``%v_opQnv3io$jzm z1*c!waC$t0Srrgz`ue5zxreXqL!ICezyhI%3VoI4JMjcTV%?9=^_f{#ai`gHo+91@ z0(N(}I+zSyFYg4@0zUxKYS~#%*l|z()FZ(A*1vWkU1_0CbR%BAs5qrW8Ui+ej|_y} zeI3{wCH^O+XsygiXyoTcb8a=j=LPU3=IEw}OMRJ=Hs^$1I0a%)t=N?r7QkBblOPuw zWnL;5P~{I+NT0TxsB;d%BAHq=)etj0Z-E_rp0{zdEJ*AE|7Y9tNtaNz0; z-;(`FB`}7?GvDSV?B2K`XKwsY6GV^q!C&5mN(9|xaZi5}97N8LTn%dM%Oj4t3|#u) z`euBjwcN1Y&B)QB|CNyMRL)1cJavu-eGPVuqN7Uw>+;Kh|T*K=8W?6`Twnx(da^nA;KMWM^^P^BZrDeKV#+p>7 z0?6*#p1t)=yvmL>f_Q?tSO2CNfo*-cgCk(o6KnW7fZfC+xehe3^Oq;5U?#`+f}_P0 zn6XCbWV$QNZmZU7aBMr!l%%q4?fmqeh-QrM#E|hAM^NUm#M=OORvH9k7I94>GcBUE zB3d$F!>*TAVG z>WLbcDVg^sSBRN!>HVhH{pRfe%I`HqNsDeWmY9V=&+Kz){f2buoZKo{ySCJ5*W3)s z=Nz@5ovU4+RB9#Pk73VLl$V{aEqb@Eb`L~`>~qqnyK2C^cYm365?X$@J+AgHZr-fU zf+%bBDs8P?d&ev+vg%yRuO!WDfl&$Q7x$>FKksQO`8P~`V2WDc6jZGsI#ANHQvpvc z10sFG1AYKx6L4UDjJ0`|KxJ|QELc7`D(-&T?3x@OD9TO^>&kN!@wXt%DciLpKBm63 z)N^wEbN_A8Zf8nh-%6n5l3VoL~=h@{w=f(yKehj`7GCt9j8M@WEH9zwEDxQPxn@Leqv{ z*@Z7U2RB`lt4ndf$M+!;y6(}<+>P7Q|{VMX6$aCUrmi6*X1!#JRI zV)pgRaUxhZbQFZp*~foq=a(^Xs`9Ye{QKiNwNgni!9x%8L|=S=b7A#Qp%%YBl?nD9 zsHh{y+&cN4Ds__(Utz7fk^+^xi?fv8M#(h$_K~NwpMxRjSd&(Psqi{9^bgBGVYQ7ca99w;7L= zQ!n=)1w73Y^a!3`dozeW5!d7pkJcc5k0=PuLS;0Q3MZPh&DHh#jh^gIQjC>kmKyQn zP;q6`7L-g-*AvD^x7rq?8Wg`I4akl2n4s<9fGAUAJZ$D#t5rsNXQ)H4UJEa?skG5| z>~_y=rElefM+Y+bqcWv>vEsuiMb!>(g>962=C-|1GY%C`O|I%Sl0x}wZw z%;=g1bI+Wt_vhi^w*i`k5(*UW+Wnn?R@<8oRX#PhZ@BS<-z|mz^e+M_CQevkOs4v~ zzB8iZ0V^L-g*c!HyGGt;BWd!{ge2WchIr374kih7B^KlV$o=P;KN?Q(O)ChpToZ)S z{25S~$!XX9l3NRZ0{CvM=y;0+XVkqbqA_3r06-*97i8U2>R0{e>2EM#y%4aIyQ!M} zXj2Qw-tIsFs=lIO802#*Ovnr*JjyTothC7wV(v-M5e#%f#n&1{M*_b=7ET7e_}Ya1ybJ4uXpDMsKJlb!K?c zF+$>G6~P=7fc;YPDy0H834d>IT7WCi0Cfmye`yL>LN0s)S}iN$URV9?UjSRLaMNyj zgh8a8R+cFXNgZy7lBSY5_+H&Ma;@1a+3aRQ(52U2)7u}A=q>$r*}qjDuHB+v{I1dM z+7S26N_Ty385KoKImjXXPS~+uhUZ-2Ot7J@{JQXONRtikfAOlXrRufNbXp~4VY_ZQ zQW!nfPwjHsHGdog#F5~D>;x-}AawKRyQ760yNT9bE{-#n$79>xwC(5kZ(aeOI~|W3 z8t$pLzc<6UTt#PH?STc;? z=;$^_Bug&UI5O=bV_+TIOM&xg5~t7`yAh2uLbuXszp8f?YJmF;`e|U_=&Cvqsl0vT zc9}&&oEZP#G}t{}T=u-%yc7j{Q$yvPDYLu{2M`qqV zjXyssfo19cwTf6O=>J_^IYZs>d3o#8-;Mjk60q4UDd~C|3MCQsX$``Si6#8ZXDM45 zgo`vgY6S>xIUk6-3hud&Zz;WA?lA+6I<)%P{{RQ-1Rian?tuKFa2=H*C{7CHU>SGM z?!e4_NrTee{1AR?c^9yk`3je$YQEu@ME}=G2Ll6LD)5~dy{I#O~-Op3hvX1Bo1qx7DTSga zLd>Daan6Sgn=LtoY&johhUIMLJa(}A{kiWy;r{je%N~#YaJ{$dy5868dYzunXYu30 zaQRrW2l_NiYIkEKgbh1vldTi$%s#ZJf?Yn(R|51y-3O4yzNjC;C*S7(rwhAJ{%WUG zBG#fDzbSBBvEj)J@P&c!OA#T#WHU#P$8}l(%&tV*>mXd|yXdl#E~ZYFHJ|@4ei-K zsEx8X4r^YzE55_xHUyDp#~oaH(Y;Sy_=gWtl77G&m*vb}Ms=%QyuXFNG(vlfx@?IcxyMDb|ro>t(g3tT&$WAR_mR|Xg0 z-C)|^?Q)%EXJldorzvPR#k2cs^8xu$Q{9JK5j4OOEGUW16X;p3J(w9-~=WxJ*GZdZv87f2wS zFAP(3svz9u?%>{0YMKyBukxocv;R(0wrWtb<)R^rG$_s14 z9C!CMK}nOcT666F4fQ}B@7R^Xu~TPE&DP;(ERj8Zcf5EU>)@d;eUGMhqytX}gx^<5 zaKAk`B4=8$Hb_>zB=H2WwvCjxVoROONKUYfzH5=oZdKOpZO^-ox-m++n_c)PN4}1o z-2Zib(P`fnVPw=tD>ae5g1Dn!OaG7hAr0g*BQ|7yNo2~1zo1eSf(Yx6{yP2a#$ykj z{j$I3fV+;1x_i$nlrngyX>}R6YdsMPWlud;9{+ZIHwX^SZqLeVwCrzX#1DT>7kzmg zz9O&h%-$N6N~AM!grEh{83)aAN8ys|RkwW%G8DB~aXybylx=O0Q!V;AzoECjGX*rv zjTM^}Iqm8rGedLF9V-nA401SjPnNzHTr~W_vG(Z~W@xlNlkENWVraQx{Tq|6f+6m_ za@nm6?-&kic<&y0?>b1VNMQf9`v=hL_Ds3uZ7E&yn5O{NWmIQoq~h0#T-HS(D^U>z z?ljISAY6 zXOx7c*AuQ1kKEpxlY^0Jp)5X#>o1a_Q?)?Fr?rmdiw^$mRP?jz{%ln(-$oQVm2?&V zVDhyO0NMR^pyFTlYRO&+x-Cpq#6fHH>oRm$_#9R?^HP{z(%Ebo@Hzi>?nlEqOmV^e zCB6z2agRi)-D7862w!d_WQHt;MY(sEGwTisR5AMH@(U{dDJ$)V@rcI_6uF(?{D#H{I$-bG|Z!gyFJ{tpioxYh#8TtvJ|oi0LHi*kx_t z#FoJ<{x2~?*8dAF#K*E}+C_5lC30i`f({VQXmJh>I-G0>nRe}g)@&BFonF?N(Uk)^ zuxjWYMt_$ggMTTVxe(*olsX~+HYn4s7;6;&;2^nc0$sY=BnJh08K<8N=r}nRZ!v>g zDl=AC^)dCh1X}?JCFrz*IlIF@?nw8ZmVE?>avK7v9=*5T19euuaPRr{TtXE5VE~|k z&M`Z7$a!C|A;7C97IRDDOz7k*PR~p4{({3g&t@)RehFY?RGAapkF}VVcHru{9q-m% zQ|?-FB+zRV4E}8fWVEf-8@2+s+i0ohITIO?t%_w8cT^J+p*(LD3&oK}dc(p2M-OXM zO<>2hD)=LlGE!OM5|kT3m{Xtm4aNLqIQMJNlx%{6-8JzZJ$4eJWoZv%Dm#L71+=%QRG zLnVlAOHi5r4aDmksh&Tf&BV8S_vMxk2oJnHN2$ z4)n~!K$*=5&Unn6VRXl=$L2qix1OWvvsXDMHgoob46UA5=8^zHpD*O5eJD(tBc&Z^|O^+mCe(#k3ei4E8(+}z&iX^-~$jfp*5(EgJ zb_%KD&bpACza{_>IiI&S^Pi6rwwwOJ9uiiicf~_fH;9}JrA%HThn*X(oqTU%5!trz zTV5zRDH-jR=ECLPzVO_M(x1aUxM1eqF(MBqUdjO^o#@}iACfXZMedZvjXlafpvN6G z4--jF0;9n^5e|~iC#aS{xra>n$e0NBA?JJ!UxI6A$U<2T7&_oHSoHWYXCApQCBYL!PhaCWYikZMoD#=^pPl|LlO0WUUudzIl3KjTXP8gap34jazjW7}2z3K3 zK1eLl5Tu5a3|hO1;mua?dCgQt3rW)j|*RNOXNwY7#i=q8iNL`GgJ>9ca7mba7 zxqVd?(YG&=H%A&%C(jnf(!3l0;r{cv`<}9;mPGsoFNcra?<#DfI}um%IJ;(DiUGU( z?zP_6J|n*$`INln7O$|olpH;?Y#8NwyJp$&{jooZ%jGCtT({3?$Q%QI7-H9Vvbw*6uYd4UE7UORArt%-km#Ns( z%!=ul#3n@msc1Cqf}`~ibVYx_d#m(_cO3%s_h83ctnNW-m)iG;2W&T$Ni93nWdI3% zh~h6tcl&-RQ9pdnVOIH%K+@2Iy@iLW3DDo3uT)y*&z~?FnjY7)kh+ddoth_SZLIx> zZ;?s;w@abcr+R4N$!Gy?8U!L%Wcc&&3KhheLcYPn>H*;;}cG~ab~C4wCv3lVL9KF z*Vqsqsh$;gYPK0xbqFdGD)r>KU=u}p(pV9uq1K3pD0RIK&`lqFB&Cm(j)J1)12Bmy zHw%C4kaO^$GaEZbZO!#NmxenO^k0N;+ll4Lf7(H(Nzk`jfYXJRR=#!?*r!g;TsoGk{y*5g(5kJ9|qr*Y3D?k{lspq3y6x6++_c zAWY6wm1J_JxvTd5x#l{ddfI&?16rh{T7utb!nl=pyV*&^iTYh_itNvrS!n69UI5c6 z^_#DZNfa>Rk68IEv%)T`4C5Ca38#%B-jCwh_|xHp?Xo~-@Cu=ihbh_~(77>_JckV|*6!c6G_Q+bF-Q)uo{oUG*HnzdG??x5j)Ye=$ zPlc$rv^cuKDa=hG>zS|(*`V_d>t3?a$X^#&=fyqUUT?SzPDoNh=#a#)@xyBZuqI`Z zHWKf*FRdHP^f*^diR_OVdSJRaZHLSs`U|=_cdJ%3WuwbiyycFa*o^eJUGD@=@t7q$ zQ6%GX$8l0(sE&U$(V^Ig-pAc5Y^n0R5j&Uc_koQlQ~)NgGK6?}<`c*8f#@=cFU@z2@$PO%?|OW;u)=$SrNA)XxY$qE3Xfi_j|JRAu*$0;)6rUWf#^J*Ml6M!AzYfj@C)o zO1%%c8AwJ=z4CcEo^I`pIpwZh{el@a)jG4e#6kY)g+2|_j^}T!q*rdQO>?mqL<+5^ z%;o+m*q+^-Y5CRjX^)QI%vY6R3T*-rzKd09Pai+Nu5?{HJ?lPW-m(A?8+s#-?m*Mf zJ$N|-0-U{VR6b5-v6?o>OgdR=2^Q&QLuoR=qvupT(mcTvqs)PcOhvq!Of+S)|8F?U zZSiV7Bn5+737HB>s?$rm6)G0MuBtVkAdZ7SxQ!qpv=^LW(I>~{`hTCI207s-IX7r7 zk69Vaj-!lS>-SFF%Zv4fVHmQDw!&72x=D-uf1<7nmly32HM1niBBZ=4`~8^pv{mv4CIxz#SW@kn z68hzhF9@$m_LO>#^l01o^>h8{bBcTnm23P;3phT^Ha7FkUU)Sy?G&czl!h_S(VX|e zociz=9S<9>YA}1jUkYS?Os!_l)fqnXlB~LZ>J+k0Hc_?kArJS+A)$)C-xE_&R~pwa zbf~B9t;8TXU#-db#+!|Z^t{~%9<(TI$D-%6gv)8VW0rriPMH3lQ|>doMo+z^v$qo| zF1NFjzc$jryYDy9#N@c)rg6Tst1tw7t(FxB*|ifO;*+NvzvX6t!`WPQ_9!WU#Ee`H zUi+KFR;^0My>dQ9|A>tUw8#M}p!ic;!arJ6BWn?gaF)QAmS{^_$ANxlok zGpaqQo!?~IZzue`BgL0n8gSHA5ZNjM-NRDurgj?xF;QJNNn)z$+Kh3T2|xcDFZZNR@4{^@4|d|86Li%3XF4RpcPt znG>OXovI(ODEx4P57c1{kE)b16B3gwl`lO~(6hew=*VciQ#n>@ zE)UOFV)<3pTRIo@wAZF~NR4jQK!#$DcvDx%D_WGHwq#?TOO;?QHyepB(E-_stLpG% z0=Xa3TG2GTrpEcQ#PFF@`qCAM-QYD6kHd?}e_4IRgSpSpUP5+%e*pKs9^JXy|K8v8 z%NnLPE->ZvYn>}X>8c%9onbbzDh4b!gX@vsE5>&RmG0kLdnUH1U|)AL=B$jGazYbN zZ=u}zk-m2znbTL4M2;PJFsQi>ab4SICKqu*t}s*i+vf@%Ifk{uDrf?AZfo`oOQynQ z)Mz&X&H{+L1J$5D5BvB|FD3fem}^8puLuPrZJDZ&Vlv#bQZw(T%t#(f9j+dl;S65F z9Jeoj?ZXyyJJBH`^ccPSW{>{aw<=Lae*+0SRSPBYy;Nwq48kkdx7eR0D%AK3zG6yz z-z^ffUjCu+@8ED-cm+G_@_`qy_m0w!CF00S813UWjxqqiH5x8EB zo=|h@`hi&2<$wt*N}_RgLc&qWJ{^M2mzM{CmRyEnS#nsvxh8+CkJ$dSb{4wknrgYP zp=Ztc6CEYNFUa)veX#5i$wYNCYju!?TsgM0l*JY4NH^FHAo#xzIDvsByKT|@=iHK! z)B1&(A+ETMriyqPhWC%(fVwT>@&f0yq0HGalOEf6d$v58a1qbixn&@Eb$RHClZw-Y~A*}>bRo}^)uLs|fI zGW34m??2vXE0McSb$)5$K9bEhMeCAt_m8(VJ-J@u-;jG=8k79w!+~c~PEz~SP`_sP z>8|sQsz5*UkG&1#*FNujyZGIMot~@B{T@zri2gyT_IP5Xhu^?3a}V-(=O>Cr@*JdI z&q`QoNXBQ7d9uB+V{L2Y7WHBEbcY74pk1vIMzq(TJ<_44v$uMc@$SNuf`yI(ey@(O z(BdN_*nK3RCif`rOK8azR3~L;WriC9>Fk!EfK#X`gOvQP^~+B^Yb;@I2&5}^ZQ~Wc zFL%Fas&VB@flFn1Uo5U^oiMCZ8GET_wwX*Ba-v$84@f>YB5y5yMNhdrvanWPhDsA; z)U3zblXcCJLAUcj1DoPr<}q+fY&Qix`o_5BqPb|R8;lbWv33=>yLXyqyH3#*EOlUF zo9i@^{7&4m7OS(tRV(3}&X|R?y0=2~qm1#AtkPnD9r@R8t6j2(v?p0O*;{WmFfqe& zGWh%WdIPL3IaVF+0;#6Pdu9CcG)Y*y=xtw=X?E|)&4RqD7Wxu0`p-1W3ZI$&=s*L_VIi0kLxx)RhKlB;^x))F`dzhYTWOaDrV6oz8 zsd5Ry{IlD$3#gHDpI1&Lt0JaHXSx+WV5pHPTC5K53=Vot-)+PQEo+4-51{%`{Mn1n z;ZJajNzdH!@HjSUI?1-lS2BVZq7~!H8|~JTDH&WK*`Ku4rG@wxz)z4F_ouAHAa(+x%vfMIy>u{@8Ae+*)T1ad-9Rvj=AsNevtJ3YnA72ug+c> z9`6~Nr8^`@?eiB~jDf@j{-FKPwTCC#v{nMrm!=gV^Zf=&<`!xz4hxE z@wC819kLJsBGpLg1A~rgf!uLCQYuEXndNNj zFV7la64la6Ps)Lm6TPn-vpmrL#(C03@W+ps!_7#AoUMk6K4Rn}wJNZr)!ePHw$O=h zi`v$)WTUtUn}rAi!vG@mD)Zq)A+8B^5=!2NuPvGq^G-4Yx>?3n?#yVkS#W4E6CK%!o#wdb;+C!G`zSBV{-qPm?l z7pv|M>0WY@@)eMh@U=UJc=h@c-vNQUf)i#a6o$f#=tKD*&Nrs&WC;xT^_BYNmiZUV zJ(-=Wy%o-@3o>X~H*z1zDiv(qT_fy7&9(Zfga!KPuCQG%tTfFrXq;!NIT$@=_!2qG z%{n^ae5FXTe_Lfwz}-l8z-Ripnf+(zm@NkD@Zf}hOn32m_=eo$@PX+6w#0z>UFx@D zbEJC7yuAIFEQ1PACK^*w$|qX($6PG8TVu(!bhX|J=PEz%-!ub;{OM-j-*r!46_P!Y z?6ObbQda198O{D`aoLjR)dtFs?rrdp&%t$GajCue+x`C6i=^cxGBB;>!A!VIJ=Mz! zg@sohq(tPD>{ttRkioCn5)8s27{I zvAiA3bXpE~U^+~MH1|qlQ$*ZGoz8I7T(hkee?LZt(QJND@pbcWfwx(b_AK z3hd#qzGVf@Y}ub(g3#waYUU=!Xop`U^~kk=Un-6BxuG4p6JmZb#%hOP4^T?O2b=&% z85r8V7#K}t2H>23mI)majp5E*QJMHQr*rM%FJfkOYtXwYsL*rg! zV!MOK6pRX|UPlpICYfdKSq0{VnV3M;x8xw0v3FYpI_I zTg?h78|n|j?!Ert{dv&kaua7pH4#JL()R^iKIr1|@bQa<{6HX~-e&VrQJ^@QJ)go) zF&-Toj~(?8jHOj#eTL?ZYagHesq=1S$XMl>WJKn|^wy(s%n0nM1nu&5sIxkLdNHjd zUS+3kq^tY|Ex6_^r+nMWso_Id{JLvy@#Xz%q0_~#=2XOV<@HjHypQn7e==joK^zNm zv&9!O3VyAmBdrByx+A!2c-R^?go0 zewq%+o7A-qw5@vxW2T=-p_D7X5d0L+z=mXjgz&0~{UKQ=P6f}Mj+BMsr}e7X+Xh#n zwpTah;s4U3))v#VZ5@d%e&}HwMLRcg<&$1_w1;=j0_n*~Q}z_KQO9C*H)q;zm7T+B z(PHpdBmhlP)Y#$gox|%rCFL8UqOR6zPDiT6!oHqvM|OA}nyX$Um*SvjBC-aeBY%N`)!L#lwS`-OXr+}wjxo(H8wttOn-ojtcR-uk3)G*M$b=&=SJ!KuZk->B5}7zxJ!#wUMF|5fv}`gKH+))L47eoM?m1 z)6Bh4wtM{g;`7fnw~XxDD3Q`c74vEz=nY(|J7c5recu{6@~l61d~1Wzk+g*+h*|Cy z*Gy+`L7&!K(*)w~ao48FZB~T*yjR6(+aPxTTjQGA%-(4^m&Tm4de&)K@2;ile>Jx`Qmd zHKp_hoib%Sszc!?W$9Wb6lSk|4&7Gb6dc1nC?4C%hEb%kp8M*B1}Sb*q`dHPzg!fM zNmq_PtNUhiyYa0)eOCO5`8v5qPd(3Q{&rwmq2qen}(7DQsHW|eJ93VpK7 z9di~=89_H`i?(%}o+~?o7u!{$#Se$$tgSRwG_fEh0?>l@E1xG>*j169{i+4PS&5*^ zZfe^ol&Ph+BWFxO`Nwdcr0g(vEpEic6OeB6QZG2sBP5~pXd|lnp47frpY?Lb!d9YV zVLQKSc<-&~iGfG#6&T#*Oazn84v7TC+l-YrV#mioW?F-XcJzPW`%AilHDaDuipdMX zSjc}Gd>;FCyzZ71m^MhnPL|fMmq{5NEwAU$180yE`bW?-$-jexlXefbfMEU+$4z(& zCL6FJlieM6{qyflVOjtE*8)cJFC|$CP8^{{JuBW?$$Umk7nlk><#ePOLq)JT;Uz%^ z&q-w;-a>Q(ohdzIIUA6S(&I>Ujt7(AVGo zM19A`Z_)9&jp)?a^|$ALm-{`*R*EL?Ci)uwQFaT|j;R-FG~A!9RCg;eIFVUtU&ri1 zsV3B9KYKCow-GjRO{ke5Mr936!!+bxV|_BACorDf7u#|($aU8>N3BeD^|9O%R?ykv zF=P~!1y%Vmb&>CY=({ovl~AR+zrjyJQ%*$nKQsNN#pV2YGyTLk4-BS0zuf?r9EZ_$ zAo}>-!r!0*kn`QqN61U#+q9tNpmtt&o8Py;>#9)RG!^hY(?I zr+qzLej&q|LXCWjdtJIXo|=Y42ZG>K#81)oLVf0*7J^OqaI%aFhk|zBStf36g|nB| zgTs;A&X9)aFzrbV+A&c1S<5bX%a=9$4@2OcKXd)cu>zBT5mCuuK3Yij(*P@+=jlwIOT$2PRV+Y(M&aD%Sjv?4xSZVXpAK3w zX0I8vqY_2BUw1kRCiyS&TduB5-3{iGu=*3be#7}sqDu5R8l%C<`56I94rfFoCiqDS z!;jae-9m7T*TGacd)hpWpgzF`S{Wq9?*7Um{O<8GgYwkcLl1wqtNo|rvXW}Z@ow4$S z&-{yEeuLm8PmgcR8#N}3w7gQBhw@Nfz$xw*>UHUgvGN<8El|0DSdCfm4FlM%dP0e_ zN%57au)PjtN92A67z&4EZe`wxYK{dqJ8%%NEY3j7EA=tTQP5#LlK4k2l z-tRUWwT(mmRp^_$*8QK$x@j}W#+(cdFIdg(H2NRfV$caN4WA@ye~j3eXGz(0TSGFi z*Au9FubZp4u#9(}Anxc|onCmnaY;-iSp|I4+aZq~7pNa-qakB+!}`q=B`$OAwH)Vb zs^{cepauDY&sE~_Cb5M(s?pObvW52}E;?|nQlM~dc0F7&q`!0sLkY1Rzk$UynvlT{ zW2@R}K8(jh>$rM%^5E3LBll8ZxRMPMY7o%FITZs02c>&nSg!(f5 zTSNDh+vl?XM9!oox7kfVPKk6~8DAk|pnK0WiJ@w

UfihaCQirj4A9t(VyJ?^5ek z4ZHN@8L}%#vYujuh8w7Z)!H?E-H0tAx&@l?KU*v}MoVK${0ZgI8*l#Z8k)t%nvlPj zFt#*tcL!mw`xw8(c}ovx!BhD*^SOPx)={xPmOn=gTEk!E0qv`@>Q?GIW4;x7^6iJ9 zXl~ZcGxLV8b2A>jAx*6x)9&yUGy=*1CvIOL4}bJ$`z)8N5gdQ4tmukXo4#u7CeSl} zj4HEF3xP^Fvl6tsgUp{=y!^R?0*v+Ui4%;7IV=qirAV1BvCv?u>vg56G>QLy7J!t< zf~~%qD))NlDrp#LhJ*-KI<|$36UPtz@jskbdL2hNT+v1DAeSB)sA?<$$t0di$fbuB zvCwG>d62O?5o5&UX%>^9js{%0fO%j7)!h@)a*YcuZ9ztGY5$lVA-59zkKTWQB{#Xk zS9tcZsl%qBo0!+|Y|2`Pr(AQg8T1VLfL*qi4o%6s~K)I79= z>KyYmPY>71F_5+>EbaX$-QFQsxP$%6eiF_&I^61wwK$?d{h&EK?5-Q*1AIqoiFCitB0nI#V#VzjtzF}mvg+eD~`jM8A@XrKms_g z6Sdyq3#$a?F3ff^LtDpCJ#_rjiP;@bdsia%_(_Qy&f8I1EA_Gbv1!Z;0!d?8s<_pk zVc42rU?Ee0h1{yWD|d!lecQhB^j{!oV;H_XJywME@#}A`bR_r5s)>}l4~C4NbW|#< z9Z=DGOy&0KaNp-7pr|3~A#<@`m={preV`>QAK0OU5Ws(P|1Qp$(_@4Irx(MI^z1|H zs;a#Ac;+7yGW&~+%oDD5L_LqB7lB`Z-RsAVV2=GnmaOTGyRbo8e+W19@%oZf2hmYM zz$k%v!T97HQog~2x(^*y$2W4`%6-bznU!P=*06sxB=5XZfSmWjxAuH@*e*cQVU2ds zBJUUoHa{?zn^7U7JFY<(Un%rvkRjwvXY=r|5hwgZTnJ0RP8+Rd@V0ym zi5RI{`fT6E1lRltHwoBWN0h#fQ>`34{NqRJTr2z97ll8iS=w?Wa;;kgb~JM%WNv#d zW$AFCO!jb$eSW-gO?78wBb*-bvv`f=99cJsZ3{IxgFNUwX$SECb-ceRsg@ObD!G0% z<}7x|HW-Te_R$1V+Bd6uIt^rMi!Lxj;$wgC7O z?`lZ;4#0Tr|HhjrOkl=?Odp5S>&DbX<4d^>P{(ZH!AWCb+M9TFxYY0^(a4>^wZ610 zIaTax#8S)|T3`6J=t-o1V<&sg@qik#%r|mHMTGumBQ8ML zRN-!TH_=g-b`)Cw)P_!x8R%fI&FA*W6aiId>r3G5#;SKE_mO3fYc$~uBMx!@>O0+# zRvFK1HPp2x>)?0VjE)O)t>TvNZxFQ%>!gYjfKf{*XYL|SPbcf8?(Rgyl|xWumR(`h z8_&$Z#>L|Tf4QMMr-uo0q*cVh5+L``>e@bbQ=_+&`Av7~jrjEM8>Ht+J-0Rxfm+TcR6w#L@ zzig~9y%7Jc!N4XBFLM4;1jvC;tH+ z*78+%*U8zL5xIg-IJ9uF5fD=!0<3>dq-ENa9gQpa4?tiY`Zic$!W-nKbIy{A?Os5U z<*9rBtSA4qmRT&hM}aJ?qaozDWIvZ?w%Aty@5sl;2euOeK8t{IH^6fU5QG9>N&tE@ zKEC_>`=kNBM8*Hu8TqUM*O`y+>yiIAui_4f)@S{qKeO-+B3;x%i*8{NH8q hzq|7P+g0Gn?I{RpTIF)D04@ig@m-5M<+oj;{txg)_F@14 literal 0 HcmV?d00001 diff --git a/.well-known/ai-plugin.json b/.well-known/ai-plugin.json new file mode 100644 index 000000000..bc08de0d4 --- /dev/null +++ b/.well-known/ai-plugin.json @@ -0,0 +1,18 @@ +{ + "schema_version": "v1", + "name_for_model": "text processing tools", + "name_for_human": "MetaGPT Text Plugin", + "description_for_model": "Plugins for text processing, including text-to-speech, text-to-image, text-to-vector, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", + "description_for_human": "Plugins for text processing, including text-to-speech, text-to-image, text-to-vector, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", + "auth": { + "type": "none", + }, + "api": { + "type": "openapi", + "url": "https://localhost:8080/.well-known/openapi.yaml", + "has_user_authentication": false + }, + "logo_url": "https://localhost:8080/.well-known/MetaGPT-logo.png", + "contact_email": "hello@contact.com", + "legal_info_url": "http://localhost:8080/legal-info" +} \ No newline at end of file diff --git a/spec/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml similarity index 96% rename from spec/metagpt_oas3_api.yaml rename to .well-known/metagpt_oas3_api.yaml index 70c15d590..e6cf25d86 100644 --- a/spec/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -56,8 +56,9 @@ paths: schema: type: object properties: - result: + wav_data: type: string + format: base64 '400': description: "Bad Request" '500': @@ -96,6 +97,7 @@ paths: properties: image_data: type: string + format: base64 '400': description: "Bad Request" '500': diff --git a/spec/openapi.yaml b/.well-known/openapi.yaml similarity index 100% rename from spec/openapi.yaml rename to .well-known/openapi.yaml diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 5d0001b27..6b1a041f3 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -12,7 +12,7 @@ import base64 import sys sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initalize_enviroment +from metagpt.utils.common import initialize_environment from metagpt.logs import logger from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer diff --git a/metagpt/tools/hello.py b/metagpt/tools/hello.py index 686fba34b..e1bad6456 100644 --- a/metagpt/tools/hello.py +++ b/metagpt/tools/hello.py @@ -22,6 +22,6 @@ def post_greeting(name: str) -> str: if __name__ == "__main__": - app = connexion.AioHttpApp(__name__, specification_dir='../../spec/') + app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') app.add_api("openapi.yaml", arguments={"title": "Hello World Example"}) app.run(port=8080) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index 921629d8c..ef3347b6c 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -10,11 +10,11 @@ from pathlib import Path import sys import connexion sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initalize_enviroment +from metagpt.utils.common import initialize_environment if __name__ == "__main__": - initalize_enviroment() + initialize_environment() - app = connexion.AioHttpApp(__name__, specification_dir='../../spec/') + app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') app.add_api("metagpt_oas3_api.yaml") app.run(port=8080) diff --git a/metagpt/tools/openai_text_2_image.py b/metagpt/tools/openai_text_2_image.py index 3d2a2bbfc..50c007626 100644 --- a/metagpt/tools/openai_text_2_image.py +++ b/metagpt/tools/openai_text_2_image.py @@ -16,7 +16,7 @@ import requests from pydantic import BaseModel sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initalize_enviroment +from metagpt.utils.common import initialize_environment from metagpt.logs import logger @@ -94,7 +94,7 @@ def oas3_openai_text_2_image(text, size_type: str = "1024x1024", openai_api_key= if __name__ == "__main__": - initalize_enviroment() + initialize_environment() v = oas3_openai_text_2_image("Panda emoji") print(v) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index b15c1d186..ea6af7e7c 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -260,10 +260,10 @@ def parse_recipient(text): return recipient.group(1) if recipient else "" -def initalize_enviroment(): +def initialize_environment(): """Load `config/config.yaml` to `os.environ`""" yaml_file_path = Path(__file__).resolve().parent.parent.parent / "config/config.yaml" with open(str(yaml_file_path), "r") as yaml_file: data = yaml.safe_load(yaml_file) for k, v in data.items(): - os.environ[k] = str(v) \ No newline at end of file + os.environ[k] = str(v) From 18ea97fcc60dbdc6de01aff374307735d424a050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 11:12:35 +0800 Subject: [PATCH 042/398] feat: update ai-plugin.json --- .well-known/MetaGPT-logo.png | Bin 50622 -> 0 bytes .well-known/ai-plugin.json | 14 +++---- metagpt/tools/openai_text_2_embedding.py | 47 +++++++++++++++++++++++ 3 files changed, 54 insertions(+), 7 deletions(-) delete mode 100644 .well-known/MetaGPT-logo.png create mode 100644 metagpt/tools/openai_text_2_embedding.py diff --git a/.well-known/MetaGPT-logo.png b/.well-known/MetaGPT-logo.png deleted file mode 100644 index 159517fcd4f62049f43eec4db62e1770d189ae89..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 50622 zcmeEt^;eVs`~Of0QNkdUvJh!SNeKl(r9@%W2x(!A(MSyy0Ra&N0g)0VH^#_~+@!l> zz!=?QG$Y2~v)A{}_@4I<_qp%gXXk9s>v_fFdR&iZq=Ei(HrDH`004mP<%?%;0D!aq z{<~P1=x>fjI8*7j^X@OqyZ`_Ro&PR|9F?R3`b!3{H_z1pMLqm0^bhBpp6WdX07_yn z9NRDg0L_LkpFK5t&#*>h{%rvqrfm5v{<#tNDJGsN$t0VJ?}qSOD9dZ5cbA_Sm|u>1 zbo0i5&gB5ZFE_qK-FaKj%5Y91^TX9(NSs#56V`jnsbf=XUcP43F3tAr?zElLnT5&6 z3tcIZ^OH^^<6x&;P5RFR0Qr^S|L<;a_T1b59-c^Yeqi|Tq2dYPzm))n&M@%)cQ?53 zzc2jH4*&Cq|HZ}s@&i2&{4Y=buN?fZ9Q>~w{I4ARe^m~OqW}Qoe6upg_PJQKjF7HO zHMP@XYtWI*cZt)btyU*4YbUdlG3&|d-+MUKvA0c3wz?#OR_0gU2Y>c@9PfsXT$cc| zS?Pqc35lVsz;PFa9#}B33635qXZxgPo%y}FeH!?~eJrH(ZmuQ41ca(hJRR;nvS?Tk_Po74Tuw6Pm&YFgj; zuVE*HcDfk>#jck=FuWBV?TDi-9DRGd-WF0q!C8=^`s0fg@jrQCWt`q+x2EB5U2zx; znFJC(O=Gs4+Zb#^LCeoTVYMSZhYKeGfz%ON_TH7>-;uXe?-4Rt2#{0#Yf3=^WA;0k zw%pV@v&lN*3xv|i ztYe!7yz)Lt12x=+sMq90V?@>6E`UO=DPnfrhr$_LY-hthaTvMaTumh0mPazheH+neG6e~4Z!=EzDBRR>#7 zcC0m^z=3E6MUGyn(M1MDari&~nsUT-s@!V$g<2b;``=!bB3y_@x`t~#1v0oX343_> z@A=diYh!AgR;tVpb7xN!-R7*W4x`~V);E~|&*QJIkVh%Td9&H3id*&A5 zxokY@2TcMY2jntx$7S(Ge?-~$OKOZgJ{X-Wi5WBaAS~;(n0^=%{V=yTuGLB9?lF8okst^O-5f2kGgG?= zPzNv>*s1PXd3DQ*JHPnGT49#UWLi+r>vqOlP?>gp(y?UW^=&BBm;}?~2BnpavI^v? zMKG9ZF0+M}9nb9^!rfmVcH^%S`oP!Mk0+F7x_}YG0W-&A`C^`p={QUA+F>*_(Ib=k zq1NXL+PXm}RdJ_Cc~ie;(L4qB7DhTHAW#2dP9PzAqDd@|7C5iP=Vpjbd3!2;_?TA@k00R^|#jNeaXFJVmE8KA>AbnBv9%O#R)elua*Ih=|R-S za0bj7En^3&ZE@_QNO52CaE$R2Hz~QS1*HtH+y3lXw)`VEc_8>kty+C{HAj&#PXeYb z&IZN(_T-q+bA~aw{L=;!c`7JWj!PWP2HT-Vjn5`*6<7nKx%~Hl}6-&WbChIcX6EX zo1e>t`;6#7W_b%j*~`Wvj;=pq_1L2{^edfF^Y(+XPTS3%XrGkKXO@#Sl{bYBmS6a9 zJeN!E55(#isQL9Ar=%StpyVqQMls7GOht;Mm2>@Bw}s za5i7Unw&1XhHHfC9YA`O*QIOYaMoDGcT{8E z$#Niz-YlN!a91kOg zv-aH_THKH38n;Kiz3o9PX03g&M!E8->_AaBkv*Z1uY^pf0LmNfjapG6>o7kwzQHmJ zpK})dsBxSW$HVrt4p+DooH7`9T;zv)&3H%GT^`uz`9&jCz*Wz{MN<|VQUUdT30Kv$?+E}pRJ^H1!alLz?E{dqC# zv-}a}4mR%+!C}5OW>7C9-H+>K)azVMiG^HAw?^eAZb&9H14w78ISADbO2)7`yZK(l zYki6O3Q74?}a0nB6mvEAp0q;=&3t3K9$``TrY;>4?&!KBjb z-iha|h=NE$W(-`AWMiiOcsELmlc|ry@(&Q8FMZBhhq*1AQDG>=r=}RIk71;$z=Ao4 z3H(*$=wlFjAd;t6v zCt-}68-F`|E1m16LZPfL=E3^tp`@ilK-~=TM$o+P_Lhnqmv?N<#<8aOfz?lpvJMNj zh4xphVj*d9yW9ywkzYSdWttroBY%Vb%7M!yPIE=xxcJ+I4hwd_TAjy+@xJc_PNIJ! z&SasuN#vL&@(k$nsO^(P0i#4zR)u7juW2Iu>eBF(ymO>-Z!I3(U9&DnLct%1|0JR>}toY?|QPc0W9+D-Y9JZCQ zeaQ?u`lP>(qd zXcRiQ(rp7lUZrjxo$o$uatx{CsIp-#cZ+drqF41^tuPO3Ms~s#2bbT8d>J|TmFj8B zLJcX5G{WmOqz{Ih4qsLY8wM0BxADZzh-hH+z9pF|Vv-5kn@^v+Ns=yqHpw3=%ZQu% zrmBN|IM>?DSY%nnQYDr4%lm!euE}ts0@-YX(N(+N`1sNCz|q^_8e2@UckkOmQ{5P* zK5@FER2%!P&rzq!Up;n%dL>m-j7^?51|M16`$|3F22R+1Dr5V1*C?5){-na1s_pvP znH)WAACmH8alS!gaW65krLrz7@OJL1SrE-y8>w%H=c{ada-4CYF74%A<3$e}i z-Tuug-UP`^l~tnW^mowtBoyNgPrCz8u%1-ivXWWxTN~&5z_h-J-J3&KL(28aMk{3V zz`lBlcWP+C(bQ}%t2y|;=Gz4~){1l*WCycM#0WWdq4xpa;%DxED6&V?p;y5OT-_}r zalJsmF6o_JUrb|>dLWAZJ6gC(n!Q}C;m?I~^qX;S$J&bB54;p<0URu zMS)?Kpz`s_dpm}|_^>i-Bm4;Ta`?D!`!XC&cf}5Omhu(8Gbg_OR_6^1?fyx+B^21L zxJzD+aFeyKvk9I5cN_g9mRyqcc8offe)@`C(ftkL=&|(qA~Tlacvq((*J^T9d zoFJhG>-jZP>M4x>0k3d{qu~!eMrbERiCM6&JCMrEtN_*s^G!V1Z z!v@_Fz-OJn0Zyizp@21b4OX&hX5^HcWFP&O55I6kS(Y%0%P9|jqha!S*!MYGg_da( zsyAk;^=(?%2uVUQM4m;5ID+H8`J(3jMBC836Yc0g+nr^pqyuEssQ199I;8+#oqs+& z%9~UgOQwt%%O0ect`{?W%8_>(x#HROG!K1d6J?a@dHsb~(Kbk;UlQW*qg))}0wp-) zs`P0W%0x&eZhsJna1iF2KMwEatt@%d&&~Fmj~>_FY=m`J1XJDM`dagopUg4|RmS#H zGHF%hPrE-Bb>o9SYJl9W3%eY#s{|HE=jDb%!dV4zv2scP^=EJlaOh}46=?RYv3UDF z5pQx48}Audic9}2+cq=7R}!OC$i_~%CA$PrHl;`Xmm%~ zK1ud)S^{P3t>}f>ni|(;s;qx_iuzHfT2>Xb_yu!8$?s58zGE!UTKK}E^p6wsq|W+b zRC5f*nG%?)9!M&~S(ZIIKQ1JWGjWbD7ph)Bbe+^jy`2~fn63|HUJ{ZkKb&5a-qHHX z+wY5`_iw^e1o@q{DTkxK$x7x)T-0j4pZ7na-vKN`0T ztSEc@;7<4?+Q!t4wVVSp6BwB|VY&kOOqfqx)#5gZWFU+VTn2KoIRuhLP+_*ZX7{rKi%yUbWjWXWaFhIy}9DoQ*;p9OW*Za%v+biY|Oh zG{01bgO;+rsfrm}-UP?PJ%7p(AI0l_mIB6|8nU2geCOM~A1-yvPbmi>L)W+8o?>@q z4(geOelQEZzj0bEe|i#lxCRcSk}-yCOvM9qM<_I^1MThE@^)$4$c)Gi-zXd~DBLEO z-QTN{E*FUwG8i@$wlF@|fVJiO%UMlTcNpfV+ol&88e})@#@M_3$X>T%W9&E6p#aM% zTO^aWaPOEv8QDLo$d$B*^Pm-FoJoJU0v>9!%^9^ui8lqPduss4i0W^)J|2897C!iJ zTZ{Firc?SPH!bjJn2vyXwC{DN4wz5adzPJYHS7|OER4ME7)^w3pD(w~4$ho*d^0>8 z$dK1qI&^LYre0RpOW*SC4`4N|nCpp9Vp(mdn}{nfdS=w^&=FT;d(ZI&_lLxo|F(IV z3`gaGBFfMkgOoYmiEM8(3+ahui$xJNC)kD+&$k^h#4>}4=5I178g#4q{@4=y4-99m zW;V{;k3BeuQRT9*tEiX}Y{cj+J*`5$0$TF9Si1M43qg6o2*1lGorZsiCC5tB;nwnn z=FL7pA~bJ4_I2$@GdX(XU4n8;syr+vZEr!KfC=|mu`DyGkwe)K=8H!~cZugD+54n; zpn??3!+*o!m50A9eFzd3R`A7DxZBBT*Z`+QopAgkjl{8K(Y};%>B>F-gD*3sol=k6 zdme`M{YLB zBA7+u(Ev&NrsDYqHvyEOKCt{L>N31808-3R9_&Buny2AH;>xjcSca9D8rs z@3z*XQaIx?OuWz183we@Vark!uXt61wl}dv%R5)P^NS6d^-B?5TkvJP-ojbkiV2k& z<`_jP9D1ouGd;N`8I)@Ad8<8baekJ^tbv2~{0J=XJL$f9Zz&X;<0 zbpLwzt7}5xG@m1M_b&RDO-ymoRSe@Y3{mB5x{hKcBd`Q04(W<(M9VmK<(bd8e3sCm zY>3zN;4u;;S^6SSC)KsSqs*#PE=aX~Oi?AytN7;omao?im#nM|T{AJn*>5lV0mhH$ z#fhusWZSnxG;bcayNlM{MXOb_{$e7IGPxRcdiV=KxZ#y-CT11&zFa&Ob~@$fCHs+} zRBX`W>@MOfi~$L~e7|lC7~QBCwpXiJ+!f(CR!neTa)=_V$~LM7$?r(a!b=Znvqw(t z-eB(%sVdV(1pY}Ll=TjI2#YiWhBf}a|NYiQDv5~5xAuClwPmsIR+401_q3#yfv*F0 zMLtoM8T)3{b>`F3!o(m>%*4vG>@y@xo!~WCm&qBS{qkMJNZ=_2b0WtyHShU4_H}iv z)dm8JpIls1chBceHIBP}FGBFd?Ak-$jbyG{r&omBA;r6yd-o9w-6##T5L%CbdIb}e zJ4cYe8mM@Y%3xM!p2gf3NXHz;EY!|1IuB&}T{Pb#ADFg2(R+Q0(@pcpn%SagjS_Z0 zEuvp}=n$}$D^;`7Tk>WN<&}k*X+&}qvGot;(H3mTP(fwt(2as{khH{Q{9TsV<-ZZL2vwv}x|gnKEdk+rhMynF7}BFPiD11(2dOw}a{_dm2S-npR_F_ynuNlgN)1m$}sB7XkQt+P9;-5GR_RfXK zfo}BNSnS8p5m#WK+TfAGJ=C$3DAg|xB80BT7d<5p;}TpehQe)5v(A+vV@tra1!t3=(pxUQ^^rT z;2p!K#zW7RRpl9#ismHYO)0NIypg}g*YIm-?7DdHhDPC(cxAsqZ)X$Q*;q`% z`l+Iu_|O-U4J29RO732v|t!Mw@xT?LehSDs&7cc`baqN(ol z@aN^tysHhRYJRVmW~bufHl3F8G;#ia>phQvpiU(XlVmaT%4wm}Q>(nrU_Uhr)-*D3 zB%965@-!aOlaplvbMbz7&3)Jv!A}YfX`aHC@AabzI30Zjk58`hE?~1a51l53ALdSp zS*QD2k~aQ2?Rb49PaaLcTf=8c>{=IXM#JZYz>=&#Ufk*HmM{f#9o*$mx`~+$6QqDu5m}9u) zdTRKrgt9bHRtgwc_U5;ALk^0cWK7c9K02g$)CtP3Kd`7(onbqN$%F<4efJi6i(`6x zMdf|ryFA}|v)S@@9+y`>GpAG`p%+QNO#H}OKfm*XkW_jLKCjNi!M_8S}d5Px0K z!s>tGS{09%Xq|47ZdJnOww!K4Zo{fC#C5~;8$7mtQiEK)Npn|_(=~|_1t7;D6REU- z$gbKQsV&fh3i4yu*i{PoH`~5)a)e96E}SWsvv7rW>T@q{Z@W#CRBA z$@n$%WZ>H&1|>rRZ*ycfqDeCP8AgVOjvOu_{SPvYB=7E*m@9E z@%@p6j;qf%sLVlMT9w4&XGxmCTWS(_35U2VmaT1-1y$n-gU8e1cjc$T6no_W9@ z>f-9bUO(h%BbQL=i|5E4F#Pk7Cn@16FvTszxYZcaa}$s(@-&qs1B8H&FB0Kbm&BUL;wWIkIe^ISlN(<1RVb=PJl{+&DV_dacK`5&aKg*;VoODgCqI^+H(rzg)`UL(~n{Q#M?#+T@8 z>5Z|oqZ7~H5;Y)#amva8l(5XCC3PJL@frY1z3e7V#YCK(Uav*z;Vwi_#`Nel{RhOns z<<9ivt$YLHdV8x5um>f}8tzd2FBzVT7?9A^sz`or79~flA!`7k4yk%$QTtQ1&D`aa zx1#QTjpZ9d^w>HXIZdrH-VDx@IL{X(U9~Az<-W?LiHU^bjTtccn^Npt%L<9r{7rLS z&-ZVgJ8No8M;A%v!*^A9WVl>uad`@TqUc9?B+CRqnh`hgI-yXyTrWbbm<{Q+YPgX&-!4;;v_r84VFZmTo-bqnS=h`eZ18Bkf}k} z)e$puRwdUZ%7V+O0--h-k^ip!$_kv3h3kun$l{N}csUQuebgp@&R>(?0|>Shx9H-QIE z@AQ%D(j{-wRh?AU#1p)yO_Y}5zn=dJK-A zY7+!M`nA1Tl$$L%w(RK3R%Xd_5s-chaCZ3%S*$2TLg!TsJ1M>qS9XUt!hpEFsK?ZPh{tJ+@K;88|JA&3ax;+Z zous^-YBt*l83^YWL*FX7E`IB!Y-OfTU5-0K!_GseA{m-X$gylQmh2_C%D<7SoxO8S z)z;GNa+y0{oLgD@V-B(kp*x zuvS3vo&RFUbhr^!G`s4O$8PORDt1Xodc;XJurLZSIlsRJ{zyY>A$IHj+(SnadfJ#5 z9l6b8VsNi=bJY>S?}JHR4}NPy!1MT4Iotc+uH?=ISP9*oo2)krx^iO&oUB=)q4OR# zbD{;^%g(^L_(^RfN(9h~I1tq?X==LueWqKZG2zyognL*iB-;o*@72vW_$q5`Jur<& z4R-ACO({L;!Of5@c|Ji7h=F-t^ZgNy(J!d~5bA{;tJ-e7<_>q*y2M8Bk_TT315wD~ zR{gWP#g|BJVaY<$1lLnJvp2~WkMw$u2Q;_W9_}H5qdF{&f9VR6rPd40BNxTk ztgIa1tipots(6701izyi*S~9-Ia=Uiut%T554JtZB z|72UMpg|IQOAz#DF53a;R&^8BOVlEO)6fd7b~T;H)z;2mpw9ZBLCv~LGpwmG;k}P$ zN{(sAkW{_f0noR9zg;;!jjPHsVDq2Tk~@b(g4H+wgq_9(`zoQhU`zSGMgE9qRMV<{ z!?SZ(QS2XJNOud3!jTIiiNoc4qf>Y**!0bsj{eqI(5&y$e@t?ynpZ7-EF#^Yn1k#L zKtdlXVt8F?+dPBSh!q?WVFOs52A$G+SNK?%HLhX3`izevs)ds}tr(>`DePe;j*rEwI zYmrb>j*y7qFi4hPhuLbbh$O!pF57gur@!g>#n#}t79{8Ko}_&@gSh-TLdOPHNX=T% zIc(}rx;z9(2|PeEPd{2~VMWy#F;5(U zdPkN+kw(f|hUs2M$`(M35mFz&pS%BZ9NqjyD4#1-u3F$G@4t9@dU z&3s?TZSbQ>$R6IbCebh zAn?Qvx@#b}_ZmH7yiYty_>lFg-p$$z+j;pk4}wFfR3Puy9|-X@KZwADb@ z3!*$MKm@Bjz0lmNfopJrYv~xD{kFq%v?}g(($E4A@rF}|uRj)tTCO*TkMpA+XiMoZ zTO=~uC5ouDF7Y*%1$Rf`pFvW7G|Myf@wu{KPfw_ffRog4z`qb1j1t^;!Q{?ZxQGt! zTjrekph2@YSCfH5w+!$mkjqF-L9CDpAD=Q%|$s%MO^Hy_e{RS~SVS8?FcMIX>@SUB~Juv0UHk z9Z>lei85MI{F^+Q`I_Ry1vr)`R%M?3Q1pP#8bqz{AS&IBingO%5R$N$NxTQyS4U(7 z;(eS(lM-Rw2yaz}Ti)AmP}6fb}BRgnJ9X>4=>%5omAS9M^e4(Q5!^_L~O zTFjzcMZO`f;Fb|$#&lgSo*Ai-I9A#BGcy_NrypALdGi-;H*3$r=msTmRI4)hJ(ADt z@~0ipAz=8+Q^Jb?)vW5&0_K5c>rL`)QSL|Q2NxFk zU}k9_19x)r9jiBXDtb#iuZvg4)c!F$zV_?jtIYWjY)hSkVRrnbY=xe5Yu|r~T`Hn36o+E(lnG{v{1N#OA^EFGC26iGTILkDsTHGUGT1dl z4sN1jzW3yeN~(0j=hFtw^yt&mCmu}bI$zozzE?bkLi&&Nw^#&0$#|Oo@m6c2Q%#!v zNTJ!G6W{onLt;p%2V;pEVlA_IzCD2vV8trg2e29t%}cu_k{MKLWPP_9sbCyzm+GE$ zuEm=M&2I>!c}A~v{ndgHkA(dblsDaKXh|})T;9ndHGufC`Gm>nw(+El$rFWoq}w7s&` zXzyQruqM$+1Nbo|$< zpuCZ=9k3^YuuXIQvISh0sG4vZ4ZmKvu(9FkCAb{DYV*LplXwf9$wWTuudqL|pGj?C zk_nhlDpelc4BS#w^Grw?l&w>`5q>|7M`3UJ-uLZn$B-(H8k3C}sy zf?upM@>7r!C*s}Xn9$pYX4D0^s)2|hnxFyoOcy2g603tw zXQL#Z8NF0(Me)s4V!2F*=BiUy$qB~~WWL)UBsIZHQ7k;6mCt(WV0L4!oetEN)>qim ze11xS^C3R>s<$3Xkr*$xvsIXKun-OdbxzHG($k5ULZFhPAxYhght`JjBF282h~)NV{wGhiJyAghTt#^IChPh=5_0by{b}JK!hj z$NO$odo2+=`FCQ4aTk)q{FO+)sZ+JAcIuI6sNsLbOlE-e8J4`XR zQ<>>e2_HVDzIiWsSBURBCx5CY*w?EhkhS`jSiErnkj2MZo~SgzaMmQ@O7EzNFEQ}( zN>jESs_m>hud2grNz8TMVbyZ&rU?9Otu}H%m6t97$z`%Euye!El01(iuYgWzSP;C+ zBureb0(Tb8HjdUf`(c<95NEp#OmBc*?;2qJE3~$|QbKJLIywG*yext-Qav^&vGGY* zryV^wT$ff85t<4%tXP+28vLVKEuDEa9rv>DT{d#IXjOc@`|Q-J>+Z!Bi?!RNmx2v@W*yjo58RgLs#c5pW9P*PN2fw!UU1X3(~w!zx0?hZ zay^y1!y&KxyS}9-MPR!oA~U`iQ$8+K{r-mh7hNRi)l}_Dg2Kng#(R5x4^Nps9z^*& zL#J;xT8YR9Eq(shu+hLspQZHbB_X6TE{*8(gs7So*b441bTu?h5M*UN4f`a0V!4Y#$#DugT%CA&?skLE4{b)!& z6$5{c?C~JGr&0NFV(UDyd9m3&8pogCeZSp1`DF669Zx5Y)}t~z$E*)NHgV6)O2ngQ z2I9o687nxT5l;?9JN(xlEy&qkdd-D5)r*nr&YV~%_;(_de5bFz+p zVAsO(A%HGtl{LxBGBI8}W1>M`Af4S@ye*%+)2sFv>aXS9x_^3SNv|4|kye(Ew5**t zu_&56;^fD?aH(wi8D>BZEi9r4a|Q6g2#AJ1>K=F7(K@rJk^L+E_3|yI0#?GVm?4(d z_F0V-34Zxr+xAOyRY&u6i~m%s4j%Khk84Pw0MaY-k; zdB{QNWI<_#TQ97D0IePa&42ZZ?F4098Y?(KAGH@tgxfOQG(6VWp%0%li|*>IrAPPR&dQ^FqfCOgNhRqk?mb5xZC;q{%=qTU$EZWzi^ z4IqvSe#kVVprB(L4W$e)RI2dT$Sdo*ntWTjF;n~JeOEaagpE1R;3 zu>i&h_V;K1*Bgk{gzVm-cM8rgBiv;h?S#SJ!8s-nqWw6$)_Iey*MyPwF%%zX_e#N( zU3G2nQ?lcJa|PE-b?w;U8{R0!-y8GX3mOD2)krb8`Te1raZ;^4;M0@a_r;!85yNb`mAx_Lo$qEMZ48OYjxRrJrn1{#|Q2e zFzH-uRk}y&@Y+IhEW=CJFxK(jtKO#1#o@MTIl#Gt*b9|<=P}Mr-s#5QiFV2u_mX3h zwbo<_Eo4&Y;FIA-T%g;j*TJdtJv&0{yX7QENP>hTWdNML&YzK72dLH$s3*b`z;rwqDQHoZG76%G5pHFue1bb}ZP^qz~X(r-6&R zDI0OAZdJOI`wZ1`C#gvExHe_N>lIz*_Y}E*2@M6*9?V8>XK@6-01RE>diy(f%{<>@ zuH|K&y^R0@d8Dtj@=P(PA?uUyemY->v~|Yu2W7>5|MN40P_Jxw$%zs%LjuNO%{BU! zHSbSj!2u~cI3U;{2GY-FvbD=Fa#C|?{AY9eQ$jVdRuc4VPa9E8UQcYk!FNbmEbl-d zu+9IZBR`H-Xbbs|xcg6uSWD8X7Fr@41`amDg8d-CM`#6|(kV!-vT=k@=cW#l;~NG~ zo#(~~)GZ}>jx;X=Z40l^Oq`Fl9bQlCZGzf{@lp9`}gy_Q<49#N+I+ z?r}lueZl8+dQNCiM7@t&w4=&>1M72de}}K4*YP_VslL1wp%%g2$qr3lE`91zQCw0y zJU&rNR8u`F`~p7;_gzi{NA_HYLljQT?ANJUf4`S2@(7f`WyiBWDug?Pq%>)xY9u(i zW3(=N4*p9mSp6f7_~jln7ai|HL`?~c?vO&tDDD>CTuvoVNN#02qxg=eA{U(l1u}n@ z8Bl^Y9+_26Gy3uRxz`b2>4(iogwMk8022A5(eJP1p=RG_6e?YxZ`iY@aD1XiIbJS6 z_J(}lwrbF-5RI-Kx;~Zb5;n3lJ}E}tsOhFa6QYY0%)(<$P5Kn6UT=4o`zw%4WrDo< z(&(pWT-r_s3~@ehHCe8y!D^3pDMD9Z$sc_pkfQjL%{kA1mu~NDJbcLvx|R?o+n|Zw z0?ue%L}20Ifv6IiDhDd7bWQLk_{(Zh%>GtFmgdkedt6Mymq|Yb_Ov3FiMM(eNN}P*A5psqt@dUqQ_i84+p}(6Nj%*y zH^ao^gzfMdV!P(ij2|jCJtKr21>Rn|ciFr>=A^7+h=gM5Pn^{41H~VzC4{mQb=(f? z7d}rp{6mGM)BCca@3d>CIH`L8ZU)}`lmPNRAF^V^oIbHKOi?~mt}m@Py_DK%W`3o>2E1Ro_zMoVk`fWr_bL{p7{HpQ%u6RZ^lsjao60iStQa-x;={FA#si z+fcS_w-Kny5+UTs(43f5CbFa~9YNkQ^mpk|qbpE{XT+P*`&cb4-b_9=ogjB)4gTCc znS~e1DaguH?0s({N5{8HtQ(w@;WH9$F~;?}uYKwXlK1A#$O&uRgbcKABHo8k_pBmN zl2o*6hbw8PbxKyRgDyx;h^KwD?eq+x0`ee@SZNSM{s({Wyb}bJ={L4R_H5A%d?B*E zSu)=0d%;97lGNKhJ!89RsonawEc{&R*Ygx zxv97=x`wD`!w;J?8*zCZ%ig&5#w@Jh>9! zzPfaQh$(;MW=*gDQn{*Hf)Uv#x+>(ay2-Jot&ggr{Jv8%mSOf}&AfMoNQn7C*r2h_ z_%Uuu_SdYHMh~4w_C2A|>+i4pxu#zFQKU99W~v7kE8h3(thK0$amy*hT@=+JUKVRF0h-4{ zz7578bJX%ee{<|=y)iCaNznNAQ_Qu$KZBlm)yT-Fz+{KHUxq?2h_pM#8D#@^oYahTLM+Ah=p5TaN`>|o zg9^LPw!BZP{K-`r#o5A}+t^n{!VXhaTOkWS&SPpQ^KVFH(TT7Nzh{Hyek^rWT$0}* zJ<|(B8)+;ZS^4atgrky)3FLl4W%Z=t_-Z-@dHfY(z( zT=(q;*r4&c9KSC2*N?r*awEQ+JdLcUg-?krDvKnD5R;9Z?0`dbJ!KT~`jK#nC&a=B ze-ND9jB(wMlz!!eJzCEo#Myv-_+dyp_9|o8e%qBTJ*NHY&pu=A;N{cWD3oE*n>5{0 zx6kk@6+gb9^EmNL+T(KlD}S8nyYqL$9;S0njQk5+R4`m~@Of&zw0Q*8t)}*V>Y?U4 z?QlB>7o`$7Q*<;6=IQNBe0M2>Pm{?I97_8G*+8$J7l9QgJk7`=}VdpDM%Qvk(|LJ4ercSqG-Sv+yZKE3IR%wxT3rbi) zvr5prI?BEuW-qjUgiqBJ>$zj1T%c zRVL-d#YO_itEv%o`c`VYi4asHa7iWZRR{WPq3wba@?5K39?4JBX!w!Y93NCBD{daE$vA7uSb^*f6goLV zFNF*ra&%Pis2TCUH2FrmZNEb8cOD*7c{L|i9B=$4;l9*^29&&OoW&kFIQiD`$rCAM z68e3YE>7j`L@IAHzUAkx1(75a{+0Lk5$)U@w>4F5d@|rtAb5 zZse5s{b#j&NqPq3XYp|1l?-w(3-Kf6Wg~d-06qJrZ+la5!86Q z#pboYcTiziZV~ERli7a8~FdvvL*p+u;!s#;~J`uvAr&1xmKuGsT?cdl~*T z?Bzg}LYt_L`EMU_Pi}g`gV*7+KaN|W{U(pn!l&{Um8DzPL!9#8iv%7h&`=xnF&d+%P>)WS4TvLK- zTLq+kR+hcr4n-JQNaa9&7?l;3wEl3n^c32nA9nr356WwljGlWj&Pv>c;E<_x8&^W@ z8_Y-*D!m$h)X3wb$vmR}Bigkv887pG$s!#@!8)vHcyy{28aa=;z=a>#dmik}Jht=@ znyjz%v_*U}Wtz@kaO&AI5om*a}l;=-9yhf-*66+V9r}#0%YC zR(ZT$X@~W58y*Xurzb0T1nYf+r#_<{_2^v7;Jx8ble^8G@(DiSMS}1&80jx|8?8kb zf$thR8v$Q@A7zzX6;wJlhmHQ23>Rerhg?06c_8&O!mftX@|@Fc@WprY!&9XwUD&gA z+&-$rGqnJY+~RUjYlDB!h~{$#8hO)?nQKFNo2Xau0^a|}(^oh&^?zX#B8r5HfJ%K0 zP)QN#QV|tNg;65~M1g^H4ln>|m68VOjWKGB9@5exqee)_7(K=q8}G&6`@Vm`?!EiO z`JD5d=XpMR^~oYfu~TFbKbP644RXi4T*aPSWMpaPQbw^H%be)SX1w7iu-s+EBQJNF zP&kW!JaoRCo$%A^>h-fHqTQm)^@JDLnG`g}cid-*peVjqAj4dGeR=Bc%aJ!wcJXHN zG)l@hY;Xab0FA2R=%bz%v(QQMRl*#*)=bWbSQdWt@ee*LKGA+w*6@r=SHG>*pOM4B zV`y+9GX1#d6ZhN`)~bsW5xn1Y;kvDu89{mN4@S6$G8LZps!#*(U6Tg{P$=LmZ{e{! zK_dM;%t5MtD)V@Ofg1*-kzn=>1v|$HpH;6DJ$LPPePLJgPp32f@zi=WdoVLDE8nZg zlhI3^?zBh&`=$x5Te4Nte+FQiI)OgOy|6JhKND+X9JrNW;)Q)L+w z&*;DJ4by+i>&^Aiq5v&46cu_jubITrG(p)K6&~m*rKs4>A#Y}QPTdw6a1nQ%+$qj^ zRDvv96M+XwKQR76y!uz$`)Dw2p-a+5GDC58FwK5?ooTx^iDYbqSRvz+8Z5Wj7;SGt z`zFn-Ui@hosNeJOW4hefN8>jS7rJfu{{7lDI!3NnaEv$AHE}%0xa1cswhw((EGzTw zqsJf~UWwZ>j}{F;Pb&|;7O&YwTevlUL(;+;gQ1Hr32LJG-EP0>2@F1G8l^rj#o<1K zW~EDcr}xuKGmn(~+&9#Oj65iletDp!yg594%%%%e!cvGBs1vUp8y$1;x&L;Jl4p>d z?h!soqaK6A`^tZmlKA%H%cJBj595oc4F`iy|0wXKP_*Tj;w!M#yk}uQX%^;0l@G-X zjQ;}XBiS+QkI$K{!z@9gF9OuWFl6$hZK>z34ucgLfy;ML z&HbsRuKV+JC4prLby-s{-%RB=b=%L-eDS+Tr#3zv=zN>h#~gKH(=1h0H!^5(xpDMoj>|P`FGFW=u9QY| z@9C-bG!~k2g-ObO;@@|rre@=82pnw6d`K;(EOw$=Im@%7>4PKeP+ttz`~U@{-jPLz zibD0rc8S41VMu+?NEM*F=QFO5CVg))1W zXuEbE32S@P150eo6fveM)QrhCxIye}#`RxUZ6dHX7FR6mLDVTfZAI#SwQGN(V4zdx zispdgnZs^amM8y6XX>1C6gNcfilX*sBYb8I>^sKq$n_ z;d4JYmmIB8ld%u({!ZL~QlVR;%s&yNv{AX4Y(bhf(uY**jL|GCKqtG=n(eMWJ;*ZH z9^OQn304COVB0*-0n$v`*o~cJzheoBl|mT>-J9Nv6R){X8!Ymmn);Lbg$cheHpOPm zF<4=$MxSW8n2ZgMVf=T3yGC!w1EEq4EO zszz@2=5w4E=F`k!8DcPABB1gB>Qgy%b?T`8iSq!&6%*t1vGSc-1U?vhYW2q)?l#l9 z7{EL3j7GgO4QMnLo*hk}kGi=SDHVGhr+lW#AF-YXT{1^-Dn+t%sSz zeSl2AXQ0+N^ScEr@9r|!wB2pXFD4HMhEnT_j32i^=6KmO(rS_BJfl|rQ`bJtTrKt& zy$`a^z6vaxIQULo-U*REKI?^~>*O0Rg_Tb>eh0*Wo{k3-Z+Ok~3m->C!WC35^|XOQ zlV*lUDeS{G8SercR5${>+)HH-ruPxIgsN5LTJ(#=PFYRqB1Y~yI6{z^VI;3a_|8ap z#=%kO9o)0Re+>g#HwC1H@^3fnshu{0`O;`ci|;MV`OP@a?!KpO*?NDAk>wXI;sFbx zf_;p!Rd4^Cdf=MNballBs2P5q=6%LBKH`9L?Dz{mR6PD}x|5rhdEv!-ZN1mAQ{9fZ zo5qRvg`dc~;RMNz&gwtMP+It`4=GNv`#%*hDHd6{FM%uAL^b}e>GRXLX&lSXV=-8- zAFiw?Su+3-Q$@8|l;Tp=>nnc~I^5tlC{)E(qvWWja$A89N^Y%O99^~X8_xV9)wg-# z%1w^mIK`V6PNcW`$zeyHE!vuF9FA}bc>KMsFdIE}if9MUAq)spY9lkD`=*NVAp%dB zF0a^tf^MZ256G{4$^qgY2se=c%^wPeq?#%gJ%_~E-8AxuQt=V8-2C8?WT+g4TY`+( z0GdLh#^s#1L=xOUz%_J+2U`Yza%Mc|G+&rytw+BnSIq^H_PRR9VUI0BmxE<3q;r-O zp>S?-XE*3VCipPy(eEU}fRtfAwKkOA^x_7O$o!LTSB(Wh_3%bBcBTE?4~G@A?}qX; zUxENyJ{MhM6wBSg)^(+I5TER;kDIn}D#FcwJdTCJgN`_Mj#}EUErda0%%^0seuzm$ zb16tyAEcg7jbZuG?xp&d92FhuLEG9FMmm9l3;QpN0O?fCuKz>!!xiLi4secw0^5GP z6tT%R6!?K%%o%b@m3UL%qr<1XDB2vZH=*sV_BS|>Q?J+bCNp$r$w=QSffe;!w6^bV zc8_r&k*MKi#?cnpS{m8y-!OBP(I^r8I>-{(gE4g8|BWzmDE|ktWo$w)Fs_`Ww*p>$ zGNXpxLj?#L!^niX)t#s5GQ&*ZMsHb&Zg{5xt=Hlap0hx)`ej{4|y+g`PCOsc; zK#9%_5q3TZfz-{IV!=d3Q7Da{g{rl~0ZQ>xtIH+%;Z}*G*f1NDoO@}%0Kw6k{E^wy zO`kMnndRCVE7!nRYaYs^n*6V&MNun?uD*yJ7N0;J0zp@v%K6Fal=o*FVD-9$+0Zve z(Xs9o0rp0~a;6tqC!d4hNgUrYI-5q=$|LFJSS@!5{e8Y!y`kgh=e}{AC3Llmo7e2( z!>~<1X%&s^U{O9w_{^aee%^JCv_@=2-sATN745yh$6mif7>K$1MP!5D{F61M1{eJ9 z^SV#llcC~Y@f(S1$~)R;fOctz*I=Dr?cU)^4KL97C*A!qt^7t|?>on@Tl*6)=>}*} z7d2Q>H{ph;=_pGW zGNIbrB7)FACz{sef&PlZDnQS9UPzJ#iSHA~dqaQEwUA;3&iBtw6z?vq)*dJQV95ww zLmuC?p))RF=tT9jFS=#f%((EOwJNNTO3q%je|a}I$(4}PDAXhv0?D`%n|uF&q7&-puwz8 z%T?^rOiI>|^@+gBmpzr|s%Lxhu*SpNk&fG`AFsMoCBv6zaPQ4+pHQDb+%CDAYk%KZ z-g)VTu7OA!euiqua-LN$C_S96U)15{_@=82v?mmNM{zs92KLF903(je9gCtrg|9er zq2+h5))*ZF15hB;OCBJrSl|^VJt?z0BRZp;fAbAdEXwW+&uUL-jt^XO7+Ci%E4`?U zoK0Snw2YN=u1e+eZBg7%vTBSLR1M;ud96BwVrw44Tg*4VftJjDLrkOneb@04zC)mg zQEM(YqHvdGCcPiIQL;@SF*rW*dkZ)5cXgKX@#UN|lWqqVRa-NNEsDCIp-c0k7Dg8+KGJuKOiaLhDK-QGq?=~h6Hi1N5r2r-l z7!I`bpmxVlW`f>&mbv?rF@zok!KF!}Ft?J{J#4i(>k!M=GOzJ~Cp-b0w3Pyf9gY_{ za#i>7Mz(6}-JA1x(V=Tw`-u|~LX*Lx)F*%h$n~3UhU|8x^3mZkycH45+01MOZ8}7C zd5U_HQTM(nk4j}t$#OI|_q@PD-@_HZTr2{JOM|ws0~e3P(a$#IF-qBJ7q6_jc$jwX zDEUwZIY-F`<$Bz3!E=hFD&FKC9YAJ?wXRM`iXjNs$l^9@aJ9Me4<(bX)jU6PHca!_ zMp3>lgVVi2$1FC`XDh5+$xYXff(RclT~^D~tTuoT?WItqM)7t)2fMm8_y$nFYL@Wf zg4f%Y4Ul8niuNBjvCC>hBNP0Bq7X`~daWn6q0yq@OyjoO5h0A#N~<*Uf1oOz){jV# z_5P{T{v7m|SFV8q3&%?-`P3^JCIMZd0C?bZOTFw7W2II7RmC45)F{N`60DhK>Bv?h zb$m-H&QvQiljK@U`+$jLN7@`)MZ{-F%e9Yx7t=OpM^{4z{23~dvQhjML>7f(Jb2++ z4X0k8pCwS)@N7?7Q-VXWr=HqGq zbN~CWWWZNTk2~7{+~~?SzB_1q3vWObcxIY=rp~nZFcrOBN8!n`WvV(8s&i#KN!5=! zXj~1$NE{`>FzwvWs}pPWy0KI8W3U?N*J9p$5obK*iF!x06^1KF37)5l-kNbA+wiGY zJkxujG63|BhgdE6|8aJTZJdNBcfx@b*Z!x(70{Je_ZICvHVZ1 z@omm07Nm=XH=e=5G zEoR%4%gP#%hkF|&jFGBLP>ttW>Mu-ueeglT#QvRgPG6&z^_7#{tiqSGKOjpPw2Cgl zC(8X}y{9&lB_=gP^6kOdaCz1!**DofK+z`fqMaLV;XSSqVs8`ko=u^Y>qDb*Tdn7J z`+QUxtnErm)@HhF)MAV)6a+u4*QY7Ey5R%Ve%GG)$TrhmGD>@{c-CzA(2Bnf8e-|N z0(0Ne#A_99&}RSaB>uwqEN2-dQY$4TlJgpC8%EHRBK)nMhw%4}$H}hKPe*t=>a4_g zL40BD|A}S$*mYdKL06ZWaj$m7%t2oJ1?O{TsVUbuet@$+Y6h!w482nNvM!&C*-Ye81Om8(6iMp zoX|oL=O-fymuLnZn?Z5PZ%LFs!Cd)LmE_ldGSw^Plkd*erJy}Kdr~(VA^~w*)UKrJ za_9ZNceJO$tBn-{4%638>?8}uvml_I^~wPw_bwiNnUmZ}zkT0fpW42&wJpU67W{a# zYHz7K3wvJ7!1=2kI2#huTcPT=!Xw$ed#;^7_kyrWR347s-VyT7Inf2Me9>GSM?DJO zZ=2;0w^CYnY!Of!P%q1R#}BK2f;{$WMF;kv)R2Rv8O7Z`6Wpb^JOq^8|JE!|j|LYS z1D_jk%#J*XTSc!Z(qKUwKVSM-p<*?ay4xJhv;);5{MkW&(Ol2ZAnt+(luea*moTw9k)YvXrSn1MlOP1! z1g)WTqVDy-QBMxo`pKN{_$Ig`vTTuJDHI@Z?=Guy|5d5mhg%Let5taf-!}KD$?YX9 z@9#U@a8p5WG=*x9TrP`DC!&bL+F}JPlX*nvaW!Dui<)^?u}4) zuZde@QrhYgw-4uNd=avUh=dF}hoABUtS<)_VKoJxS?(GosBrM!=EXQ+%Qmpfqt}@a z8(p=ytW*}d35#h$V1ahrYnX_$q-#-`v!v3>IeZshi4`-F$0(RoxYYPp*vLVoBuYt= z`JB{pEUjNpw+yEaCTx!Enl!SIwF=SVC6s0%%^!3J5BNs<^tYUSD2%rM8h~0Ov zUgh)o?PPY%lk05#rP@>GpITJq*1N0kkx)#t=fher*{cYdqF&Z_J2$meo0y0J4HGlJ z&09*YH<}iP)qV*ILUjYPW8*W0R5K4cR!kDPa~XnOlDQ%v6G1v2+8qF$@OGg#6hkK( z$%c_VWqkHyS;z)pO;=}*4f?(Tc(*W=Jab}ou6ncu$H59?^lQB!J*H->*u<%=*%3~; ziXf2|zX#$?(}z?>Gy0m{h=CbeHzzs#S<+(&f5YxW3m5TsV?FY$2ky))9`$?Fn`}E9 zUbROBzfatufJcKUDb_Ppuler|P62z12R@hiA@@nlMQaV*Uj#CN8pnIH{mdITE^8|S zq)0qAP^rq5V?VimqWJ2UMEk{`1{&Gsz$v{beXBuNk+?U>vj8`6C~Pf$jNI-s6jE|o zOxQF|Z^uQk9J%3g51`6{!cT+Tc!BO(qzFyBthJ2JQxcfrcGw8byxm)FFE*p2^o~-D zY2`sj$whcWsm|ExE-_KI=5J4G#1PE|>Xl*vVql0Wwa#ld39qZHLefxh+%USK!bF+$S%6T;7P-j_OY}!F$|eH7Q7Lqy>RhAr#@l^P z+0Re@vn^G!%$eZUMK^BU3IeMgMFQ16TB<(jjTM>TI#*ewzYflc_xg$G*;foi#I_>6 zR*NeaiWMZ6I)^;q6$|(;&QY7Kd#fKZ9Wz#OMa92^+=+FqBMAe1L~nYd0vOfV`aJjl z9bGeCz*kZ<4KXMXux)J$wU#=9o%vjfFx3m%C-mD~tor2mPT@Zasw#kA^_i+_!LIC_ zQ(=F6K+s+ZrT>}v5ZI+QX~9Wq_3Esh(+!zJMJ<88g;`QXnyN74=~9vA?#F;8`1*yA z_Yy`ZdXp2z2c0MmAHMg?#-Zfe9IV>S-(4-y(kD#}A77K~ay1arQRNg(NYcuvBR$<+ ztSi>5=^RR2z6_2rBk|%gyfoyhRgD4_>DsBd`11JJHI3y`bG5DOeH7=d%sZH_#iPB%AxM zP0sPEv~}-D*H^gP8>8+!z^JvJf{h!-n@mS3H_%^R2=ZNfAXhXoa9WAUA5f(*WGkKg zB@jw56U68FVY-@<1MlwGG#C10J+)(Q7(!6AncC-LEK8gz{rW4pGyv5Q^d`%5Z8zZf zA#I4|XPu7erjj{poyd?=6Y)iD3-A7e?r%vBw%YF67*AgSAM2q<(6M@+qg2`e@|a|i z9JZ?~mH@X)3xo)TIbtTWuf{gLK{W^|uG6O%k2w4?T$Irb96=0F_%W~Dj@d0}`L`A% zJ&rS~q$$nd^JkLP7lvdwcv*fDrxeoxU zoP2Hn>z{jEl9yQfdkY<*UfI%E*$axU)Sqoy01I}h-yMX&_N6}nfK4N?!joYW^_{1z zJ>f*TUOTk<;mvg_{`4zNzqv}YEXu^SlxQUvuhro{yEO4piv!OyMNJIr4kn?qMvnD2 z{tUQzUXDfmy&Os{CrtmG*9)pIQA*f~+Au^(Tmz0I(n*YjIjSi?jkf;FUN`zwb?TNN z&9j7oLTrl3n=@eNFyGWsno2{U6#V zXXx)cZc;B077Hh{DIJ z94-3|+HNj1??c&kDW2t-@g9QIh`^yp?do*KsTJ>|Nyf74KsTc9JY!i1mut-X>rXAe z{90o)mo9fxVy|6;Qzvv|EmOAws7%4}S(8=&vDS6ZjVhc6w0L ze@Bs)E_N0BWikstT$`STSn8v)xST^zdENr{O(<+%J)SPwa=Y#t;(TCEOq_-cHt2w> zMv%HEf}lHkzB-&0O!*Lq;0kFB%y;4YEv8xM^&MhANo#F~?7*c-DBWJooMQPA0PFfH zjhuWk!e19g*O=|7CMpoHE=4~GJ&9|n0PJiC^b#BGa((P8@IN&cu^eCgr@m-=gCn<` z!Y!ez9(pjX3Gj{h>!oIlCEib;k5FGjaMv{5kEp^fH(jT{H==(l6#>3r7fv4cgXyYC zt+z^*o~*w^Iro@R=q(*E0`!6q2%US2*m5FKKuY^{ZM=mhiLW7{5ra*#>9zOy7|SEj zl8_9$*3Av(qd2<8Jm3>y`>$p7xqvNp4nzoF`uo!`XD}ePZVPb7;P;pmT_CC}a$rWS zwxUlE5Nhdj1wqSJv+46;IxA7SqvX`B3$B7l7M&pketH;oxB(zo411|CY6;+9HuaW^ ze$Wf}U{L@BlvbcD<-?C?2e@;KVdt`bVO*{XTBo%qK0(RhcU+r@FUwCW^k8IZOWU$8 zkc!yO{r~XnU<+LgkOlbv&VKkY@A(+Z*%e(N>6I`<`NY8Ywb<5d-?ktvNCDWl8V6V> zzxg%?t5uoyB(1aa%r9<5eT<0!&AqznDJ@V)bel*7ch{*K^JMct%6^^(9Fll}a_CVN zpcf&KHFOk$W^OH#L!;E+U7ei4r~sivDqF8$vIfO^`GAGqpI_-T-O0kCOHv9de#n26&lRLMM*7w82#yxNh|c($ zHEdobU2n8f9U}UJ=XfpBEvdF1o?{ZOJMlTkxQI2#=-;pZExQ#J@3~z7Y8KCFS0IaR z?-a0f*+k{eQC?SUx*mdRLH_HX$ z72HWpZfZnB*z;~DsFCoo!gCQ_^9*s;1zJl^a{4tE;mS=%S7rx%FP&H@1%R|`X)O+| z(@hX{#D|y7I%fIpOtvT_lKp3?%ml4+e-OJ+vn$5~{AA%f0~_6g@P8=8S``M>;wXK3 zL>po?Ji@~7YHLXGXv%o8G{P*`6Q6BcRIPBJW$b(n&k9YXqYDvw%5<47gdJ!!>kR-$ z1UJ1b8>R_0UgPL`@HBKL&wV{iIc-SxS{b!qA!3oV9nSvv(4+P5`K{ zseJxJV5Sp5>)*NphKjy^gLHWAxKyMn>CQc*t9}0-_fws-U=m=8gQQQ=XJaRwMc zO=YCm?Zj+4LzkY)($Q(1bP_^Gtb(D%-yE`_Mj^28JLJN}TBa!MC5IUXLbXbFB;an- z<>xjE3AM|s;;tb=D{JMA!nHW_2&8ar|8wd{{d1MC`@V@3PYk$ci})&CX?A}K=7r52 zo}&qKP>;xVz92t*u2@tuZs7!a-1cOb6VWaN?sf z2_gV{>rXU1>lb60tFBAfzCHFdf!Qj>v*(5zTC@5Q%WQ25lGqpN|4x2{SI*3!L)UJF zYwe?)7JbQ?lR(BMplkyBWaT)Bryb#V79#&-0!F}#plPv;BfgQH=jM#*#vU=#8%1Y& zQ>NFtK+!SONwL^^pwF{7EGoy7tt$|GV}=hK5gWyB-!rl zO$LX;AX3BK%*JPD?1f{iyYq$_5G@eDff+j$x26pe+dX%<@i~1t5XJNbO0M!f%$nss zu<3!5$*Vm%(;4U~0k++46CSv$mR9X-R>_Y_oBFYlR4TC?P5$d-zuqSIyf8C;w?SKl zY|OBC{olGcncIIccn^7`a+{IJh$K#fgqe_6F zqp*;NepbjWeuU<$z;P`8svY=smC^VsW9;QS4}24CFS`!FLK64i4E|SqwffN`q_6)pW+951dgpAN4v%x|bffB-`jOYUBa{h> zU5=SE(yGJ-$cm}Nef&qgz#f1;0PjDJ={@@Ce1Rx$pj)R$+v=KOT4CZ)-U=~f1BiJ4 zvev4Tz9iOgJDJxoj^CTu{*iVY==DEj0$EWl_+USuKtCUDjmlYE?hhmyYp>M{7iN zqeK8}znM{?jx5kVOjMLab-u&TpR@>z+|a*%7+pSHXUa7LILoGuKj1nUEW`HdJ0i_1 z-lMuHq5zEglhBFNE>NYY_@^~3j*D@-q0fP)?7BTlwq$jr-@e~}?@P#m|4QJUx#PsX z?kr%hy5~45h=O4ULaCub>F+rft^*1gS9hAG8fSJIXxrg2;BR;u2Drz27Q;gS0%R`r z5EmlW=zzM9)vNn&+%rBU%?5CWZ*-%&IQw}!g;rJ#rJ*0C&FyGM+L$P&Pecr8Rc?@a zmXDugrq_2NL}LlqS3}RMs5GUSgA8-GX~|P7M%p2#%YK~v7f?Im%hEuWY9q9t3u*Pa2FQ zj0$vmNV~V!V!mun9)RiVVyNT#D3-gx{{2@%{65Vl7OXb_eb}#hNOJt1_zQjSB5DKu zea`$7t^l{z72a*aKjIWlKaGoD#6P#+cs4An^ioe$j5atCD4Zsk5$?|h*0YJeFCrpHM&ovwdC zPEjMvB^TiJzor|5*vh^S9y&jND7Sv0WTU=+5VSb9D;KW|96+7xa?)oTpP&?%<9_tF zXBE0FSG{?NSBPsA5{zZ?j1p;5Fl*16eE|C3$oa&XGDo|(4?T>#gySc#A#p8T?4>(+ zuQVMs)A3LEA;3i;D21abDC6Da#XHN7R+CG|UI9B%pz^pka0;2FnvoUXhHC$@{~x>Kd zUSh3$?lJU0(tRUBjVqh?mINpXxK!uoOQC-irZl8GYq0>?>kS%~KM=Wc zHNVBFWFz$z1ld-$u9n)g15Q0fh-<{rc)P|Tj&EetZNp|3)sc^x+&HU^hjK=5&dOIi z^MZsVM$QBix~+cMFHeg%ciROE8@fHX4v<3s)l4Ok;)c=={grEOoBHnn4-5s# zYrZdbF1z(rz)s(KKbB>?>i>1Y>DUht7<32a7A|uBc+Z6ZtcBhNpRPN}u3rJT)i;<8 z|6QXnPSss_L!x{s31m}#DG(>p5cH#xrQUQ2{xc}pI#_VjagwEYUYHga=!>2mDu(w5 z>jxcg&4Av>iN-t8MBU0Rv$Xh zOO(#_c18swvQ%;N{h%3txL5Icr=c&I>B+8MMxYH7a^eqhGoASA@mNnzm8 z09)Z1SNZA7D=t8$5jB2i5agT>Ta#2-Q8m`10Zy+y!D81f|7^~KS-w0WFs5@fnenye znprtL*84odSoun;LWKsf9E3VEj%8O9EJ#j(i4Q?%WIHwQ5IGUmN_FFcwZ;j|KFhNb1iZBev>jdy`S@DTTFjL+Ly;qQlQ9tZkB*VW>+2ne z52>2ZoRLQb0jI*>@d>K}Jff`_g54DRD0N~hV5<~3NkS#VH2dACZh>O)!|?fRc!8tS zNaE~*yl|7icFnf0L(pvfPE;1vG-UBtwVE33VE&~{$uU!%ATX=$APcR>y!Vp|Pl(Tj zg4^dVphjeRJj=YYyvlm#9o;n}$RZ{d}hfERb?Rwps}3P$ps|4geI6w#ZuTSC`!jxa!Hl+@dn6-!t5 z1(8$=2*6&-R5wz_9mq_)J<|a3Ox#v&3+P8rtL=8bNs@U~=sI#c^@IB9Ui0{}Z9S@`X$KyD?$f(34ZSp?aXoU*f_B5)&&d8(##|sAUn1@U^xfX9w4gx#{;-9^ z7(7YjXxbN>>%QoR@qX&#^d;sF+wu+pxB3A7uF22ac=^_l9=2}y0fG^=jsFle- z<~v+Wu3)5AOTWm6>EST{#8VAF~Q)JNm5fFP2C+xRF<3q(ypzWNj+GT zDc5NfFdFu^jNr^5LR+LjpQ~p{Ew1bTWmMpUkjkXdc7-~6a~fYHgBR<@>UoKHRY9(< zc{2yWPdd2W-N3uCPBB#5tK{|p+HHqGn>xClp}ke?(eWjG@cU`tc47YqpwUa?EMbbXs>LHP-Hba$`I9#2*NBQ@c&!WX!^JS_pGA^E7#%P^jB%Uo{ zGXsP}71OGf_L^`cON+1qv&>d^TQjeEx#hh1QRQ^m`(K_bd41!8;7xJox!nQt_P_~y z=|y<5u?NPe#J73}j3r_~(8{YtSAm0<5%7{10?Bx-8PX?>oahS*GLPVS7UK!9dIG(y z?!$e&jRwdUaF=!~Iqk5~3JM_s7R`RR5hty}CV964Whn<6@Z~tpv_C9W7qGCYf!!fN zJUjpGOz(uo2@)Ao35f zU^d5!Y$eE#{|;0p4DRwYd$Aw*N_;CB4gOv~o z57^q6{q04I;+Ksk1G=lG>^nvUKfjmrKy#3ckPm~hYIMyE;hvQ#r*~>AkA3u_WL=*eSc@xl>unP(N4_{y->anB(_7 zk!)G}qdS|Tx6pG>Ht2qV@uCwBhe zf_TgtqxRoYmjIBV@A~fZA!mL8V{O)v{Vc)Y&7j)9CE!yg7lACKWPJv>qf8pehNgjX z*#B7~xTJ`|R&$JdPPCtz_!+iw2rF{&(H&vb@}p$GzAB7ks7zz$Z+U9~P<%bE5Pt%P zXfCN90iw6O+%I!B=hgE+-~Wh|S@onFz6aHy5rfqt)Y{yqzm8Rp5_;2!+R9B408IeX z;Gb@oAOsPV)QgoZhx)IjNH2g#kF`UR;js8D1We)u11u8Y!hI1zKLiG-he!!O7C!M& zX!*2U(&)$w`|kfB;cXbIOlpk0+W#cNI^l=A!y`np2v;Dp9MR0I_ z6}!Aq;1s0N7y1VYCte$xQWa!LB;z_^J3vS1%h^KbI~jFz4OZQ3#;TWF(GieeZ{g*Z zqiLmjy0CKC903XT8EbAhbfWG!#Ay~?hI=($UmQe?Dv+$aB&zjAIOZ>!-aS@Oc#~C3 zuCxBDR=hJqsHUa>webm`C+>=pcZnev$olA36nE2YX6PJlr){0-n_-|Xi!g^zJP!IVKU@|@c! z%V5@tO+jLepqeG!9ybshY}f1iK$)_dT$ib}N0R{yPR2CF(inonMR(@_OTG`U@BMw= z9YBCp&mX-QCRAL17qrxc?!nD8W_q?5m2ey_=XJkuy z&@x&VtfPJ!D=oy4+tMBE4`VRgDhu$L_hwH(UC?GW(8~Swq)~dxZvPGs;!O9<(FFfJ za&9y}^_@XBdKAQ(kH)`lR?Mv@8uD?Kp?eLmkZo5}1f!Yn}4T^;@vetb9UAl)!2pYK^3Y)YOj)AWvS#~Sjp zCBOxG2wwb%EU`LTm-6EKT7^(RJi6r+0=OWDPL4@rPGbUj1wR$0lXb{TM4vN%L5;b~ zCj=Z0Jgzwi5^Mn6U9*5rNOluo_Yk~Cta$VDoMs~W?Q{AUq0cb|#^2V;*!$fE z4d2}2Ha7L{H1!!vb*CAtUeBnbMOyU$wHxP0@1i7G^|D?;r9X*lH67?;+_&A3tE0C& zJtP?R-txq<$58-Zo`hrQg60LPugzHUGjiX@y9&6X(Kbv(c6I2*_%J(L&qcZFSGc&) zb6Lp)>mCJwDIw28==fhSaScgms{kDK{;}U)#0%?u6RDg{Hd^sdySoqGSx6^q606Tx z!u(SM_X+KNc|&O>jIu#gM`y-;H1X$dWSZ(%*E_>!mG(qFSv5AvWr<9QG5vY<&HZJu zOy6Z$s)8KW%wEtAoV?kv%)QeOkv$Vf#rAAUnAK`w&2Qx2<$VM%Jh;h9GOnHDQI;&a#Tx{-q#_Pq-pua4Hy@Brx0ep2$M7A{5OBpx z21M9#WnJ<}zsY-1a>_@;uY7FhmWQACxlyj1yHP!sQRj97zQ61F!JZaMA=_T|;{BSd zJN0m%){Dynlp#S&oK}*;pn4Cf*}S2FoH||sWwsKClY2}PW@^``%v_opQnv3io$jzm z1*c!waC$t0Srrgz`ue5zxreXqL!ICezyhI%3VoI4JMjcTV%?9=^_f{#ai`gHo+91@ z0(N(}I+zSyFYg4@0zUxKYS~#%*l|z()FZ(A*1vWkU1_0CbR%BAs5qrW8Ui+ej|_y} zeI3{wCH^O+XsygiXyoTcb8a=j=LPU3=IEw}OMRJ=Hs^$1I0a%)t=N?r7QkBblOPuw zWnL;5P~{I+NT0TxsB;d%BAHq=)etj0Z-E_rp0{zdEJ*AE|7Y9tNtaNz0; z-;(`FB`}7?GvDSV?B2K`XKwsY6GV^q!C&5mN(9|xaZi5}97N8LTn%dM%Oj4t3|#u) z`euBjwcN1Y&B)QB|CNyMRL)1cJavu-eGPVuqN7Uw>+;Kh|T*K=8W?6`Twnx(da^nA;KMWM^^P^BZrDeKV#+p>7 z0?6*#p1t)=yvmL>f_Q?tSO2CNfo*-cgCk(o6KnW7fZfC+xehe3^Oq;5U?#`+f}_P0 zn6XCbWV$QNZmZU7aBMr!l%%q4?fmqeh-QrM#E|hAM^NUm#M=OORvH9k7I94>GcBUE zB3d$F!>*TAVG z>WLbcDVg^sSBRN!>HVhH{pRfe%I`HqNsDeWmY9V=&+Kz){f2buoZKo{ySCJ5*W3)s z=Nz@5ovU4+RB9#Pk73VLl$V{aEqb@Eb`L~`>~qqnyK2C^cYm365?X$@J+AgHZr-fU zf+%bBDs8P?d&ev+vg%yRuO!WDfl&$Q7x$>FKksQO`8P~`V2WDc6jZGsI#ANHQvpvc z10sFG1AYKx6L4UDjJ0`|KxJ|QELc7`D(-&T?3x@OD9TO^>&kN!@wXt%DciLpKBm63 z)N^wEbN_A8Zf8nh-%6n5l3VoL~=h@{w=f(yKehj`7GCt9j8M@WEH9zwEDxQPxn@Leqv{ z*@Z7U2RB`lt4ndf$M+!;y6(}<+>P7Q|{VMX6$aCUrmi6*X1!#JRI zV)pgRaUxhZbQFZp*~foq=a(^Xs`9Ye{QKiNwNgni!9x%8L|=S=b7A#Qp%%YBl?nD9 zsHh{y+&cN4Ds__(Utz7fk^+^xi?fv8M#(h$_K~NwpMxRjSd&(Psqi{9^bgBGVYQ7ca99w;7L= zQ!n=)1w73Y^a!3`dozeW5!d7pkJcc5k0=PuLS;0Q3MZPh&DHh#jh^gIQjC>kmKyQn zP;q6`7L-g-*AvD^x7rq?8Wg`I4akl2n4s<9fGAUAJZ$D#t5rsNXQ)H4UJEa?skG5| z>~_y=rElefM+Y+bqcWv>vEsuiMb!>(g>962=C-|1GY%C`O|I%Sl0x}wZw z%;=g1bI+Wt_vhi^w*i`k5(*UW+Wnn?R@<8oRX#PhZ@BS<-z|mz^e+M_CQevkOs4v~ zzB8iZ0V^L-g*c!HyGGt;BWd!{ge2WchIr374kih7B^KlV$o=P;KN?Q(O)ChpToZ)S z{25S~$!XX9l3NRZ0{CvM=y;0+XVkqbqA_3r06-*97i8U2>R0{e>2EM#y%4aIyQ!M} zXj2Qw-tIsFs=lIO802#*Ovnr*JjyTothC7wV(v-M5e#%f#n&1{M*_b=7ET7e_}Ya1ybJ4uXpDMsKJlb!K?c zF+$>G6~P=7fc;YPDy0H834d>IT7WCi0Cfmye`yL>LN0s)S}iN$URV9?UjSRLaMNyj zgh8a8R+cFXNgZy7lBSY5_+H&Ma;@1a+3aRQ(52U2)7u}A=q>$r*}qjDuHB+v{I1dM z+7S26N_Ty385KoKImjXXPS~+uhUZ-2Ot7J@{JQXONRtikfAOlXrRufNbXp~4VY_ZQ zQW!nfPwjHsHGdog#F5~D>;x-}AawKRyQ760yNT9bE{-#n$79>xwC(5kZ(aeOI~|W3 z8t$pLzc<6UTt#PH?STc;? z=;$^_Bug&UI5O=bV_+TIOM&xg5~t7`yAh2uLbuXszp8f?YJmF;`e|U_=&Cvqsl0vT zc9}&&oEZP#G}t{}T=u-%yc7j{Q$yvPDYLu{2M`qqV zjXyssfo19cwTf6O=>J_^IYZs>d3o#8-;Mjk60q4UDd~C|3MCQsX$``Si6#8ZXDM45 zgo`vgY6S>xIUk6-3hud&Zz;WA?lA+6I<)%P{{RQ-1Rian?tuKFa2=H*C{7CHU>SGM z?!e4_NrTee{1AR?c^9yk`3je$YQEu@ME}=G2Ll6LD)5~dy{I#O~-Op3hvX1Bo1qx7DTSga zLd>Daan6Sgn=LtoY&johhUIMLJa(}A{kiWy;r{je%N~#YaJ{$dy5868dYzunXYu30 zaQRrW2l_NiYIkEKgbh1vldTi$%s#ZJf?Yn(R|51y-3O4yzNjC;C*S7(rwhAJ{%WUG zBG#fDzbSBBvEj)J@P&c!OA#T#WHU#P$8}l(%&tV*>mXd|yXdl#E~ZYFHJ|@4ei-K zsEx8X4r^YzE55_xHUyDp#~oaH(Y;Sy_=gWtl77G&m*vb}Ms=%QyuXFNG(vlfx@?IcxyMDb|ro>t(g3tT&$WAR_mR|Xg0 z-C)|^?Q)%EXJldorzvPR#k2cs^8xu$Q{9JK5j4OOEGUW16X;p3J(w9-~=WxJ*GZdZv87f2wS zFAP(3svz9u?%>{0YMKyBukxocv;R(0wrWtb<)R^rG$_s14 z9C!CMK}nOcT666F4fQ}B@7R^Xu~TPE&DP;(ERj8Zcf5EU>)@d;eUGMhqytX}gx^<5 zaKAk`B4=8$Hb_>zB=H2WwvCjxVoROONKUYfzH5=oZdKOpZO^-ox-m++n_c)PN4}1o z-2Zib(P`fnVPw=tD>ae5g1Dn!OaG7hAr0g*BQ|7yNo2~1zo1eSf(Yx6{yP2a#$ykj z{j$I3fV+;1x_i$nlrngyX>}R6YdsMPWlud;9{+ZIHwX^SZqLeVwCrzX#1DT>7kzmg zz9O&h%-$N6N~AM!grEh{83)aAN8ys|RkwW%G8DB~aXybylx=O0Q!V;AzoECjGX*rv zjTM^}Iqm8rGedLF9V-nA401SjPnNzHTr~W_vG(Z~W@xlNlkENWVraQx{Tq|6f+6m_ za@nm6?-&kic<&y0?>b1VNMQf9`v=hL_Ds3uZ7E&yn5O{NWmIQoq~h0#T-HS(D^U>z z?ljISAY6 zXOx7c*AuQ1kKEpxlY^0Jp)5X#>o1a_Q?)?Fr?rmdiw^$mRP?jz{%ln(-$oQVm2?&V zVDhyO0NMR^pyFTlYRO&+x-Cpq#6fHH>oRm$_#9R?^HP{z(%Ebo@Hzi>?nlEqOmV^e zCB6z2agRi)-D7862w!d_WQHt;MY(sEGwTisR5AMH@(U{dDJ$)V@rcI_6uF(?{D#H{I$-bG|Z!gyFJ{tpioxYh#8TtvJ|oi0LHi*kx_t z#FoJ<{x2~?*8dAF#K*E}+C_5lC30i`f({VQXmJh>I-G0>nRe}g)@&BFonF?N(Uk)^ zuxjWYMt_$ggMTTVxe(*olsX~+HYn4s7;6;&;2^nc0$sY=BnJh08K<8N=r}nRZ!v>g zDl=AC^)dCh1X}?JCFrz*IlIF@?nw8ZmVE?>avK7v9=*5T19euuaPRr{TtXE5VE~|k z&M`Z7$a!C|A;7C97IRDDOz7k*PR~p4{({3g&t@)RehFY?RGAapkF}VVcHru{9q-m% zQ|?-FB+zRV4E}8fWVEf-8@2+s+i0ohITIO?t%_w8cT^J+p*(LD3&oK}dc(p2M-OXM zO<>2hD)=LlGE!OM5|kT3m{Xtm4aNLqIQMJNlx%{6-8JzZJ$4eJWoZv%Dm#L71+=%QRG zLnVlAOHi5r4aDmksh&Tf&BV8S_vMxk2oJnHN2$ z4)n~!K$*=5&Unn6VRXl=$L2qix1OWvvsXDMHgoob46UA5=8^zHpD*O5eJD(tBc&Z^|O^+mCe(#k3ei4E8(+}z&iX^-~$jfp*5(EgJ zb_%KD&bpACza{_>IiI&S^Pi6rwwwOJ9uiiicf~_fH;9}JrA%HThn*X(oqTU%5!trz zTV5zRDH-jR=ECLPzVO_M(x1aUxM1eqF(MBqUdjO^o#@}iACfXZMedZvjXlafpvN6G z4--jF0;9n^5e|~iC#aS{xra>n$e0NBA?JJ!UxI6A$U<2T7&_oHSoHWYXCApQCBYL!PhaCWYikZMoD#=^pPl|LlO0WUUudzIl3KjTXP8gap34jazjW7}2z3K3 zK1eLl5Tu5a3|hO1;mua?dCgQt3rW)j|*RNOXNwY7#i=q8iNL`GgJ>9ca7mba7 zxqVd?(YG&=H%A&%C(jnf(!3l0;r{cv`<}9;mPGsoFNcra?<#DfI}um%IJ;(DiUGU( z?zP_6J|n*$`INln7O$|olpH;?Y#8NwyJp$&{jooZ%jGCtT({3?$Q%QI7-H9Vvbw*6uYd4UE7UORArt%-km#Ns( z%!=ul#3n@msc1Cqf}`~ibVYx_d#m(_cO3%s_h83ctnNW-m)iG;2W&T$Ni93nWdI3% zh~h6tcl&-RQ9pdnVOIH%K+@2Iy@iLW3DDo3uT)y*&z~?FnjY7)kh+ddoth_SZLIx> zZ;?s;w@abcr+R4N$!Gy?8U!L%Wcc&&3KhheLcYPn>H*;;}cG~ab~C4wCv3lVL9KF z*Vqsqsh$;gYPK0xbqFdGD)r>KU=u}p(pV9uq1K3pD0RIK&`lqFB&Cm(j)J1)12Bmy zHw%C4kaO^$GaEZbZO!#NmxenO^k0N;+ll4Lf7(H(Nzk`jfYXJRR=#!?*r!g;TsoGk{y*5g(5kJ9|qr*Y3D?k{lspq3y6x6++_c zAWY6wm1J_JxvTd5x#l{ddfI&?16rh{T7utb!nl=pyV*&^iTYh_itNvrS!n69UI5c6 z^_#DZNfa>Rk68IEv%)T`4C5Ca38#%B-jCwh_|xHp?Xo~-@Cu=ihbh_~(77>_JckV|*6!c6G_Q+bF-Q)uo{oUG*HnzdG??x5j)Ye=$ zPlc$rv^cuKDa=hG>zS|(*`V_d>t3?a$X^#&=fyqUUT?SzPDoNh=#a#)@xyBZuqI`Z zHWKf*FRdHP^f*^diR_OVdSJRaZHLSs`U|=_cdJ%3WuwbiyycFa*o^eJUGD@=@t7q$ zQ6%GX$8l0(sE&U$(V^Ig-pAc5Y^n0R5j&Uc_koQlQ~)NgGK6?}<`c*8f#@=cFU@z2@$PO%?|OW;u)=$SrNA)XxY$qE3Xfi_j|JRAu*$0;)6rUWf#^J*Ml6M!AzYfj@C)o zO1%%c8AwJ=z4CcEo^I`pIpwZh{el@a)jG4e#6kY)g+2|_j^}T!q*rdQO>?mqL<+5^ z%;o+m*q+^-Y5CRjX^)QI%vY6R3T*-rzKd09Pai+Nu5?{HJ?lPW-m(A?8+s#-?m*Mf zJ$N|-0-U{VR6b5-v6?o>OgdR=2^Q&QLuoR=qvupT(mcTvqs)PcOhvq!Of+S)|8F?U zZSiV7Bn5+737HB>s?$rm6)G0MuBtVkAdZ7SxQ!qpv=^LW(I>~{`hTCI207s-IX7r7 zk69Vaj-!lS>-SFF%Zv4fVHmQDw!&72x=D-uf1<7nmly32HM1niBBZ=4`~8^pv{mv4CIxz#SW@kn z68hzhF9@$m_LO>#^l01o^>h8{bBcTnm23P;3phT^Ha7FkUU)Sy?G&czl!h_S(VX|e zociz=9S<9>YA}1jUkYS?Os!_l)fqnXlB~LZ>J+k0Hc_?kArJS+A)$)C-xE_&R~pwa zbf~B9t;8TXU#-db#+!|Z^t{~%9<(TI$D-%6gv)8VW0rriPMH3lQ|>doMo+z^v$qo| zF1NFjzc$jryYDy9#N@c)rg6Tst1tw7t(FxB*|ifO;*+NvzvX6t!`WPQ_9!WU#Ee`H zUi+KFR;^0My>dQ9|A>tUw8#M}p!ic;!arJ6BWn?gaF)QAmS{^_$ANxlok zGpaqQo!?~IZzue`BgL0n8gSHA5ZNjM-NRDurgj?xF;QJNNn)z$+Kh3T2|xcDFZZNR@4{^@4|d|86Li%3XF4RpcPt znG>OXovI(ODEx4P57c1{kE)b16B3gwl`lO~(6hew=*VciQ#n>@ zE)UOFV)<3pTRIo@wAZF~NR4jQK!#$DcvDx%D_WGHwq#?TOO;?QHyepB(E-_stLpG% z0=Xa3TG2GTrpEcQ#PFF@`qCAM-QYD6kHd?}e_4IRgSpSpUP5+%e*pKs9^JXy|K8v8 z%NnLPE->ZvYn>}X>8c%9onbbzDh4b!gX@vsE5>&RmG0kLdnUH1U|)AL=B$jGazYbN zZ=u}zk-m2znbTL4M2;PJFsQi>ab4SICKqu*t}s*i+vf@%Ifk{uDrf?AZfo`oOQynQ z)Mz&X&H{+L1J$5D5BvB|FD3fem}^8puLuPrZJDZ&Vlv#bQZw(T%t#(f9j+dl;S65F z9Jeoj?ZXyyJJBH`^ccPSW{>{aw<=Lae*+0SRSPBYy;Nwq48kkdx7eR0D%AK3zG6yz z-z^ffUjCu+@8ED-cm+G_@_`qy_m0w!CF00S813UWjxqqiH5x8EB zo=|h@`hi&2<$wt*N}_RgLc&qWJ{^M2mzM{CmRyEnS#nsvxh8+CkJ$dSb{4wknrgYP zp=Ztc6CEYNFUa)veX#5i$wYNCYju!?TsgM0l*JY4NH^FHAo#xzIDvsByKT|@=iHK! z)B1&(A+ETMriyqPhWC%(fVwT>@&f0yq0HGalOEf6d$v58a1qbixn&@Eb$RHClZw-Y~A*}>bRo}^)uLs|fI zGW34m??2vXE0McSb$)5$K9bEhMeCAt_m8(VJ-J@u-;jG=8k79w!+~c~PEz~SP`_sP z>8|sQsz5*UkG&1#*FNujyZGIMot~@B{T@zri2gyT_IP5Xhu^?3a}V-(=O>Cr@*JdI z&q`QoNXBQ7d9uB+V{L2Y7WHBEbcY74pk1vIMzq(TJ<_44v$uMc@$SNuf`yI(ey@(O z(BdN_*nK3RCif`rOK8azR3~L;WriC9>Fk!EfK#X`gOvQP^~+B^Yb;@I2&5}^ZQ~Wc zFL%Fas&VB@flFn1Uo5U^oiMCZ8GET_wwX*Ba-v$84@f>YB5y5yMNhdrvanWPhDsA; z)U3zblXcCJLAUcj1DoPr<}q+fY&Qix`o_5BqPb|R8;lbWv33=>yLXyqyH3#*EOlUF zo9i@^{7&4m7OS(tRV(3}&X|R?y0=2~qm1#AtkPnD9r@R8t6j2(v?p0O*;{WmFfqe& zGWh%WdIPL3IaVF+0;#6Pdu9CcG)Y*y=xtw=X?E|)&4RqD7Wxu0`p-1W3ZI$&=s*L_VIi0kLxx)RhKlB;^x))F`dzhYTWOaDrV6oz8 zsd5Ry{IlD$3#gHDpI1&Lt0JaHXSx+WV5pHPTC5K53=Vot-)+PQEo+4-51{%`{Mn1n z;ZJajNzdH!@HjSUI?1-lS2BVZq7~!H8|~JTDH&WK*`Ku4rG@wxz)z4F_ouAHAa(+x%vfMIy>u{@8Ae+*)T1ad-9Rvj=AsNevtJ3YnA72ug+c> z9`6~Nr8^`@?eiB~jDf@j{-FKPwTCC#v{nMrm!=gV^Zf=&<`!xz4hxE z@wC819kLJsBGpLg1A~rgf!uLCQYuEXndNNj zFV7la64la6Ps)Lm6TPn-vpmrL#(C03@W+ps!_7#AoUMk6K4Rn}wJNZr)!ePHw$O=h zi`v$)WTUtUn}rAi!vG@mD)Zq)A+8B^5=!2NuPvGq^G-4Yx>?3n?#yVkS#W4E6CK%!o#wdb;+C!G`zSBV{-qPm?l z7pv|M>0WY@@)eMh@U=UJc=h@c-vNQUf)i#a6o$f#=tKD*&Nrs&WC;xT^_BYNmiZUV zJ(-=Wy%o-@3o>X~H*z1zDiv(qT_fy7&9(Zfga!KPuCQG%tTfFrXq;!NIT$@=_!2qG z%{n^ae5FXTe_Lfwz}-l8z-Ripnf+(zm@NkD@Zf}hOn32m_=eo$@PX+6w#0z>UFx@D zbEJC7yuAIFEQ1PACK^*w$|qX($6PG8TVu(!bhX|J=PEz%-!ub;{OM-j-*r!46_P!Y z?6ObbQda198O{D`aoLjR)dtFs?rrdp&%t$GajCue+x`C6i=^cxGBB;>!A!VIJ=Mz! zg@sohq(tPD>{ttRkioCn5)8s27{I zvAiA3bXpE~U^+~MH1|qlQ$*ZGoz8I7T(hkee?LZt(QJND@pbcWfwx(b_AK z3hd#qzGVf@Y}ub(g3#waYUU=!Xop`U^~kk=Un-6BxuG4p6JmZb#%hOP4^T?O2b=&% z85r8V7#K}t2H>23mI)majp5E*QJMHQr*rM%FJfkOYtXwYsL*rg! zV!MOK6pRX|UPlpICYfdKSq0{VnV3M;x8xw0v3FYpI_I zTg?h78|n|j?!Ert{dv&kaua7pH4#JL()R^iKIr1|@bQa<{6HX~-e&VrQJ^@QJ)go) zF&-Toj~(?8jHOj#eTL?ZYagHesq=1S$XMl>WJKn|^wy(s%n0nM1nu&5sIxkLdNHjd zUS+3kq^tY|Ex6_^r+nMWso_Id{JLvy@#Xz%q0_~#=2XOV<@HjHypQn7e==joK^zNm zv&9!O3VyAmBdrByx+A!2c-R^?go0 zewq%+o7A-qw5@vxW2T=-p_D7X5d0L+z=mXjgz&0~{UKQ=P6f}Mj+BMsr}e7X+Xh#n zwpTah;s4U3))v#VZ5@d%e&}HwMLRcg<&$1_w1;=j0_n*~Q}z_KQO9C*H)q;zm7T+B z(PHpdBmhlP)Y#$gox|%rCFL8UqOR6zPDiT6!oHqvM|OA}nyX$Um*SvjBC-aeBY%N`)!L#lwS`-OXr+}wjxo(H8wttOn-ojtcR-uk3)G*M$b=&=SJ!KuZk->B5}7zxJ!#wUMF|5fv}`gKH+))L47eoM?m1 z)6Bh4wtM{g;`7fnw~XxDD3Q`c74vEz=nY(|J7c5recu{6@~l61d~1Wzk+g*+h*|Cy z*Gy+`L7&!K(*)w~ao48FZB~T*yjR6(+aPxTTjQGA%-(4^m&Tm4de&)K@2;ile>Jx`Qmd zHKp_hoib%Sszc!?W$9Wb6lSk|4&7Gb6dc1nC?4C%hEb%kp8M*B1}Sb*q`dHPzg!fM zNmq_PtNUhiyYa0)eOCO5`8v5qPd(3Q{&rwmq2qen}(7DQsHW|eJ93VpK7 z9di~=89_H`i?(%}o+~?o7u!{$#Se$$tgSRwG_fEh0?>l@E1xG>*j169{i+4PS&5*^ zZfe^ol&Ph+BWFxO`Nwdcr0g(vEpEic6OeB6QZG2sBP5~pXd|lnp47frpY?Lb!d9YV zVLQKSc<-&~iGfG#6&T#*Oazn84v7TC+l-YrV#mioW?F-XcJzPW`%AilHDaDuipdMX zSjc}Gd>;FCyzZ71m^MhnPL|fMmq{5NEwAU$180yE`bW?-$-jexlXefbfMEU+$4z(& zCL6FJlieM6{qyflVOjtE*8)cJFC|$CP8^{{JuBW?$$Umk7nlk><#ePOLq)JT;Uz%^ z&q-w;-a>Q(ohdzIIUA6S(&I>Ujt7(AVGo zM19A`Z_)9&jp)?a^|$ALm-{`*R*EL?Ci)uwQFaT|j;R-FG~A!9RCg;eIFVUtU&ri1 zsV3B9KYKCow-GjRO{ke5Mr936!!+bxV|_BACorDf7u#|($aU8>N3BeD^|9O%R?ykv zF=P~!1y%Vmb&>CY=({ovl~AR+zrjyJQ%*$nKQsNN#pV2YGyTLk4-BS0zuf?r9EZ_$ zAo}>-!r!0*kn`QqN61U#+q9tNpmtt&o8Py;>#9)RG!^hY(?I zr+qzLej&q|LXCWjdtJIXo|=Y42ZG>K#81)oLVf0*7J^OqaI%aFhk|zBStf36g|nB| zgTs;A&X9)aFzrbV+A&c1S<5bX%a=9$4@2OcKXd)cu>zBT5mCuuK3Yij(*P@+=jlwIOT$2PRV+Y(M&aD%Sjv?4xSZVXpAK3w zX0I8vqY_2BUw1kRCiyS&TduB5-3{iGu=*3be#7}sqDu5R8l%C<`56I94rfFoCiqDS z!;jae-9m7T*TGacd)hpWpgzF`S{Wq9?*7Um{O<8GgYwkcLl1wqtNo|rvXW}Z@ow4$S z&-{yEeuLm8PmgcR8#N}3w7gQBhw@Nfz$xw*>UHUgvGN<8El|0DSdCfm4FlM%dP0e_ zN%57au)PjtN92A67z&4EZe`wxYK{dqJ8%%NEY3j7EA=tTQP5#LlK4k2l z-tRUWwT(mmRp^_$*8QK$x@j}W#+(cdFIdg(H2NRfV$caN4WA@ye~j3eXGz(0TSGFi z*Au9FubZp4u#9(}Anxc|onCmnaY;-iSp|I4+aZq~7pNa-qakB+!}`q=B`$OAwH)Vb zs^{cepauDY&sE~_Cb5M(s?pObvW52}E;?|nQlM~dc0F7&q`!0sLkY1Rzk$UynvlT{ zW2@R}K8(jh>$rM%^5E3LBll8ZxRMPMY7o%FITZs02c>&nSg!(f5 zTSNDh+vl?XM9!oox7kfVPKk6~8DAk|pnK0WiJ@w

UfihaCQirj4A9t(VyJ?^5ek z4ZHN@8L}%#vYujuh8w7Z)!H?E-H0tAx&@l?KU*v}MoVK${0ZgI8*l#Z8k)t%nvlPj zFt#*tcL!mw`xw8(c}ovx!BhD*^SOPx)={xPmOn=gTEk!E0qv`@>Q?GIW4;x7^6iJ9 zXl~ZcGxLV8b2A>jAx*6x)9&yUGy=*1CvIOL4}bJ$`z)8N5gdQ4tmukXo4#u7CeSl} zj4HEF3xP^Fvl6tsgUp{=y!^R?0*v+Ui4%;7IV=qirAV1BvCv?u>vg56G>QLy7J!t< zf~~%qD))NlDrp#LhJ*-KI<|$36UPtz@jskbdL2hNT+v1DAeSB)sA?<$$t0di$fbuB zvCwG>d62O?5o5&UX%>^9js{%0fO%j7)!h@)a*YcuZ9ztGY5$lVA-59zkKTWQB{#Xk zS9tcZsl%qBo0!+|Y|2`Pr(AQg8T1VLfL*qi4o%6s~K)I79= z>KyYmPY>71F_5+>EbaX$-QFQsxP$%6eiF_&I^61wwK$?d{h&EK?5-Q*1AIqoiFCitB0nI#V#VzjtzF}mvg+eD~`jM8A@XrKms_g z6Sdyq3#$a?F3ff^LtDpCJ#_rjiP;@bdsia%_(_Qy&f8I1EA_Gbv1!Z;0!d?8s<_pk zVc42rU?Ee0h1{yWD|d!lecQhB^j{!oV;H_XJywME@#}A`bR_r5s)>}l4~C4NbW|#< z9Z=DGOy&0KaNp-7pr|3~A#<@`m={preV`>QAK0OU5Ws(P|1Qp$(_@4Irx(MI^z1|H zs;a#Ac;+7yGW&~+%oDD5L_LqB7lB`Z-RsAVV2=GnmaOTGyRbo8e+W19@%oZf2hmYM zz$k%v!T97HQog~2x(^*y$2W4`%6-bznU!P=*06sxB=5XZfSmWjxAuH@*e*cQVU2ds zBJUUoHa{?zn^7U7JFY<(Un%rvkRjwvXY=r|5hwgZTnJ0RP8+Rd@V0ym zi5RI{`fT6E1lRltHwoBWN0h#fQ>`34{NqRJTr2z97ll8iS=w?Wa;;kgb~JM%WNv#d zW$AFCO!jb$eSW-gO?78wBb*-bvv`f=99cJsZ3{IxgFNUwX$SECb-ceRsg@ObD!G0% z<}7x|HW-Te_R$1V+Bd6uIt^rMi!Lxj;$wgC7O z?`lZ;4#0Tr|HhjrOkl=?Odp5S>&DbX<4d^>P{(ZH!AWCb+M9TFxYY0^(a4>^wZ610 zIaTax#8S)|T3`6J=t-o1V<&sg@qik#%r|mHMTGumBQ8ML zRN-!TH_=g-b`)Cw)P_!x8R%fI&FA*W6aiId>r3G5#;SKE_mO3fYc$~uBMx!@>O0+# zRvFK1HPp2x>)?0VjE)O)t>TvNZxFQ%>!gYjfKf{*XYL|SPbcf8?(Rgyl|xWumR(`h z8_&$Z#>L|Tf4QMMr-uo0q*cVh5+L``>e@bbQ=_+&`Av7~jrjEM8>Ht+J-0Rxfm+TcR6w#L@ zzig~9y%7Jc!N4XBFLM4;1jvC;tH+ z*78+%*U8zL5xIg-IJ9uF5fD=!0<3>dq-ENa9gQpa4?tiY`Zic$!W-nKbIy{A?Os5U z<*9rBtSA4qmRT&hM}aJ?qaozDWIvZ?w%Aty@5sl;2euOeK8t{IH^6fU5QG9>N&tE@ zKEC_>`=kNBM8*Hu8TqUM*O`y+>yiIAui_4f)@S{qKeO-+B3;x%i*8{NH8q hzq|7P+g0Gn?I{RpTIF)D04@ig@m-5M<+oj;{txg)_F@14 diff --git a/.well-known/ai-plugin.json b/.well-known/ai-plugin.json index bc08de0d4..44e8435f2 100644 --- a/.well-known/ai-plugin.json +++ b/.well-known/ai-plugin.json @@ -2,17 +2,17 @@ "schema_version": "v1", "name_for_model": "text processing tools", "name_for_human": "MetaGPT Text Plugin", - "description_for_model": "Plugins for text processing, including text-to-speech, text-to-image, text-to-vector, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", - "description_for_human": "Plugins for text processing, including text-to-speech, text-to-image, text-to-vector, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", + "description_for_model": "Plugins for text processing, including text-to-speech, text-to-image, text-to-embedding, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", + "description_for_human": "Plugins for text processing, including text-to-speech, text-to-image, text-to-embedding, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", "auth": { - "type": "none", + "type": "none" }, "api": { "type": "openapi", - "url": "https://localhost:8080/.well-known/openapi.yaml", + "url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/.well-known/metagpt_oas3_api.yaml", "has_user_authentication": false }, - "logo_url": "https://localhost:8080/.well-known/MetaGPT-logo.png", - "contact_email": "hello@contact.com", - "legal_info_url": "http://localhost:8080/legal-info" + "logo_url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/docs/resources/MetaGPT-logo.png", + "contact_email": "mashenquan@fuzhi.cn", + "legal_info_url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/docs/README_CN.md" } \ No newline at end of file diff --git a/metagpt/tools/openai_text_2_embedding.py b/metagpt/tools/openai_text_2_embedding.py new file mode 100644 index 000000000..822c5af00 --- /dev/null +++ b/metagpt/tools/openai_text_2_embedding.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : openai_text_2_vector.py +@Desc : OpenAI Text-to-Vector OAS3 api, which provides text-to-vector functionality. +""" +import os + +class OpenAIText2Vector: + def __init__(self, openai_api_key): + """ + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + """ + self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') + + def text_2_vector(self, text, size_type="1024x1024"): + """Text to image + + :param text: The text used for image conversion. + :param size_type: One of ['256x256', '512x512', '1024x1024'] + :return: The image data is returned in Base64 encoding. + """ + + class ImageUrl(BaseModel): + url: str + + class ImageResult(BaseModel): + data: List[ImageUrl] + created: int + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.openai_api_key}" + } + data = {"prompt": text, "n": 1, "size": size_type} + try: + response = requests.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) + response.raise_for_status() # Raise an exception for 4xx or 5xx responses + result = ImageResult(**response.json()) + except requests.exceptions.RequestException as e: + logger.error(f"An error occurred:{e}") + return "" + if len(result.data) > 0: + return OpenAIText2Image.get_image_data(result.data[0].url) + return "" \ No newline at end of file From 8aa30c35d2da9345a4d04c073d38abccd08d5f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 12:13:52 +0800 Subject: [PATCH 043/398] feat: +hello.py oas3 --- .well-known/metagpt_oas3_api.yaml | 87 +++++++++++++++++++++- metagpt/tools/azure_tts.py | 2 +- metagpt/tools/metagpt_oas3_api_svc.py | 1 + metagpt/tools/openai_text_2_embedding.py | 91 ++++++++++++++++++------ 4 files changed, 156 insertions(+), 25 deletions(-) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index e6cf25d86..4999bf38a 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -101,4 +101,89 @@ paths: '400': description: "Bad Request" '500': - description: "Internal Server Error" \ No newline at end of file + description: "Internal Server Error" + /txt2embedding/openai: + post: + summary: Text to embedding + operationId: openai_text_2_embedding.oas3_openai_text_2_embedding + description: Retrieve an embedding for the provided text using the OpenAI API. + requestBody: + content: + application/json: + schema: + type: object + properties: + input: + type: string + description: The text used for embedding. + model: + type: string + description: "ID of the model to use. For more details, checkout: [models](https://api.openai.com/v1/models)" + enum: + - text-embedding-ada-002 + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: "#/components/schemas/ResultEmbedding" + "4XX": + description: Client error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "5XX": + description: Server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" +components: + schemas: + Embedding: + type: object + description: Represents an embedding vector returned by the embedding endpoint. + properties: + object: + type: string + example: embedding + embedding: + type: array + items: + type: number + example: [0.0023064255, -0.009327292, ...] + index: + type: integer + example: 0 + Usage: + type: object + properties: + prompt_tokens: + type: integer + example: 8 + total_tokens: + type: integer + example: 8 + ResultEmbedding: + type: object + properties: + object: + type: string + example: result_embedding + data: + type: array + items: + $ref: "#/components/schemas/Embedding" + model: + type: string + example: text-embedding-ada-002 + usage: + $ref: "#/components/schemas/Usage" + Error: + type: object + properties: + error: + type: string + example: An error occurred \ No newline at end of file diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 6b1a041f3..2ec1539ef 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -108,7 +108,7 @@ def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key if __name__ == "__main__": - initalize_enviroment() + initialize_environment() v = oas3_azsure_tts("测试，test") print(v) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index ef3347b6c..aa5f50cb2 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -17,4 +17,5 @@ if __name__ == "__main__": app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') app.add_api("metagpt_oas3_api.yaml") + app.add_api("openapi.yaml") app.run(port=8080) diff --git a/metagpt/tools/openai_text_2_embedding.py b/metagpt/tools/openai_text_2_embedding.py index 822c5af00..eb90a1ea9 100644 --- a/metagpt/tools/openai_text_2_embedding.py +++ b/metagpt/tools/openai_text_2_embedding.py @@ -1,47 +1,92 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/8/17 +@Time : 2023/8/18 @Author : mashenquan -@File : openai_text_2_vector.py -@Desc : OpenAI Text-to-Vector OAS3 api, which provides text-to-vector functionality. +@File : openai_text_2_embedding.py +@Desc : OpenAI Text-to-Embedding OAS3 api, which provides text-to-embedding functionality. + For more details, checkout: `https://platform.openai.com/docs/api-reference/embeddings/object` """ import os +from pathlib import Path +from typing import List -class OpenAIText2Vector: +import requests +from pydantic import BaseModel +import sys + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.utils.common import initialize_environment +from metagpt.logs import logger + + +class Embedding(BaseModel): + """Represents an embedding vector returned by embedding endpoint.""" + object: str # The object type, which is always "embedding". + embedding: List[ + float] # The embedding vector, which is a list of floats. The length of vector depends on the model as listed in the embedding guide. + index: int # The index of the embedding in the list of embeddings. + + +class Usage(BaseModel): + prompt_tokens: int + total_tokens: int + + +class ResultEmbedding(BaseModel): + object: str + data: List[Embedding] + model: str + usage: Usage + + +class OpenAIText2Embedding: def __init__(self, openai_api_key): """ :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` """ self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') - def text_2_vector(self, text, size_type="1024x1024"): - """Text to image + def text_2_embedding(self, text, model="text-embedding-ada-002"): + """Text to embedding - :param text: The text used for image conversion. - :param size_type: One of ['256x256', '512x512', '1024x1024'] - :return: The image data is returned in Base64 encoding. + :param text: The text used for embedding. + :param model: One of ['text-embedding-ada-002'], ID of the model to use. For more details, checkout: `https://api.openai.com/v1/models`. + :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. """ - class ImageUrl(BaseModel): - url: str - - class ImageResult(BaseModel): - data: List[ImageUrl] - created: int - headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.openai_api_key}" } - data = {"prompt": text, "n": 1, "size": size_type} + data = {"input": text, "model": model} try: - response = requests.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) + response = requests.post("https://api.openai.com/v1/embeddings", headers=headers, json=data) response.raise_for_status() # Raise an exception for 4xx or 5xx responses - result = ImageResult(**response.json()) + return response.json() except requests.exceptions.RequestException as e: logger.error(f"An error occurred:{e}") - return "" - if len(result.data) > 0: - return OpenAIText2Image.get_image_data(result.data[0].url) - return "" \ No newline at end of file + return {} + + +# Export +def oas3_openai_text_2_embedding(text, model="text-embedding-ada-002", openai_api_key=""): + """Text to embedding + + :param text: The text used for embedding. + :param model: One of ['text-embedding-ada-002'], ID of the model to use. For more details, checkout: `https://api.openai.com/v1/models`. + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. + """ + if not text: + return "" + if not openai_api_key: + openai_api_key = os.environ.get("OPENAI_API_KEY") + return OpenAIText2Embedding(openai_api_key).text_2_embedding(text, model=model) + + +if __name__ == "__main__": + initialize_environment() + + v = oas3_openai_text_2_embedding("Panda emoji") + print(v) From 34d46829ec62bf5b41f23cca5d566f2adcaa2f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 13:43:47 +0800 Subject: [PATCH 044/398] feat: + server port --- .well-known/metagpt_oas3_api.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index 4999bf38a..7a0058b50 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -5,6 +5,11 @@ info: version: "1.0" servers: - url: "/oas3" + variables: + port: + enum: + - '8080' + default: '8080' paths: /tts/azsure: From 2b19a7118d54420f689f98b69c85fe98c8e3417f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 14:04:23 +0800 Subject: [PATCH 045/398] feat: +servers http port --- .well-known/metagpt_oas3_api.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index 7a0058b50..7ae10579c 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -7,9 +7,8 @@ servers: - url: "/oas3" variables: port: - enum: - - '8080' default: '8080' + description: HTTP service port paths: /tts/azsure: From d97231933fbb02a19f8954efe49679fd8eefed76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 14:45:14 +0800 Subject: [PATCH 046/398] feat: +async oas3 http service demo --- metagpt/tools/metagpt_oas3_api_svc.py | 31 +++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index aa5f50cb2..34ae6a563 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -6,16 +6,43 @@ @File : metagpt_oas3_api_svc.py @Desc : MetaGPT OpenAPI Specification 3.0 REST API service """ +import asyncio from pathlib import Path import sys +from time import sleep + import connexion +import threading + sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.utils.common import initialize_environment -if __name__ == "__main__": + +def oas_http_svc(): + """Start the OAS 3.0 OpenAPI HTTP service""" initialize_environment() - app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') + app = connexion.FlaskApp(__name__, specification_dir='../../.well-known/') app.add_api("metagpt_oas3_api.yaml") app.add_api("openapi.yaml") app.run(port=8080) + + +async def async_main(): + """Start the OAS 3.0 OpenAPI HTTP service in the background.""" + loop = asyncio.get_event_loop() + loop.run_in_executor(None, oas_http_svc) + + # TODO: replace following codes: + while True: + await asyncio.sleep(1) + print("sleep") + + +def main(): + oas_http_svc() + + +if __name__ == "__main__": + # asyncio.run(async_main()) + main() From 3c93573f93cfbbbf79a523dec2e3cff5d2e719c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 14:46:33 +0800 Subject: [PATCH 047/398] feat: +async oas3 http service demo --- metagpt/tools/metagpt_oas3_api_svc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index 34ae6a563..277d41dfb 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -9,10 +9,8 @@ import asyncio from pathlib import Path import sys -from time import sleep import connexion -import threading sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.utils.common import initialize_environment From 866c5bcb15b1e51af743272892e38e9e3795d5b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 15:10:23 +0800 Subject: [PATCH 048/398] fixbug: merge bug --- metagpt/provider/openai_api.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 0f7100db8..88343373f 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -25,26 +25,6 @@ from metagpt.utils.token_counter import ( ) -<<<<<<< HEAD -def retry(max_retries): - def decorator(f): - @wraps(f) - async def wrapper(*args, **kwargs): - for i in range(max_retries): - try: - return await f(*args, **kwargs) - except Exception as e: - error_str = traceback.format_exc() - logger.warning(f"Exception occurred: {str(e)}, stack:{error_str}. Retrying...") - if i == max_retries - 1: - raise - await asyncio.sleep(2 ** i) - return wrapper - return decorator - - -======= ->>>>>>> main class RateLimiter: """Rate control class, each call goes through wait_if_needed, sleep if rate control is needed""" From 321f4a5a17bae23be5360f2ba7d2fadb76e66f9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 15:11:35 +0800 Subject: [PATCH 049/398] fixbug: merge bug --- metagpt/provider/openai_api.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 0f7100db8..88343373f 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -25,26 +25,6 @@ from metagpt.utils.token_counter import ( ) -<<<<<<< HEAD -def retry(max_retries): - def decorator(f): - @wraps(f) - async def wrapper(*args, **kwargs): - for i in range(max_retries): - try: - return await f(*args, **kwargs) - except Exception as e: - error_str = traceback.format_exc() - logger.warning(f"Exception occurred: {str(e)}, stack:{error_str}. Retrying...") - if i == max_retries - 1: - raise - await asyncio.sleep(2 ** i) - return wrapper - return decorator - - -======= ->>>>>>> main class RateLimiter: """Rate control class, each call goes through wait_if_needed, sleep if rate control is needed""" From 341037601a89af510e9efdb598168562c4a278d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 19:01:07 +0800 Subject: [PATCH 050/398] feat: + unit test --- metagpt/learn/text_to_embedding.py | 23 +++++++++++ metagpt/learn/text_to_image.py | 23 +++++++++++ metagpt/learn/text_to_speech.py | 29 +++++++++++++ metagpt/tools/azure_tts.py | 2 +- tests/metagpt/learn/__init__.py | 0 tests/metagpt/learn/test_text_to_embedding.py | 40 ++++++++++++++++++ tests/metagpt/learn/test_text_to_image.py | 41 +++++++++++++++++++ tests/metagpt/learn/test_text_to_speech.py | 40 ++++++++++++++++++ 8 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 metagpt/learn/text_to_embedding.py create mode 100644 metagpt/learn/text_to_image.py create mode 100644 metagpt/learn/text_to_speech.py create mode 100644 tests/metagpt/learn/__init__.py create mode 100644 tests/metagpt/learn/test_text_to_embedding.py create mode 100644 tests/metagpt/learn/test_text_to_image.py create mode 100644 tests/metagpt/learn/test_text_to_speech.py diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py new file mode 100644 index 000000000..b1395a61a --- /dev/null +++ b/metagpt/learn/text_to_embedding.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : text_to_embedding.py +@Desc : Text-to-Embedding skill, which provides text-to-embedding functionality. +""" + +from metagpt.tools.openai_text_2_embedding import oas3_openai_text_2_embedding +from metagpt.utils.common import initialize_environment + + +def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): + """Text to embedding + + :param text: The text used for embedding. + :param model: One of ['text-embedding-ada-002'], ID of the model to use. For more details, checkout: `https://api.openai.com/v1/models`. + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. + """ + initialize_environment() + return oas3_openai_text_2_embedding(text, model=model, openai_api_key=openai_api_key) \ No newline at end of file diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py new file mode 100644 index 000000000..87668a13f --- /dev/null +++ b/metagpt/learn/text_to_image.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : text_to_image.py +@Desc : Text-to-Image skill, which provides text-to-image functionality. +""" + +from metagpt.tools.openai_text_2_image import oas3_openai_text_2_image +from metagpt.utils.common import initialize_environment + + +def text_to_image(text, size_type: str = "1024x1024", openai_api_key=""): + """Text to image + + :param text: The text used for image conversion. + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + :param size_type: One of ['256x256', '512x512', '1024x1024'] + :return: The image data is returned in Base64 encoding. + """ + initialize_environment() + return oas3_openai_text_2_image(text, size_type, openai_api_key) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py new file mode 100644 index 000000000..909a9dca1 --- /dev/null +++ b/metagpt/learn/text_to_speech.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : text_to_speech.py +@Desc : Text-to-Speech skill, which provides text-to-speech functionality +""" + +from metagpt.tools.azure_tts import oas3_azsure_tts +from metagpt.utils.common import initialize_environment + + +def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", subscription_key="", region=""): + """Text to speech + For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + + :param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param voice: For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`, `https://speech.microsoft.com/portal/voicegallery` + :param style: Speaking style to express different emotions like cheerfulness, empathy, and calm. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param role: With roles, the same voice can act as a different age and gender. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param text: The text used for voice conversion. + :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` + :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. + :return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string. + + """ + initialize_environment() + return oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 2ec1539ef..21e8f1b6c 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -62,7 +62,7 @@ class AzureTTS: # Export def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): - """oas3/tts/azsure + """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` :param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` diff --git a/tests/metagpt/learn/__init__.py b/tests/metagpt/learn/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/metagpt/learn/test_text_to_embedding.py b/tests/metagpt/learn/test_text_to_embedding.py new file mode 100644 index 000000000..c85e5dde8 --- /dev/null +++ b/tests/metagpt/learn/test_text_to_embedding.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : test_text_to_embedding.py +@Desc : Unit tests. +""" + +import asyncio +import base64 + +from pydantic import BaseModel + +from metagpt.learn.text_to_embedding import text_to_embedding + + +async def mock_text_to_embedding(): + class Input(BaseModel): + input: str + + inputs = [ + {"input": "Panda emoji"} + ] + + for i in inputs: + seed = Input(**i) + data = text_to_embedding(seed.input) + v = ResultEmbedding(**data) + assert len(v.data) > 0 + + +def test_suite(): + loop = asyncio.get_event_loop() + task = loop.create_task(mock_text_to_embedding()) + loop.run_until_complete(task) + + +if __name__ == '__main__': + test_suite() diff --git a/tests/metagpt/learn/test_text_to_image.py b/tests/metagpt/learn/test_text_to_image.py new file mode 100644 index 000000000..bfcb1db25 --- /dev/null +++ b/tests/metagpt/learn/test_text_to_image.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : test_text_to_image.py +@Desc : Unit tests. +""" +import asyncio +import base64 + +from pydantic import BaseModel + +from metagpt.learn.text_to_image import text_to_image + + +async def mock_text_to_image(): + class Input(BaseModel): + input: str + size_type: str + + inputs = [ + {"input": "Panda emoji", "size_type": "256x256"} + ] + + for i in inputs: + seed = Input(**i) + base64_data = text_to_image(seed.input) + assert base64_data != "" + print(f"{seed.input} -> {base64_data}") + assert base64.b64decode(base64_data, validate=True) + + +def test_suite(): + loop = asyncio.get_event_loop() + task = loop.create_task(mock_text_to_image()) + loop.run_until_complete(task) + + +if __name__ == '__main__': + test_suite() diff --git a/tests/metagpt/learn/test_text_to_speech.py b/tests/metagpt/learn/test_text_to_speech.py new file mode 100644 index 000000000..dbb599e38 --- /dev/null +++ b/tests/metagpt/learn/test_text_to_speech.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : test_text_to_speech.py +@Desc : Unit tests. +""" +import asyncio +import base64 + +from pydantic import BaseModel + +from metagpt.learn.text_to_speech import text_to_speech + + +async def mock_text_to_speech(): + class Input(BaseModel): + input: str + + inputs = [ + {"input": "Panda emoji"} + ] + + for i in inputs: + seed = Input(**i) + base64_data = text_to_speech(seed.input) + assert base64_data != "" + print(f"{seed.input} -> {base64_data}") + assert base64.b64decode(base64_data, validate=True) + + +def test_suite(): + loop = asyncio.get_event_loop() + task = loop.create_task(mock_text_to_speech()) + loop.run_until_complete(task) + + +if __name__ == '__main__': + test_suite() \ No newline at end of file From 4f8187b6719689783352653fb9e0b5ef9eb55ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 19:29:51 +0800 Subject: [PATCH 051/398] feat: + METAGPT_TEXT_TO_IMAGE_MODEL --- config/config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/config.yaml b/config/config.yaml index 303f4824b..6e9a61931 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -70,3 +70,6 @@ SD_T2I_API: "/sdapi/v1/txt2img" ### for Research MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k + +### Meta Models +#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL \ No newline at end of file From 99c143e8f301f89738eccdb4988552fc0a4a8cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 20:09:06 +0800 Subject: [PATCH 052/398] feat: +metagpt text to image --- .gitignore | 1 + .well-known/metagpt_oas3_api.yaml | 47 +++++++- metagpt/tools/metagpt_text_to_image.py | 112 ++++++++++++++++++ ...bedding.py => openai_text_to_embedding.py} | 6 +- ...ext_2_image.py => openai_text_to_image.py} | 6 +- 5 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 metagpt/tools/metagpt_text_to_image.py rename metagpt/tools/{openai_text_2_embedding.py => openai_text_to_embedding.py} (94%) rename metagpt/tools/{openai_text_2_image.py => openai_text_to_image.py} (94%) diff --git a/.gitignore b/.gitignore index c4c79c733..2cba27484 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,4 @@ workspace/* *.mmd tmp output.wav +tmp.png diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index 7ae10579c..a226181a5 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -71,7 +71,7 @@ paths: /txt2img/openai: post: summary: "Convert Text to Base64-encoded Image Data Stream" - operationId: openai_text_2_image.oas3_openai_text_2_image + operationId: openai_text_to_image.oas3_openai_text_to_image requestBody: required: true content: @@ -109,7 +109,7 @@ paths: /txt2embedding/openai: post: summary: Text to embedding - operationId: openai_text_2_embedding.oas3_openai_text_2_embedding + operationId: openai_text_to_embedding.oas3_openai_text_to_embedding description: Retrieve an embedding for the provided text using the OpenAI API. requestBody: content: @@ -144,6 +144,49 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" + + /txt2image/metagpt: + post: + summary: "Text to Image" + description: "Generate an image from the provided text using the MetaGPT Text-to-Image API." + operationId: metagpt_text_to_image.oas3_metagpt_text_to_image + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - text + properties: + text: + type: string + description: "The text used for image conversion." + size_type: + type: string + enum: ["512x512", "512x768"] + default: "512x512" + description: "Size of the generated image." + model_url: + type: string + description: "Model reset API URL for text-to-image." + default: "" + responses: + '200': + description: "Base64-encoded image data." + content: + application/json: + schema: + type: object + properties: + image_data: + type: string + format: base64 + '400': + description: "Bad Request" + '500': + description: "Internal Server Error" + components: schemas: Embedding: diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py new file mode 100644 index 000000000..393215df0 --- /dev/null +++ b/metagpt/tools/metagpt_text_to_image.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : metagpt_text_to_image.py +@Desc : MetaGPT Text-to-Image OAS3 api, which provides text-to-image functionality. +""" +import base64 +import os +import sys +from pathlib import Path +from typing import List, Dict + +import requests +from pydantic import BaseModel + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.utils.common import initialize_environment +from metagpt.logs import logger + + +class MetaGPTText2Image: + def __init__(self, model_url): + """ + :param model_url: Model reset api url + """ + self.model_url = model_url if model_url else os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL') + + def text_2_image(self, text, size_type="512x512"): + """Text to image + + :param text: The text used for image conversion. + :param size_type: One of ['512x512', '512x768'] + :return: The image data is returned in Base64 encoding. + """ + + headers = { + "Content-Type": "application/json" + } + dims = size_type.split("x") + data = { + "prompt": text, + "negative_prompt": "(easynegative:0.8),black, dark,Low resolution", + "override_settings": {"sd_model_checkpoint": "galaxytimemachinesGTM_photoV20"}, + "seed": -1, + "batch_size": 1, + "n_iter": 1, + "steps": 20, + "cfg_scale": 11, + "width": int(dims[0]), + "height": int(dims[1]), # 768, + "restore_faces": False, + "tiling": False, + "do_not_save_samples": False, + "do_not_save_grid": False, + "enable_hr": False, + "hr_scale": 2, + "hr_upscaler": "Latent", + "hr_second_pass_steps": 0, + "hr_resize_x": 0, + "hr_resize_y": 0, + "hr_upscale_to_x": 0, + "hr_upscale_to_y": 0, + "truncate_x": 0, + "truncate_y": 0, + "applied_old_hires_behavior_to": None, + "eta": None, + "sampler_index": "DPM++ SDE Karras", + "alwayson_scripts": {}, + } + + class ImageResult(BaseModel): + images: List + parameters: Dict + + try: + response = requests.post(self.model_url, headers=headers, json=data) + response.raise_for_status() # Raise an exception for 4xx or 5xx responses + result = ImageResult(**response.json()) + if len(result.images) == 0: + return "" + return result.images[0] + except requests.exceptions.RequestException as e: + logger.error(f"An error occurred:{e}") + return "" + + +# Export +def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): + """Text to image + + :param text: The text used for image conversion. + :param model_url: Model reset api + :param size_type: One of ['512x512', '512x768'] + :return: The image data is returned in Base64 encoding. + """ + if not text: + return "" + if not model_url: + model_url = os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL') + return MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type) + + +if __name__ == "__main__": + initialize_environment() + + v = oas3_metagpt_text_2_image("Panda emoji") + data = base64.b64decode(v) + with open("tmp.png", mode="wb") as writer: + writer.write(data) + print(v) diff --git a/metagpt/tools/openai_text_2_embedding.py b/metagpt/tools/openai_text_to_embedding.py similarity index 94% rename from metagpt/tools/openai_text_2_embedding.py rename to metagpt/tools/openai_text_to_embedding.py index eb90a1ea9..9eddd5bc1 100644 --- a/metagpt/tools/openai_text_2_embedding.py +++ b/metagpt/tools/openai_text_to_embedding.py @@ -3,7 +3,7 @@ """ @Time : 2023/8/18 @Author : mashenquan -@File : openai_text_2_embedding.py +@File : openai_text_to_embedding.py @Desc : OpenAI Text-to-Embedding OAS3 api, which provides text-to-embedding functionality. For more details, checkout: `https://platform.openai.com/docs/api-reference/embeddings/object` """ @@ -70,7 +70,7 @@ class OpenAIText2Embedding: # Export -def oas3_openai_text_2_embedding(text, model="text-embedding-ada-002", openai_api_key=""): +def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): """Text to embedding :param text: The text used for embedding. @@ -88,5 +88,5 @@ def oas3_openai_text_2_embedding(text, model="text-embedding-ada-002", openai_ap if __name__ == "__main__": initialize_environment() - v = oas3_openai_text_2_embedding("Panda emoji") + v = oas3_openai_text_to_embedding("Panda emoji") print(v) diff --git a/metagpt/tools/openai_text_2_image.py b/metagpt/tools/openai_text_to_image.py similarity index 94% rename from metagpt/tools/openai_text_2_image.py rename to metagpt/tools/openai_text_to_image.py index 50c007626..6ec96d166 100644 --- a/metagpt/tools/openai_text_2_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -3,7 +3,7 @@ """ @Time : 2023/8/17 @Author : mashenquan -@File : openai_text_2_image.py +@File : openai_text_to_image.py @Desc : OpenAI Text-to-Image OAS3 api, which provides text-to-image functionality. """ import base64 @@ -78,7 +78,7 @@ class OpenAIText2Image: # Export -def oas3_openai_text_2_image(text, size_type: str = "1024x1024", openai_api_key=""): +def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_api_key=""): """Text to image :param text: The text used for image conversion. @@ -96,5 +96,5 @@ def oas3_openai_text_2_image(text, size_type: str = "1024x1024", openai_api_key= if __name__ == "__main__": initialize_environment() - v = oas3_openai_text_2_image("Panda emoji") + v = oas3_openai_text_to_image("Panda emoji") print(v) From 3715a69e3f3df119477fd9f20e1d526afd94c115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 20:22:52 +0800 Subject: [PATCH 053/398] feat: update text_to_image skill --- metagpt/learn/text_to_embedding.py | 7 +++++-- metagpt/learn/text_to_image.py | 15 +++++++++++---- metagpt/learn/text_to_speech.py | 7 ++++++- tests/metagpt/learn/test_text_to_image.py | 2 +- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py index b1395a61a..281815ca6 100644 --- a/metagpt/learn/text_to_embedding.py +++ b/metagpt/learn/text_to_embedding.py @@ -6,8 +6,9 @@ @File : text_to_embedding.py @Desc : Text-to-Embedding skill, which provides text-to-embedding functionality. """ +import os -from metagpt.tools.openai_text_2_embedding import oas3_openai_text_2_embedding +from metagpt.tools.openai_text_to_embedding import oas3_openai_text_to_embedding from metagpt.utils.common import initialize_environment @@ -20,4 +21,6 @@ def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. """ initialize_environment() - return oas3_openai_text_2_embedding(text, model=model, openai_api_key=openai_api_key) \ No newline at end of file + if os.environ.get("OPENAI_API_KEY") or openai_api_key: + return oas3_openai_text_to_embedding(text, model=model, openai_api_key=openai_api_key) + raise EnvironmentError diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 87668a13f..0932dfe07 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -6,18 +6,25 @@ @File : text_to_image.py @Desc : Text-to-Image skill, which provides text-to-image functionality. """ +import os -from metagpt.tools.openai_text_2_image import oas3_openai_text_2_image +from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image +from metagpt.tools.openai_text_to_image import oas3_openai_text_to_image from metagpt.utils.common import initialize_environment -def text_to_image(text, size_type: str = "1024x1024", openai_api_key=""): +def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url=""): """Text to image :param text: The text used for image conversion. :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` - :param size_type: One of ['256x256', '512x512', '1024x1024'] + :param size_type: If using OPENAI, the available size options are ['256x256', '512x512', '1024x1024'], while for MetaGPT, the options are ['512x512', '512x768']. + :param model_url: MetaGPT model url :return: The image data is returned in Base64 encoding. """ initialize_environment() - return oas3_openai_text_2_image(text, size_type, openai_api_key) + if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL") or model_url: + return oas3_metagpt_text_to_image(text, size_type, model_url) + if os.environ.get("OPENAI_API_KEY") or openai_api_key: + return oas3_openai_text_to_image(text, size_type, openai_api_key) + raise EnvironmentError diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 909a9dca1..b89b5a9c4 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -6,6 +6,7 @@ @File : text_to_speech.py @Desc : Text-to-Speech skill, which provides text-to-speech functionality """ +import os from metagpt.tools.azure_tts import oas3_azsure_tts from metagpt.utils.common import initialize_environment @@ -26,4 +27,8 @@ def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affect """ initialize_environment() - return oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + if (os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") and os.environ.get("AZURE_TTS_REGION")) or \ + (subscription_key and region): + return oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + + raise EnvironmentError diff --git a/tests/metagpt/learn/test_text_to_image.py b/tests/metagpt/learn/test_text_to_image.py index bfcb1db25..545c8a3ef 100644 --- a/tests/metagpt/learn/test_text_to_image.py +++ b/tests/metagpt/learn/test_text_to_image.py @@ -20,7 +20,7 @@ async def mock_text_to_image(): size_type: str inputs = [ - {"input": "Panda emoji", "size_type": "256x256"} + {"input": "Panda emoji", "size_type": "512x512"} ] for i in inputs: From df5a50f6e677fda08605fcbb44d7048642e76fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 20:23:33 +0800 Subject: [PATCH 054/398] feat: update text_to_image skill --- metagpt/learn/text_to_speech.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index b89b5a9c4..1b81097b8 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -12,7 +12,8 @@ from metagpt.tools.azure_tts import oas3_azsure_tts from metagpt.utils.common import initialize_environment -def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", subscription_key="", region=""): +def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", + subscription_key="", region=""): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -28,7 +29,7 @@ def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affect """ initialize_environment() if (os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") and os.environ.get("AZURE_TTS_REGION")) or \ - (subscription_key and region): + (subscription_key and region): return oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) raise EnvironmentError From f31b60309ad56faa4acb363f38f5b4dbd55a22c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 19 Aug 2023 21:57:09 +0800 Subject: [PATCH 055/398] feat: Config isolation at the object level. --- metagpt/config.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 21f180455..ac969f2f9 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -提供配置，单例 +@Desc: Provide configuration, singleton. +@Modified By: mashenquan, replace `CONFIG` with `os.environ` to support personal config """ import os @@ -28,10 +29,13 @@ class NotConfiguredException(Exception): class Config(metaclass=Singleton): """ - 常规使用方法： + For example: + + ```python config = Config("config.yaml") secret_key = config.get_key("MY_SECRET_KEY") print("Secret key:", secret_key) + ``` """ _instance = None @@ -41,12 +45,13 @@ class Config(metaclass=Singleton): def __init__(self, yaml_file=default_yaml_file): self._configs = {} self._init_with_config_files_and_env(self._configs, yaml_file) + logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -85,20 +90,27 @@ class Config(metaclass=Singleton): self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") + # Update environment variables + for k, v in self._configs.items(): + os.environ[k] = str(v) + for attribute, value in vars(self).items(): + if attribute == "_configs": + continue + os.environ[attribute] = str(value) + def _init_with_config_files_and_env(self, configs: dict, yaml_file): - """从config/key.yaml / config/config.yaml / env三处按优先级递减加载""" + """Load in decreasing priority from `config/key.yaml`, `config/config.yaml`, and environment variables.""" configs.update(os.environ) for _yaml_file in [yaml_file, self.key_yaml_file]: if not _yaml_file.exists(): continue - # 加载本地 YAML 文件 + # Load local YAML file. with open(_yaml_file, "r", encoding="utf-8") as file: yaml_data = yaml.safe_load(file) if not yaml_data: continue - os.environ.update({k: v for k, v in yaml_data.items() if isinstance(v, str)}) configs.update(yaml_data) def _get(self, *args, **kwargs): @@ -111,5 +123,3 @@ class Config(metaclass=Singleton): raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") return value - -CONFIG = Config() From 291af5ad01bef9f6dbaa29305a3b13b29e21763b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 10:19:43 +0800 Subject: [PATCH 056/398] feat: + Config.options --- metagpt/config.py | 25 ++++++++++++++++--------- tests/metagpt/utils/test_config.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index ac969f2f9..6f3f9732a 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -3,6 +3,8 @@ """ @Desc: Provide configuration, singleton. @Modified By: mashenquan, replace `CONFIG` with `os.environ` to support personal config +@Desc: `os.environ` doesn't support personalization, while `Config` does. + Hence, the parameter reading priority is `Config` first, and if not found, then `os.environ`. """ import os @@ -90,14 +92,6 @@ class Config(metaclass=Singleton): self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") - # Update environment variables - for k, v in self._configs.items(): - os.environ[k] = str(v) - for attribute, value in vars(self).items(): - if attribute == "_configs": - continue - os.environ[attribute] = str(value) - def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load in decreasing priority from `config/key.yaml`, `config/config.yaml`, and environment variables.""" configs.update(os.environ) @@ -117,9 +111,22 @@ class Config(metaclass=Singleton): return self._configs.get(*args, **kwargs) def get(self, key, *args, **kwargs): - """从config/key.yaml / config/config.yaml / env三处找值，找不到报错""" + """Retrieve value from `config/key.yaml`, `config/config.yaml`, and environment variables. + Raise an error if not found.""" value = self._get(key, *args, **kwargs) if value is None: raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") return value + @property + def options(self): + """Return key-value configuration parameters.""" + opts = {} + for k, v in self._configs.items(): + opts[k] = v + for attribute, value in vars(self).items(): + if attribute == "_configs": + continue + opts[attribute] = value + return opts + diff --git a/tests/metagpt/utils/test_config.py b/tests/metagpt/utils/test_config.py index 558a4e5a4..475bac22b 100644 --- a/tests/metagpt/utils/test_config.py +++ b/tests/metagpt/utils/test_config.py @@ -4,7 +4,9 @@ @Time : 2023/5/1 11:19 @Author : alexanderwu @File : test_config.py +@Modified By: mashenquan, 2013/8/20, add `test_options` """ +from pathlib import Path import pytest @@ -29,3 +31,14 @@ def test_config_yaml_file_not_exists(): with pytest.raises(Exception) as exc_info: config.get('OPENAI_BASE_URL') assert str(exc_info.value) == "Key 'OPENAI_BASE_URL' not found in environment variables or in the YAML file" + + +def test_options(): + filename = Path(__file__).resolve().parent.parent.parent.parent / "config/config.yaml" + config = Config(filename) + opts = config.options + assert opts + + +if __name__ == '__main__': + test_options() From d764b8e6fa3fbbdcfc6f289b0f4495b6c7289d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 10:26:26 +0800 Subject: [PATCH 057/398] feat: Remove global configuration CONFIG --- metagpt/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 6f3f9732a..6e2cf0a3f 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -29,7 +29,7 @@ class NotConfiguredException(Exception): super().__init__(self.message) -class Config(metaclass=Singleton): +class Config: """ For example: @@ -40,7 +40,6 @@ class Config(metaclass=Singleton): ``` """ - _instance = None key_yaml_file = PROJECT_ROOT / "config/key.yaml" default_yaml_file = PROJECT_ROOT / "config/config.yaml" From f45a8e52842ca2b03f936132b3c51afaeeb2e9a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 17:33:13 +0800 Subject: [PATCH 058/398] feat: Remove global configuration , enable configuration support for business isolation. --- metagpt/actions/action.py | 8 +- metagpt/actions/analyze_dep_libs.py | 5 +- metagpt/actions/debug_error.py | 5 +- metagpt/actions/design_api.py | 11 +-- metagpt/actions/design_api_review.py | 5 +- metagpt/actions/design_filenames.py | 5 +- metagpt/actions/project_management.py | 5 +- metagpt/actions/research.py | 24 ++++-- metagpt/actions/run_code.py | 5 +- metagpt/actions/search_and_summarize.py | 11 ++- metagpt/actions/write_code.py | 5 +- metagpt/actions/write_code_review.py | 5 +- metagpt/actions/write_prd.py | 7 +- metagpt/actions/write_prd_review.py | 5 +- metagpt/actions/write_test.py | 5 +- metagpt/config.py | 4 +- metagpt/document_store/faiss_store.py | 8 +- metagpt/llm.py | 20 ----- metagpt/management/skill_manager.py | 3 +- metagpt/manager.py | 5 +- metagpt/memory/longterm_memory.py | 9 +- metagpt/memory/memory_storage.py | 9 +- metagpt/provider/anthropic_api.py | 15 +++- metagpt/provider/openai_api.py | 82 +++++++++++++------ metagpt/roles/architect.py | 6 +- metagpt/roles/engineer.py | 6 +- metagpt/roles/product_manager.py | 6 +- metagpt/roles/project_manager.py | 6 +- metagpt/roles/qa_engineer.py | 4 +- metagpt/roles/role.py | 30 ++++--- metagpt/software_company.py | 25 +++++- metagpt/tools/search_engine.py | 38 +++++---- metagpt/tools/search_engine_ddg.py | 48 +++++------ metagpt/tools/search_engine_googleapi.py | 13 +-- metagpt/tools/search_engine_serpapi.py | 6 +- metagpt/tools/search_engine_serper.py | 4 +- metagpt/tools/web_browser_engine.py | 26 ++++-- .../tools/web_browser_engine_playwright.py | 24 ++++-- metagpt/tools/web_browser_engine_selenium.py | 19 +++-- metagpt/utils/mermaid.py | 22 +++-- startup.py | 16 ++-- tests/metagpt/actions/test_write_code.py | 14 +++- tests/metagpt/memory/test_longterm_memory.py | 21 +++-- tests/metagpt/test_environment.py | 41 +++++++--- tests/metagpt/test_llm.py | 7 +- tests/metagpt/tools/test_search_engine.py | 9 +- .../metagpt/tools/test_web_browser_engine.py | 8 +- .../test_web_browser_engine_playwright.py | 20 +++-- .../tools/test_web_browser_engine_selenium.py | 15 ++-- tests/metagpt/utils/test_config.py | 15 +--- 50 files changed, 437 insertions(+), 278 deletions(-) delete mode 100644 metagpt/llm.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index fa0d592a3..899c2515c 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : action.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from abc import ABC from typing import Optional @@ -11,15 +12,14 @@ from typing import Optional from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput -from metagpt.llm import LLM from metagpt.utils.common import OutputParser from metagpt.logs import logger + class Action(ABC): - def __init__(self, name: str = '', context=None, llm: LLM = None): + def __init__(self, options, name: str = '', context=None, llm=None): + self.options = options self.name: str = name - if llm is None: - llm = LLM() self.llm = llm self.context = context self.prefix = "" diff --git a/metagpt/actions/analyze_dep_libs.py b/metagpt/actions/analyze_dep_libs.py index 23c35cdf8..d7b251ead 100644 --- a/metagpt/actions/analyze_dep_libs.py +++ b/metagpt/actions/analyze_dep_libs.py @@ -4,6 +4,7 @@ @Time : 2023/5/19 12:01 @Author : alexanderwu @File : analyze_dep_libs.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import Action @@ -26,8 +27,8 @@ Focus only on the names of shared dependencies, do not add any other explanation class AnalyzeDepLibs(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name, context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) self.desc = "根据上下文，分析程序运行依赖库" async def run(self, requirement, filepaths_string): diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index d69a22dba..78c970337 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : debug_error.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import re @@ -25,8 +26,8 @@ Now you should start rewriting the code: ## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. """ class DebugError(Action): - def __init__(self, name="DebugError", context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="DebugError", context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) # async def run(self, code, error): # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 1447eacc3..eb08cb9f0 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : design_api.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import shutil from pathlib import Path @@ -90,8 +91,8 @@ OUTPUT_MAPPING = { class WriteDesign(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name, context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) self.desc = "Based on the PRD, think about the system design, and design the corresponding APIs, " \ "data structures, library tables, processes, and paths. Please provide your design, feedback " \ "clearly and in detail." @@ -106,15 +107,15 @@ class WriteDesign(Action): def _save_prd(self, docs_path, resources_path, prd): prd_file = docs_path / 'prd.md' quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) - mermaid_to_file(quadrant_chart, resources_path / 'competitive_analysis') + mermaid_to_file(options=self.options, mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / 'competitive_analysis') logger.info(f"Saving PRD to {prd_file}") prd_file.write_text(prd) def _save_system_design(self, docs_path, resources_path, content): data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content) seq_flow = CodeParser.parse_code(block="Program call flow", text=content) - mermaid_to_file(data_api_design, resources_path / 'data_api_design') - mermaid_to_file(seq_flow, resources_path / 'seq_flow') + mermaid_to_file(options=self.options, mermaid_code=data_api_design, output_file_without_suffix=resources_path / 'data_api_design') + mermaid_to_file(options=self.options, mermaid_code=seq_flow, output_file_without_suffix=resources_path / 'seq_flow') system_design_file = docs_path / 'system_design.md' logger.info(f"Saving System Designs to {system_design_file}") system_design_file.write_text(content) diff --git a/metagpt/actions/design_api_review.py b/metagpt/actions/design_api_review.py index 687a33652..ca4147cca 100644 --- a/metagpt/actions/design_api_review.py +++ b/metagpt/actions/design_api_review.py @@ -4,13 +4,14 @@ @Time : 2023/5/11 19:31 @Author : alexanderwu @File : design_api_review.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action class DesignReview(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name, context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) async def run(self, prd, api_design): prompt = f"Here is the Product Requirement Document (PRD):\n\n{prd}\n\nHere is the list of APIs designed " \ diff --git a/metagpt/actions/design_filenames.py b/metagpt/actions/design_filenames.py index 6c3d8e803..1f71e9530 100644 --- a/metagpt/actions/design_filenames.py +++ b/metagpt/actions/design_filenames.py @@ -4,6 +4,7 @@ @Time : 2023/5/19 11:50 @Author : alexanderwu @File : design_filenames.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import Action from metagpt.logs import logger @@ -15,8 +16,8 @@ Do not add any other explanations, just return a Python string list.""" class DesignFilenames(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name, context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) self.desc = "Based on the PRD, consider system design, and carry out the basic design of the corresponding " \ "APIs, data structures, and database tables. Please give your design, feedback clearly and in detail." diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 89c59dcda..3d8aa9322 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 19:12 @Author : alexanderwu @File : project_management.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import List, Tuple @@ -103,8 +104,8 @@ OUTPUT_MAPPING = { class WriteTasks(Action): - def __init__(self, name="CreateTasks", context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="CreateTasks", context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) def _save(self, context, rsp): ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) diff --git a/metagpt/actions/research.py b/metagpt/actions/research.py index 81eb876dd..22b0eaa1d 100644 --- a/metagpt/actions/research.py +++ b/metagpt/actions/research.py @@ -1,5 +1,9 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + from __future__ import annotations import asyncio @@ -9,7 +13,6 @@ from typing import Callable from pydantic import parse_obj_as from metagpt.actions import Action -from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.tools.search_engine import SearchEngine from metagpt.tools.web_browser_engine import WebBrowserEngine, WebBrowserEngineType @@ -79,14 +82,15 @@ class CollectLinks(Action): """Action class to collect links from a search engine.""" def __init__( self, + options, name: str = "", *args, rank_func: Callable[[list[str]], None] | None = None, **kwargs, ): - super().__init__(name, *args, **kwargs) + super().__init__(options=options, name=name, *args, **kwargs) self.desc = "Collect links from a search engine." - self.search_engine = SearchEngine() + self.search_engine = SearchEngine(options=options) self.rank_func = rank_func async def run( @@ -126,7 +130,7 @@ class CollectLinks(Action): remove.pop() if len(remove) == 0: break - prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, CONFIG.max_tokens_rsp) + prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, self.options.get("max_tokens_rsp")) logger.debug(prompt) queries = await self._aask(prompt, [system_text]) try: @@ -178,9 +182,10 @@ class WebBrowseAndSummarize(Action): **kwargs, ): super().__init__(*args, **kwargs) - if CONFIG.model_for_researcher_summary: - self.llm.model = CONFIG.model_for_researcher_summary + if self.options.get("model_for_researcher_summary"): + self.llm.model = self.options.get("model_for_researcher_summary") self.web_browser_engine = WebBrowserEngine( + options=self.options, engine=WebBrowserEngineType.CUSTOM if browse_func else None, run_func=browse_func, ) @@ -213,7 +218,8 @@ class WebBrowseAndSummarize(Action): for u, content in zip([url, *urls], contents): content = content.inner_text chunk_summaries = [] - for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, CONFIG.max_tokens_rsp): + for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, + self.options.get("max_tokens_rsp")): logger.debug(prompt) summary = await self._aask(prompt, [system_text]) if summary == "Not relevant.": @@ -239,8 +245,8 @@ class ConductResearch(Action): """Action class to conduct research and generate a research report.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if CONFIG.model_for_researcher_report: - self.llm.model = CONFIG.model_for_researcher_report + if self.options.get("model_for_researcher_report"): + self.llm.model = self.options.get("model_for_researcher_report") async def run( self, diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index f69d2cd1a..824ed83fa 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : run_code.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import os import subprocess @@ -57,8 +58,8 @@ standard errors: {errs}; class RunCode(Action): - def __init__(self, name="RunCode", context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="RunCode", context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) @classmethod async def run_text(cls, code) -> Tuple[str, str]: diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 5e4cdaea0..80d1c52e4 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -4,11 +4,11 @@ @Time : 2023/5/23 17:26 @Author : alexanderwu @File : search_google.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import pydantic from metagpt.actions import Action -from metagpt.config import Config from metagpt.logs import logger from metagpt.schema import Message from metagpt.tools.search_engine import SearchEngine @@ -101,17 +101,16 @@ You are a member of a professional butler team and will provide helpful suggesti class SearchAndSummarize(Action): - def __init__(self, name="", context=None, llm=None, engine=None, search_func=None): - self.config = Config() - self.engine = engine or self.config.search_engine + def __init__(self, options, name="", context=None, llm=None, engine=None, search_func=None): + self.engine = engine or options.get("search_engine") try: - self.search_engine = SearchEngine(self.engine, run_func=search_func) + self.search_engine = SearchEngine(options=options, engine=self.engine, run_func=search_func) except pydantic.ValidationError: self.search_engine = None self.result = "" - super().__init__(name, context, llm) + super().__init__(options=options, name=name, context=context, llm=llm) async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYSTEM) -> str: if self.search_engine is None: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index cc122ef7a..9a2a2f81a 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import WriteDesign from metagpt.actions.action import Action @@ -43,8 +44,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc class WriteCode(Action): - def __init__(self, name="WriteCode", context: list[Message] = None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="WriteCode", context: list[Message] = None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) def _is_invalid(self, filename): return any(i in filename for i in ["mp3", "wav"]) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 7f6a7a38e..d256c6bcb 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code_review.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action @@ -62,8 +63,8 @@ FORMAT_EXAMPLE = """ class WriteCodeReview(Action): - def __init__(self, name="WriteCodeReview", context: list[Message] = None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="WriteCodeReview", context: list[Message] = None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt): diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 0edd24d55..794d3ee9d 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import List, Tuple @@ -127,11 +128,11 @@ OUTPUT_MAPPING = { class WritePRD(Action): - def __init__(self, name="", context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="", context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) async def run(self, requirements, *args, **kwargs) -> ActionOutput: - sas = SearchAndSummarize() + sas = SearchAndSummarize(options=self.options, llm=self.llm) # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) rsp = "" info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" diff --git a/metagpt/actions/write_prd_review.py b/metagpt/actions/write_prd_review.py index 5ff9624c5..8c22f9c0a 100644 --- a/metagpt/actions/write_prd_review.py +++ b/metagpt/actions/write_prd_review.py @@ -4,13 +4,14 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd_review.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action class WritePRDReview(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name, context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) self.prd = None self.desc = "Based on the PRD, conduct a PRD Review, providing clear and detailed feedback" self.prd_review_prompt_template = """ diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 5e50fdb55..94006005f 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_test.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action from metagpt.utils.common import CodeParser @@ -30,8 +31,8 @@ you should correctly import the necessary classes based on these file locations! class WriteTest(Action): - def __init__(self, name="WriteTest", context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="WriteTest", context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) async def write_code(self, prompt): code_rsp = await self._aask(prompt) diff --git a/metagpt/config.py b/metagpt/config.py index 6e2cf0a3f..076bc5eb7 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -118,8 +118,8 @@ class Config: return value @property - def options(self): - """Return key-value configuration parameters.""" + def runtime_options(self): + """Runtime key-value configuration parameters.""" opts = {} for k, v in self._configs.items(): opts[k] = v diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 051bc2507..d15eb4c21 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -4,6 +4,7 @@ @Time : 2023/5/25 10:20 @Author : alexanderwu @File : faiss_store.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import pickle from pathlib import Path @@ -36,8 +37,11 @@ class FaissStore(LocalStore): store.index = index return store - def _write(self, docs, metadatas): - store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07"), metadatas=metadatas) + def _write(self, docs, metadatas, **kwargs): + store = FAISS.from_texts(docs, + OpenAIEmbeddings(openai_api_version="2020-11-07", + openai_api_key=kwargs.get("OPENAI_API_KEY")), + metadatas=metadatas) return store def persist(self): diff --git a/metagpt/llm.py b/metagpt/llm.py deleted file mode 100644 index 6a9a9132f..000000000 --- a/metagpt/llm.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/11 14:45 -@Author : alexanderwu -@File : llm.py -""" - -from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM - -DEFAULT_LLM = LLM() -CLAUDE_LLM = Claude() - - -async def ai_func(prompt): - """使用LLM进行QA - QA with LLMs - """ - return await DEFAULT_LLM.aask(prompt) diff --git a/metagpt/management/skill_manager.py b/metagpt/management/skill_manager.py index f067e6df6..4f141832a 100644 --- a/metagpt/management/skill_manager.py +++ b/metagpt/management/skill_manager.py @@ -4,11 +4,11 @@ @Time : 2023/6/5 01:44 @Author : alexanderwu @File : skill_manager.py +@Modified By: mashenquan, 2023/8/20. Remove useless `_llm` """ from metagpt.actions import Action from metagpt.const import PROMPT_PATH from metagpt.document_store.chromadb_store import ChromaStore -from metagpt.llm import LLM from metagpt.logs import logger Skill = Action @@ -18,7 +18,6 @@ class SkillManager: """用来管理所有技能""" def __init__(self): - self._llm = LLM() self._store = ChromaStore('skill_manager') self._skills: dict[str: Skill] = {} diff --git a/metagpt/manager.py b/metagpt/manager.py index 9d238c621..c4565808e 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -4,14 +4,15 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : manager.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ -from metagpt.llm import LLM + from metagpt.logs import logger from metagpt.schema import Message class Manager: - def __init__(self, llm: LLM = LLM()): + def __init__(self, llm): self.llm = llm # Large Language Model self.role_directions = { "BOSS": "Product Manager", diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index 3c2963613..041d335ac 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -1,6 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the implement of Long-term memory +""" +@Desc : the implement of Long-term memory +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" from metagpt.logs import logger from metagpt.memory import Memory @@ -34,13 +37,13 @@ class LongTermMemory(Memory): self.add_batch(messages) self.msg_from_recover = False - def add(self, message: Message): + def add(self, message: Message, **kwargs): super(LongTermMemory, self).add(message) for action in self.rc.watch: if message.cause_by == action and not self.msg_from_recover: # currently, only add role's watching messages to its memory_storage # and ignore adding messages from recover repeatedly - self.memory_storage.add(message) + self.memory_storage.add(message, **kwargs) def remember(self, observed: list[Message], k=0) -> list[Message]: """ diff --git a/metagpt/memory/memory_storage.py b/metagpt/memory/memory_storage.py index 5421e9e65..09cd67410 100644 --- a/metagpt/memory/memory_storage.py +++ b/metagpt/memory/memory_storage.py @@ -1,6 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the implement of memory storage +""" +@Desc : the implement of memory storage +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" from typing import List from pathlib import Path @@ -61,13 +64,13 @@ class MemoryStorage(FaissStore): super(MemoryStorage, self).persist() logger.debug(f'Agent {self.role_id} persist memory into local') - def add(self, message: Message) -> bool: + def add(self, message: Message, **kwargs) -> bool: """ add message into memory storage""" docs = [message.content] metadatas = [{"message_ser": serialize_message(message)}] if not self.store: # init Faiss - self.store = self._write(docs, metadatas) + self.store = self._write(docs, metadatas, **kwargs) self._initialized = True else: self.store.add_texts(texts=docs, metadatas=metadatas) diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 03802a716..326d23a5c 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -4,17 +4,22 @@ @Time : 2023/7/21 11:15 @Author : Leo Xiao @File : anthropic_api.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ import anthropic from anthropic import Anthropic -from metagpt.config import CONFIG +from metagpt.config import Config class Claude2: + def __init__(self, options=None): + self.options = options or Config().runtime_options + def ask(self, prompt): - client = Anthropic(api_key=CONFIG.claude_api_key) + client = Anthropic(api_key=self.claude_api_key) res = client.completions.create( model="claude-2", @@ -24,7 +29,7 @@ class Claude2: return res.completion async def aask(self, prompt): - client = Anthropic(api_key=CONFIG.claude_api_key) + client = Anthropic(api_key=self.claude_api_key) res = client.completions.create( model="claude-2", @@ -32,3 +37,7 @@ class Claude2: max_tokens_to_sample=1000, ) return res.completion + + @property + def claude_api_key(self): + return self.options.get("claude_api_key") diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 79121c8de..2e951b36f 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -3,6 +3,8 @@ @Time : 2023/5/5 23:08 @Author : alexanderwu @File : openai.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ import asyncio import time @@ -12,10 +14,8 @@ import openai from openai.error import APIConnectionError from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type -from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.utils.singleton import Singleton from metagpt.utils.token_counter import ( TOKEN_COSTS, count_message_tokens, @@ -56,13 +56,13 @@ class Costs(NamedTuple): total_budget: float -class CostManager(metaclass=Singleton): +class CostManager: """计算使用接口的开销""" - def __init__(self): + def __init__(self, options): self.total_prompt_tokens = 0 self.total_completion_tokens = 0 - self.total_cost = 0 + self.options = options self.total_budget = 0 def update_cost(self, prompt_tokens, completion_tokens, model): @@ -79,10 +79,9 @@ class CostManager(metaclass=Singleton): cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model]["completion"]) / 1000 self.total_cost += cost logger.info( - f"Total running cost: ${self.total_cost:.3f} | Max budget: ${CONFIG.max_budget:.3f} | " + f"Total running cost: ${self.total_cost:.3f} | Max budget: ${self.max_budget:.3f} | " f"Current cost: ${cost:.3f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}" ) - CONFIG.total_cost = self.total_cost def get_total_prompt_tokens(self): """ @@ -115,6 +114,18 @@ class CostManager(metaclass=Singleton): """获得所有开销""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) + @property + def total_cost(self): + return self.options.get("total_cost", 0) + + @total_cost.setter + def total_cost(self, v): + self.options["total_cost"] = v + + @property + def max_budget(self): + return self.options.get("max_budget", 0) + def log_and_reraise(retry_state): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") @@ -130,22 +141,23 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): Check https://platform.openai.com/examples for examples """ - def __init__(self): - self.__init_openai(CONFIG) + def __init__(self, options, cost_manager): + self._options = options + self.__init_openai() self.llm = openai - self.model = CONFIG.openai_api_model + self.model = self.openai_api_model self.auto_max_tokens = False - self._cost_manager = CostManager() + self._cost_manager = cost_manager RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self, config): - openai.api_key = config.openai_api_key - if config.openai_api_base: - openai.api_base = config.openai_api_base - if config.openai_api_type: - openai.api_type = config.openai_api_type - openai.api_version = config.openai_api_version - self.rpm = int(config.get("RPM", 10)) + def __init_openai(self): + openai.api_key = self.openai_api_key + if self.openai_api_base: + openai.api_base = self.openai_api_base + if self.openai_api_type: + openai.api_type = self.openai_api_type + openai.api_version = self.openai_api_version + self.rpm = int(self._options.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: response = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages), stream=True) @@ -168,9 +180,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return full_reply_content def _cons_kwargs(self, messages: list[dict]) -> dict: - if CONFIG.openai_api_type == "azure": + if self._options.get("openai_api_type") == "azure": kwargs = { - "deployment_id": CONFIG.deployment_id, + "deployment_id": self._options.get("deployment_id"), "messages": messages, "max_tokens": self.get_max_tokens(messages), "n": 1, @@ -225,7 +237,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def _calc_usage(self, messages: list[dict], rsp: str) -> dict: usage = {} - if CONFIG.calc_usage: + if self._options.get("calc_usage"): try: prompt_tokens = count_message_tokens(messages, self.model) completion_tokens = count_string_tokens(rsp, self.model) @@ -264,7 +276,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return results def _update_costs(self, usage: dict): - if CONFIG.calc_usage: + if self._options.get("calc_usage"): try: prompt_tokens = int(usage['prompt_tokens']) completion_tokens = int(usage['completion_tokens']) @@ -277,5 +289,25 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def get_max_tokens(self, messages: list[dict]): if not self.auto_max_tokens: - return CONFIG.max_tokens_rsp - return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) + return self._options.get("max_tokens_rsp") + return get_max_completion_tokens(messages, self.model, self._options.get("max_tokens_rsp")) + + @property + def openai_api_model(self): + return self._options.get("openai_api_model") + + @property + def openai_api_key(self): + return self._options.get("openai_api_key") + + @property + def openai_api_base(self): + return self._options.get("openai_api_base") + + @property + def openai_api_type(self): + return self._options.get("openai_api_type") + + @property + def openai_api_version(self): + return self._options.get("openai_api_version") diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 00b6cb2eb..5a498c50b 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -4,6 +4,8 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : architect.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ from metagpt.actions import WriteDesign, WritePRD @@ -12,8 +14,8 @@ from metagpt.roles import Role class Architect(Role): """Architect: Listen to PRD, responsible for designing API, designing code files""" - def __init__(self, name="Bob", profile="Architect", goal="Design a concise, usable, complete python system", + def __init__(self, options, cost_manager, name="Bob", profile="Architect", goal="Design a concise, usable, complete python system", constraints="Try to specify good open source tools as much as possible"): - super().__init__(name, profile, goal, constraints) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) self._init_actions([WriteDesign]) self._watch({WritePRD}) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 072e53998..9da2b5a09 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -47,10 +47,10 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): - def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", + def __init__(self, options, cost_manager, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", n_borg=1, use_code_review=False): - super().__init__(name, profile, goal, constraints) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) self._init_actions([WriteCode]) self.use_code_review = use_code_review if self.use_code_review: @@ -131,7 +131,7 @@ class Engineer(Role): async def _act_sp(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later for todo in self.todos: - code = await WriteCode().run( + code = await WriteCode(options=self.options, llm=self._llm).run( context=self._rc.history, filename=todo ) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index b42e9bb29..bb69c8dfd 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -4,14 +4,16 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : product_manager.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ from metagpt.actions import BossRequirement, WritePRD from metagpt.roles import Role class ProductManager(Role): - def __init__(self, name="Alice", profile="Product Manager", goal="Efficiently create a successful product", + def __init__(self, options, cost_manager, name="Alice", profile="Product Manager", goal="Efficiently create a successful product", constraints=""): - super().__init__(name, profile, goal, constraints) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) self._init_actions([WritePRD]) self._watch([BossRequirement]) diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index ff374de13..3e8b36550 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -4,14 +4,16 @@ @Time : 2023/5/11 15:04 @Author : alexanderwu @File : project_manager.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ from metagpt.actions import WriteDesign, WriteTasks from metagpt.roles import Role class ProjectManager(Role): - def __init__(self, name="Eve", profile="Project Manager", + def __init__(self, options, cost_manager, name="Eve", profile="Project Manager", goal="Improve team efficiency and deliver with quality and quantity", constraints=""): - super().__init__(name, profile, goal, constraints) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) self._init_actions([WriteTasks]) self._watch([WriteDesign]) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 65bf2cc5b..ac5df0dbd 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -20,13 +20,15 @@ from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP class QaEngineer(Role): def __init__( self, + options, + cost_manager, name="Edward", profile="QaEngineer", goal="Write comprehensive and robust tests to ensure codes will work as expected without bugs", constraints="The test code you write should conform to code standard like PEP8, be modular, easy to read and maintain", test_round_allowed=5, ): - super().__init__(name, profile, goal, constraints) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index d3750495f..3c72876a5 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,17 +4,16 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ from __future__ import annotations -from typing import Iterable, Type +from typing import Iterable, Type, Dict from pydantic import BaseModel, Field - -# from metagpt.environment import Environment -from metagpt.config import CONFIG +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM from metagpt.actions import Action, ActionOutput -from metagpt.llm import LLM from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory from metagpt.schema import Message @@ -71,12 +70,13 @@ class RoleContext(BaseModel): todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) + options: Dict class Config: arbitrary_types_allowed = True def check(self, role_id: str): - if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: + if self.options.get("long_term_memory"): self.long_term_memory.recover_memory(role_id, self) self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation @@ -93,13 +93,15 @@ class RoleContext(BaseModel): class Role: """角色/代理""" - def __init__(self, name="", profile="", goal="", constraints="", desc=""): - self._llm = LLM() + def __init__(self, options, cost_manager, name="", profile="", goal="", constraints="", desc=""): + self._options = options if options else {} + self._cost_manager = cost_manager + self._llm = LLM(options=self._options, cost_manager=cost_manager) self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) self._states = [] self._actions = [] self._role_id = str(self._setting) - self._rc = RoleContext() + self._rc = RoleContext(options=options) def _reset(self): self._states = [] @@ -109,7 +111,7 @@ class Role: self._reset() for idx, action in enumerate(actions): if not isinstance(action, Action): - i = action("") + i = action(options=self._options, name="", llm=self._llm) else: i = action i.set_prefix(self._get_prefix(), self.profile) @@ -137,6 +139,14 @@ class Role: """获取角色描述（职位）""" return self._setting.profile + @property + def options(self): + return self._options + + @options.setter + def options(self, opts): + self._options.update(opts) + def _get_prefix(self): """获取角色前缀""" if self._setting.desc: diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 8f173ebf3..3f6f484b4 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,16 +4,21 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ +from typing import Dict + from pydantic import BaseModel, Field from metagpt.actions import BossRequirement -from metagpt.config import CONFIG from metagpt.environment import Environment from metagpt.logs import logger +from metagpt.provider.openai_api import CostManager from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import NoMoneyException +from metagpt.config import Config class SoftwareCompany(BaseModel): @@ -24,6 +29,8 @@ class SoftwareCompany(BaseModel): environment: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") + options: Dict = Field(default=Config().runtime_options) + cost_manager: CostManager = Field(default=CostManager(Config().runtime_options)) class Config: arbitrary_types_allowed = True @@ -35,12 +42,12 @@ class SoftwareCompany(BaseModel): def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" self.investment = investment - CONFIG.max_budget = investment + self.options["max_budget"] = investment logger.info(f'Investment: ${investment}.') def _check_balance(self): - if CONFIG.total_cost > CONFIG.max_budget: - raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') + if self.total_cost > self.max_budget: + raise NoMoneyException(self.total_cost, f'Insufficient funds: {self.max_budget}') def start_project(self, idea): """Start a project from publishing boss requirement.""" @@ -59,3 +66,13 @@ class SoftwareCompany(BaseModel): self._check_balance() await self.environment.run() return self.environment.history + + @property + def max_budget(self): + return self.options.get("max_budget", 0) + + @property + def total_cost(self): + return self.options.get("total_cost", 0) + + diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index d28700054..c82ae6595 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -4,13 +4,13 @@ @Time : 2023/5/6 20:15 @Author : alexanderwu @File : search_engine.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from __future__ import annotations import importlib -from typing import Callable, Coroutine, Literal, overload +from typing import Callable, Coroutine, Literal, overload, Dict -from metagpt.config import CONFIG from metagpt.tools import SearchEngineType @@ -25,24 +25,26 @@ class SearchEngine: run_func: The function to run the search. engine: The search engine type. """ + def __init__( - self, - engine: SearchEngineType | None = None, - run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None, + self, + options: Dict, + engine: SearchEngineType | None = None, + run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None ): - engine = engine or CONFIG.search_engine + engine = engine or options.get("search_engine") if engine == SearchEngineType.SERPAPI_GOOGLE: module = "metagpt.tools.search_engine_serpapi" - run_func = importlib.import_module(module).SerpAPIWrapper().run + run_func = importlib.import_module(module).SerpAPIWrapper(**options).run elif engine == SearchEngineType.SERPER_GOOGLE: module = "metagpt.tools.search_engine_serper" - run_func = importlib.import_module(module).SerperWrapper().run + run_func = importlib.import_module(module).SerperWrapper(**options).run elif engine == SearchEngineType.DIRECT_GOOGLE: module = "metagpt.tools.search_engine_googleapi" - run_func = importlib.import_module(module).GoogleAPIWrapper().run + run_func = importlib.import_module(module).GoogleAPIWrapper(**options).run elif engine == SearchEngineType.DUCK_DUCK_GO: module = "metagpt.tools.search_engine_ddg" - run_func = importlib.import_module(module).DDGAPIWrapper().run + run_func = importlib.import_module(module).DDGAPIWrapper(**options).run elif engine == SearchEngineType.CUSTOM_ENGINE: pass # run_func = run_func else: @@ -52,19 +54,19 @@ class SearchEngine: @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[True] = True, + self, + query: str, + max_results: int = 8, + as_string: Literal[True] = True, ) -> str: ... @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[False] = False, + self, + query: str, + max_results: int = 8, + as_string: Literal[False] = False, ) -> list[dict[str, str]]: ... diff --git a/metagpt/tools/search_engine_ddg.py b/metagpt/tools/search_engine_ddg.py index 57bc61b82..78562c77e 100644 --- a/metagpt/tools/search_engine_ddg.py +++ b/metagpt/tools/search_engine_ddg.py @@ -1,11 +1,14 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" from __future__ import annotations import asyncio import json from concurrent import futures -from typing import Literal, overload +from typing import Literal, overload, Optional try: from duckduckgo_search import DDGS @@ -15,8 +18,6 @@ except ImportError: "You can install it by running the command: `pip install -e.[search-ddg]`" ) -from metagpt.config import CONFIG - class DDGAPIWrapper: """Wrapper around duckduckgo_search API. @@ -25,43 +26,44 @@ class DDGAPIWrapper: """ def __init__( - self, - *, - loop: asyncio.AbstractEventLoop | None = None, - executor: futures.Executor | None = None, + self, + *, + global_proxy: Optional[str] = None, + loop: asyncio.AbstractEventLoop | None = None, + executor: futures.Executor | None = None, ): kwargs = {} - if CONFIG.global_proxy: - kwargs["proxies"] = CONFIG.global_proxy + if global_proxy: + kwargs["proxies"] = global_proxy self.loop = loop self.executor = executor self.ddgs = DDGS(**kwargs) @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[True] = True, - focus: list[str] | None = None, + self, + query: str, + max_results: int = 8, + as_string: Literal[True] = True, + focus: list[str] | None = None, ) -> str: ... @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[False] = False, - focus: list[str] | None = None, + self, + query: str, + max_results: int = 8, + as_string: Literal[False] = False, + focus: list[str] | None = None, ) -> list[dict[str, str]]: ... async def run( - self, - query: str, - max_results: int = 8, - as_string: bool = True, + self, + query: str, + max_results: int = 8, + as_string: bool = True, ) -> str | list[dict]: """Return the results of a Google search using the official Google API diff --git a/metagpt/tools/search_engine_googleapi.py b/metagpt/tools/search_engine_googleapi.py index b9faf2ced..b5aeb5875 100644 --- a/metagpt/tools/search_engine_googleapi.py +++ b/metagpt/tools/search_engine_googleapi.py @@ -1,5 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" from __future__ import annotations import asyncio @@ -11,7 +14,6 @@ from urllib.parse import urlparse import httplib2 from pydantic import BaseModel, validator -from metagpt.config import CONFIG from metagpt.logs import logger try: @@ -27,6 +29,7 @@ except ImportError: class GoogleAPIWrapper(BaseModel): google_api_key: Optional[str] = None google_cse_id: Optional[str] = None + global_proxy: Optional[str] = None loop: Optional[asyncio.AbstractEventLoop] = None executor: Optional[futures.Executor] = None @@ -36,7 +39,6 @@ class GoogleAPIWrapper(BaseModel): @validator("google_api_key", always=True) @classmethod def check_google_api_key(cls, val: str): - val = val or CONFIG.google_api_key if not val: raise ValueError( "To use, make sure you provide the google_api_key when constructing an object. Alternatively, " @@ -47,8 +49,7 @@ class GoogleAPIWrapper(BaseModel): @validator("google_cse_id", always=True) @classmethod - def check_google_cse_id(cls, val: str): - val = val or CONFIG.google_cse_id + def check_google_cse_id(cls, val): if not val: raise ValueError( "To use, make sure you provide the google_cse_id when constructing an object. Alternatively, " @@ -60,8 +61,8 @@ class GoogleAPIWrapper(BaseModel): @property def google_api_client(self): build_kwargs = {"developerKey": self.google_api_key} - if CONFIG.global_proxy: - parse_result = urlparse(CONFIG.global_proxy) + if self.global_proxy: + parse_result = urlparse(self.global_proxy) proxy_type = parse_result.scheme if proxy_type == "https": proxy_type = "http" diff --git a/metagpt/tools/search_engine_serpapi.py b/metagpt/tools/search_engine_serpapi.py index 750184198..1b93a91e9 100644 --- a/metagpt/tools/search_engine_serpapi.py +++ b/metagpt/tools/search_engine_serpapi.py @@ -4,13 +4,14 @@ @Time : 2023/5/23 18:27 @Author : alexanderwu @File : search_engine_serpapi.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import Any, Dict, Optional, Tuple import aiohttp from pydantic import BaseModel, Field, validator -from metagpt.config import CONFIG +from metagpt.config import Config class SerpAPIWrapper(BaseModel): @@ -32,7 +33,6 @@ class SerpAPIWrapper(BaseModel): @validator("serpapi_api_key", always=True) @classmethod def check_serpapi_api_key(cls, val: str): - val = val or CONFIG.serpapi_api_key if not val: raise ValueError( "To use, make sure you provide the serpapi_api_key when constructing an object. Alternatively, " @@ -112,4 +112,4 @@ class SerpAPIWrapper(BaseModel): if __name__ == "__main__": import fire - fire.Fire(SerpAPIWrapper().run) + fire.Fire(SerpAPIWrapper(Config().runtime_options).run) diff --git a/metagpt/tools/search_engine_serper.py b/metagpt/tools/search_engine_serper.py index 0eec2694b..849839f05 100644 --- a/metagpt/tools/search_engine_serper.py +++ b/metagpt/tools/search_engine_serper.py @@ -4,6 +4,7 @@ @Time : 2023/5/23 18:27 @Author : alexanderwu @File : search_engine_serpapi.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import json from typing import Any, Dict, Optional, Tuple @@ -11,8 +12,6 @@ from typing import Any, Dict, Optional, Tuple import aiohttp from pydantic import BaseModel, Field, validator -from metagpt.config import CONFIG - class SerperWrapper(BaseModel): search_engine: Any #: :meta private: @@ -26,7 +25,6 @@ class SerperWrapper(BaseModel): @validator("serper_api_key", always=True) @classmethod def check_serper_api_key(cls, val: str): - val = val or CONFIG.serper_api_key if not val: raise ValueError( "To use, make sure you provide the serper_api_key when constructing an object. Alternatively, " diff --git a/metagpt/tools/web_browser_engine.py b/metagpt/tools/web_browser_engine.py index 453d87f31..da208dbc9 100644 --- a/metagpt/tools/web_browser_engine.py +++ b/metagpt/tools/web_browser_engine.py @@ -1,29 +1,33 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" from __future__ import annotations import importlib -from typing import Any, Callable, Coroutine, Literal, overload +from typing import Any, Callable, Coroutine, Literal, overload, Dict -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.tools import WebBrowserEngineType from metagpt.utils.parse_html import WebPage class WebBrowserEngine: def __init__( - self, - engine: WebBrowserEngineType | None = None, - run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None, + self, + options: Dict, + engine: WebBrowserEngineType | None = None, + run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None, ): - engine = engine or CONFIG.web_browser_engine + engine = engine or options.get("web_browser_engine") if engine == WebBrowserEngineType.PLAYWRIGHT: module = "metagpt.tools.web_browser_engine_playwright" - run_func = importlib.import_module(module).PlaywrightWrapper().run + run_func = importlib.import_module(module).PlaywrightWrapper(options=options).run elif engine == WebBrowserEngineType.SELENIUM: module = "metagpt.tools.web_browser_engine_selenium" - run_func = importlib.import_module(module).SeleniumWrapper().run + run_func = importlib.import_module(module).SeleniumWrapper(options=options).run elif engine == WebBrowserEngineType.CUSTOM: run_func = run_func else: @@ -47,6 +51,10 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, engine_type: Literal["playwright", "selenium"] = "playwright", **kwargs): - return await WebBrowserEngine(WebBrowserEngineType(engine_type), **kwargs).run(url, *urls) + conf = Config() + return await WebBrowserEngine(options=conf.runtime_options, + engine=WebBrowserEngineType(engine_type), + **kwargs).run(url, *urls) + fire.Fire(main) diff --git a/metagpt/tools/web_browser_engine_playwright.py b/metagpt/tools/web_browser_engine_playwright.py index 030e7701b..199f8a0d1 100644 --- a/metagpt/tools/web_browser_engine_playwright.py +++ b/metagpt/tools/web_browser_engine_playwright.py @@ -1,14 +1,18 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + from __future__ import annotations import asyncio import sys from pathlib import Path -from typing import Literal +from typing import Literal, Dict from playwright.async_api import async_playwright -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.logs import logger from metagpt.utils.parse_html import WebPage @@ -24,18 +28,20 @@ class PlaywrightWrapper: def __init__( self, + options: Dict, browser_type: Literal["chromium", "firefox", "webkit"] | None = None, launch_kwargs: dict | None = None, **kwargs, ) -> None: + self.options = options if browser_type is None: - browser_type = CONFIG.playwright_browser_type + browser_type = options.get("playwright_browser_type") self.browser_type = browser_type launch_kwargs = launch_kwargs or {} - if CONFIG.global_proxy and "proxy" not in launch_kwargs: + if options.get("global_proxy") and "proxy" not in launch_kwargs: args = launch_kwargs.get("args", []) if not any(str.startswith(i, "--proxy-server=") for i in args): - launch_kwargs["proxy"] = {"server": CONFIG.global_proxy} + launch_kwargs["proxy"] = {"server": options.get("global_proxy")} self.launch_kwargs = launch_kwargs context_kwargs = {} if "ignore_https_errors" in kwargs: @@ -75,8 +81,8 @@ class PlaywrightWrapper: executable_path = Path(browser_type.executable_path) if not executable_path.exists() and "executable_path" not in self.launch_kwargs: kwargs = {} - if CONFIG.global_proxy: - kwargs["env"] = {"ALL_PROXY": CONFIG.global_proxy} + if self.options.get("global_proxy"): + kwargs["env"] = {"ALL_PROXY": self.options.get("global_proxy")} await _install_browsers(self.browser_type, **kwargs) if self._has_run_precheck: @@ -144,6 +150,8 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, browser_type: str = "chromium", **kwargs): - return await PlaywrightWrapper(browser_type, **kwargs).run(url, *urls) + return await PlaywrightWrapper(options=Config().runtime_options, + browser_type=browser_type, + **kwargs).run(url, *urls) fire.Fire(main) diff --git a/metagpt/tools/web_browser_engine_selenium.py b/metagpt/tools/web_browser_engine_selenium.py index d727709b8..b0fcb3fe1 100644 --- a/metagpt/tools/web_browser_engine_selenium.py +++ b/metagpt/tools/web_browser_engine_selenium.py @@ -1,17 +1,21 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + from __future__ import annotations import asyncio import importlib from concurrent import futures from copy import deepcopy -from typing import Literal +from typing import Literal, Dict from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.utils.parse_html import WebPage @@ -29,6 +33,7 @@ class SeleniumWrapper: def __init__( self, + options: Dict, browser_type: Literal["chrome", "firefox", "edge", "ie"] | None = None, launch_kwargs: dict | None = None, *, @@ -36,11 +41,11 @@ class SeleniumWrapper: executor: futures.Executor | None = None, ) -> None: if browser_type is None: - browser_type = CONFIG.selenium_browser_type + browser_type = options.get("selenium_browser_type") self.browser_type = browser_type launch_kwargs = launch_kwargs or {} - if CONFIG.global_proxy and "proxy-server" not in launch_kwargs: - launch_kwargs["proxy-server"] = CONFIG.global_proxy + if options.get("global_proxy") and "proxy-server" not in launch_kwargs: + launch_kwargs["proxy-server"] = options.get("global_proxy") self.executable_path = launch_kwargs.pop("executable_path", None) self.launch_args = [f"--{k}={v}" for k, v in launch_kwargs.items()] @@ -118,6 +123,8 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, browser_type: str = "chrome", **kwargs): - return await SeleniumWrapper(browser_type, **kwargs).run(url, *urls) + return await SeleniumWrapper(options=Config().runtime_options, + browser_type=browser_type, + **kwargs).run(url, *urls) fire.Fire(main) diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 24aabe8ae..1245671fb 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -4,19 +4,21 @@ @Time : 2023/7/4 10:53 @Author : alexanderwu @File : mermaid.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import subprocess from pathlib import Path -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.const import PROJECT_ROOT from metagpt.logs import logger from metagpt.utils.common import check_cmd_exists -def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: +def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: """suffix: png/svg/pdf + :param options: runtime context options, created by `Config` class object and changed in flow pipeline :param mermaid_code: mermaid code :param output_file_without_suffix: output filename :param width: @@ -36,12 +38,12 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height # Call the `mmdc` command to convert the Mermaid code to a PNG logger.info(f"Generating {output_file}..") - if CONFIG.puppeteer_config: + if options.get("puppeteer_config"): subprocess.run( [ - CONFIG.mmdc, + options.get("mmdc"), "-p", - CONFIG.puppeteer_config, + options.get("puppeteer_config"), "-i", str(tmp), "-o", @@ -53,7 +55,7 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height ] ) else: - subprocess.run([CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) + subprocess.run([options.get("mmdc"), "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) return 0 @@ -109,6 +111,8 @@ MMC2 = """sequenceDiagram if __name__ == "__main__": - # logger.info(print_members(print_members)) - mermaid_to_file(MMC1, PROJECT_ROOT / "tmp/1.png") - mermaid_to_file(MMC2, PROJECT_ROOT / "tmp/2.png") + conf = Config() + mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC1, + output_file_without_suffix=PROJECT_ROOT / "tmp/1.png") + mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC2, + output_file_without_suffix=PROJECT_ROOT / "tmp/2.png") diff --git a/startup.py b/startup.py index f37b5286c..116e4073d 100644 --- a/startup.py +++ b/startup.py @@ -1,5 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. +""" + import asyncio import fire @@ -11,14 +16,15 @@ from metagpt.software_company import SoftwareCompany async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False, run_tests: bool = False): """Run a startup. Be a boss.""" + company = SoftwareCompany() - company.hire([ProductManager(), - Architect(), - ProjectManager(), - Engineer(n_borg=5, use_code_review=code_review)]) + company.hire([ProductManager(options=company.options, cost_manager=company.cost_manager), + Architect(options=company.options, cost_manager=company.cost_manager), + ProjectManager(options=company.options, cost_manager=company.cost_manager), + Engineer(n_borg=5, use_code_review=code_review, options=company.options, cost_manager=company.cost_manager)]) if run_tests: # developing features: run tests on the spot and identify bugs (bug fixing capability comes soon!) - company.hire([QaEngineer()]) + company.hire([QaEngineer(options=company.options, cost_manager=company.cost_manager)]) company.invest(investment) company.start_project(idea) await company.run(n_round=n_round) diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 7bb18ddf2..04216ad7c 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -4,11 +4,13 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_code.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import pytest +from metagpt.config import Config +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager from metagpt.actions.write_code import WriteCode -from metagpt.llm import LLM from metagpt.logs import logger from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE @@ -16,9 +18,12 @@ from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE @pytest.mark.asyncio async def test_write_code(): api_design = "设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。" - write_code = WriteCode("write_code") + conf = Config() + cost_manager = CostManager(conf.runtime_options) + llm = LLM(options=conf.runtime_options, cost_manager=cost_manager) + write_code = WriteCode(options=conf.runtime_options, name="write_code", llm=llm) - code = await write_code.run(api_design) + code = await write_code.run(api_design, "filename") logger.info(code) # 我们不能精确地预测生成的代码，但我们可以检查某些关键字 @@ -29,6 +34,7 @@ async def test_write_code(): @pytest.mark.asyncio async def test_write_code_directly(): prompt = WRITE_CODE_PROMPT_SAMPLE + '\n' + TASKS_2[0] - llm = LLM() + options = Config().runtime_options + llm = LLM(options=options, cost_manager=CostManager(options=options)) rsp = await llm.aask(prompt) logger.info(rsp) diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index 62a3a2361..457e665fa 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -1,8 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : unittest of `metagpt/memory/longterm_memory.py` - -from metagpt.config import CONFIG +""" +@Desc : unittest of `metagpt/memory/longterm_memory.py` +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" +from metagpt.config import Config from metagpt.schema import Message from metagpt.actions import BossRequirement from metagpt.roles.role import RoleContext @@ -10,12 +12,13 @@ from metagpt.memory import LongTermMemory def test_ltm_search(): - assert hasattr(CONFIG, "long_term_memory") is True - openai_api_key = CONFIG.openai_api_key + conf = Config() + assert hasattr(conf, "long_term_memory") is True + openai_api_key = conf.openai_api_key assert len(openai_api_key) > 20 role_id = 'UTUserLtm(Product Manager)' - rc = RoleContext(watch=[BossRequirement]) + rc = RoleContext(options=conf.runtime_options, watch=[BossRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) @@ -23,19 +26,19 @@ def test_ltm_search(): message = Message(role='BOSS', content=idea, cause_by=BossRequirement) news = ltm.remember([message]) assert len(news) == 1 - ltm.add(message) + ltm.add(message, **conf.runtime_options) sim_idea = 'Write a game of cli snake' sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) news = ltm.remember([sim_message]) assert len(news) == 0 - ltm.add(sim_message) + ltm.add(sim_message, **conf.runtime_options) new_idea = 'Write a 2048 web game' new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) news = ltm.remember([new_message]) assert len(news) == 1 - ltm.add(new_message) + ltm.add(new_message, **conf.runtime_options) # restore from local index ltm_new = LongTermMemory() diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index a0f1f6257..d10c93ec0 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -4,14 +4,17 @@ @Time : 2023/5/12 00:47 @Author : alexanderwu @File : test_environment.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. + """ import pytest from metagpt.actions import BossRequirement +from metagpt.config import Config from metagpt.environment import Environment from metagpt.logs import logger -from metagpt.manager import Manager +from metagpt.provider.openai_api import CostManager from metagpt.roles import Architect, ProductManager, Role from metagpt.schema import Message @@ -22,33 +25,45 @@ def env(): def test_add_role(env: Environment): - role = ProductManager("Alice", "product manager", "create a new product", "limited resources") + conf = Config() + cost_manager = CostManager(options=conf.runtime_options) + role = ProductManager(options=conf.runtime_options, + cost_manager=cost_manager, + name="Alice", + profile="product manager", + goal="create a new product", + constraints="limited resources") env.add_role(role) assert env.get_role(role.profile) == role def test_get_roles(env: Environment): - role1 = Role("Alice", "product manager", "create a new product", "limited resources") - role2 = Role("Bob", "engineer", "develop the new product", "short deadline") + conf = Config() + cost_manager = CostManager(options=conf.runtime_options) + role1 = Role(options=conf.runtime_options, cost_manager=cost_manager, name="Alice", profile="product manager", + goal="create a new product", constraints="limited resources") + role2 = Role(options=conf.runtime_options, cost_manager=cost_manager, name="Bob", profile="engineer", + goal="develop the new product", constraints="short deadline") env.add_role(role1) env.add_role(role2) roles = env.get_roles() assert roles == {role1.profile: role1, role2.profile: role2} -def test_set_manager(env: Environment): - manager = Manager() - env.set_manager(manager) - assert env.manager == manager - - @pytest.mark.asyncio async def test_publish_and_process_message(env: Environment): - product_manager = ProductManager("Alice", "Product Manager", "做AI Native产品", "资源有限") - architect = Architect("Bob", "Architect", "设计一个可用、高效、较低成本的系统，包括数据结构与接口", "资源有限，需要节省成本") + conf = Config() + cost_manager = CostManager(options=conf.runtime_options) + product_manager = ProductManager(options=conf.runtime_options, + cost_manager=cost_manager, + name="Alice", profile="Product Manager", + goal="做AI Native产品", constraints="资源有限") + architect = Architect(options=conf.runtime_options, + cost_manager=cost_manager, + name="Bob", profile="Architect", goal="设计一个可用、高效、较低成本的系统，包括数据结构与接口", + constraints="资源有限，需要节省成本") env.add_roles([product_manager, architect]) - env.set_manager(Manager()) env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement)) await env.run(k=2) diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 11503af1d..77de6df0c 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -4,16 +4,19 @@ @Time : 2023/5/11 14:45 @Author : alexanderwu @File : test_llm.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import pytest -from metagpt.llm import LLM +from metagpt.config import Config +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager @pytest.fixture() def llm(): - return LLM() + options = Config().runtime_options + return LLM(options=options, cost_manager=CostManager(options)) @pytest.mark.asyncio diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index a7fe063a6..35ccdf78b 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -4,11 +4,13 @@ @Time : 2023/5/2 17:46 @Author : alexanderwu @File : test_search_engine.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from __future__ import annotations import pytest +from metagpt.config import Config from metagpt.logs import logger from metagpt.tools import SearchEngineType from metagpt.tools.search_engine import SearchEngine @@ -37,9 +39,10 @@ class MockSearchEnine: ], ) -async def test_search_engine(search_engine_typpe, run_func, max_results, as_string, ): - search_engine = SearchEngine(search_engine_typpe, run_func) - rsp = await search_engine.run("metagpt", max_results=max_results, as_string=as_string) +async def test_search_engine(search_engine_typpe, run_func, max_results, as_string): + conf = Config() + search_engine = SearchEngine(options=conf.runtime_options, engine=search_engine_typpe, run_func=run_func) + rsp = await search_engine.run(query="metagpt", max_results=max_results, as_string=as_string) logger.info(rsp) if as_string: assert isinstance(rsp, str) diff --git a/tests/metagpt/tools/test_web_browser_engine.py b/tests/metagpt/tools/test_web_browser_engine.py index b08d0ca10..283633bd6 100644 --- a/tests/metagpt/tools/test_web_browser_engine.py +++ b/tests/metagpt/tools/test_web_browser_engine.py @@ -1,5 +1,10 @@ +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + import pytest +from metagpt.config import Config from metagpt.tools import WebBrowserEngineType, web_browser_engine @@ -13,7 +18,8 @@ from metagpt.tools import WebBrowserEngineType, web_browser_engine ids=["playwright", "selenium"], ) async def test_scrape_web_page(browser_type, url, urls): - browser = web_browser_engine.WebBrowserEngine(browser_type) + conf = Config() + browser = web_browser_engine.WebBrowserEngine(options=conf.runtime_options, engine=browser_type) result = await browser.run(url) assert isinstance(result, str) assert "深度赋智" in result diff --git a/tests/metagpt/tools/test_web_browser_engine_playwright.py b/tests/metagpt/tools/test_web_browser_engine_playwright.py index 69e1339e7..add2b2f63 100644 --- a/tests/metagpt/tools/test_web_browser_engine_playwright.py +++ b/tests/metagpt/tools/test_web_browser_engine_playwright.py @@ -1,6 +1,10 @@ +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + import pytest -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.tools import web_browser_engine_playwright @@ -15,22 +19,24 @@ from metagpt.tools import web_browser_engine_playwright ids=["chromium-normal", "firefox-normal", "webkit-normal"], ) async def test_scrape_web_page(browser_type, use_proxy, kwagrs, url, urls, proxy, capfd): + conf = Config() + global_proxy = conf.global_proxy try: - global_proxy = CONFIG.global_proxy if use_proxy: - CONFIG.global_proxy = proxy - browser = web_browser_engine_playwright.PlaywrightWrapper(browser_type, **kwagrs) + conf.global_proxy = proxy + browser = web_browser_engine_playwright.PlaywrightWrapper(options=conf.runtime_options, + browser_type=browser_type, **kwagrs) result = await browser.run(url) result = result.inner_text assert isinstance(result, str) - assert "Deepwisdom" in result + assert "DeepWisdom" in result if urls: results = await browser.run(url, *urls) assert isinstance(results, list) assert len(results) == len(urls) + 1 - assert all(("Deepwisdom" in i) for i in results) + assert all(("DeepWisdom" in i) for i in results) if use_proxy: assert "Proxy:" in capfd.readouterr().out finally: - CONFIG.global_proxy = global_proxy + conf.global_proxy = global_proxy diff --git a/tests/metagpt/tools/test_web_browser_engine_selenium.py b/tests/metagpt/tools/test_web_browser_engine_selenium.py index ce322f7bd..278c35c91 100644 --- a/tests/metagpt/tools/test_web_browser_engine_selenium.py +++ b/tests/metagpt/tools/test_web_browser_engine_selenium.py @@ -1,6 +1,10 @@ +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + import pytest -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.tools import web_browser_engine_selenium @@ -15,11 +19,12 @@ from metagpt.tools import web_browser_engine_selenium ids=["chrome-normal", "firefox-normal", "edge-normal"], ) async def test_scrape_web_page(browser_type, use_proxy, url, urls, proxy, capfd): + conf = Config() + global_proxy = conf.global_proxy try: - global_proxy = CONFIG.global_proxy if use_proxy: - CONFIG.global_proxy = proxy - browser = web_browser_engine_selenium.SeleniumWrapper(browser_type) + conf.global_proxy = proxy + browser = web_browser_engine_selenium.SeleniumWrapper(options=conf.runtime_options, browser_type=browser_type) result = await browser.run(url) result = result.inner_text assert isinstance(result, str) @@ -33,4 +38,4 @@ async def test_scrape_web_page(browser_type, use_proxy, url, urls, proxy, capfd) if use_proxy: assert "Proxy:" in capfd.readouterr().out finally: - CONFIG.global_proxy = global_proxy + conf.global_proxy = global_proxy diff --git a/tests/metagpt/utils/test_config.py b/tests/metagpt/utils/test_config.py index 475bac22b..510892c2f 100644 --- a/tests/metagpt/utils/test_config.py +++ b/tests/metagpt/utils/test_config.py @@ -4,7 +4,7 @@ @Time : 2023/5/1 11:19 @Author : alexanderwu @File : test_config.py -@Modified By: mashenquan, 2013/8/20, add `test_options` +@Modified By: mashenquan, 2013/8/20, Add `test_options`; remove global configuration `CONFIG`, enable configuration support for business isolation. """ from pathlib import Path @@ -13,12 +13,6 @@ import pytest from metagpt.config import Config -def test_config_class_is_singleton(): - config_1 = Config() - config_2 = Config() - assert config_1 == config_2 - - def test_config_class_get_key_exception(): with pytest.raises(Exception) as exc_info: config = Config() @@ -27,16 +21,15 @@ def test_config_class_get_key_exception(): def test_config_yaml_file_not_exists(): - config = Config('wtf.yaml') with pytest.raises(Exception) as exc_info: - config.get('OPENAI_BASE_URL') - assert str(exc_info.value) == "Key 'OPENAI_BASE_URL' not found in environment variables or in the YAML file" + Config(Path('wtf.yaml')) + assert str(exc_info.value) == "Set OPENAI_API_KEY or Anthropic_API_KEY first" def test_options(): filename = Path(__file__).resolve().parent.parent.parent.parent / "config/config.yaml" config = Config(filename) - opts = config.options + opts = config.runtime_options assert opts From 88da7aa76145b9dd01e9d26f60afeebd3bc1ec5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 19:23:05 +0800 Subject: [PATCH 059/398] feat: +skill meta data decorator --- metagpt/learn/skill_metadata.py | 25 +++++++++++++++++++++++++ metagpt/learn/text_to_embedding.py | 4 ++++ metagpt/learn/text_to_image.py | 4 ++++ metagpt/learn/text_to_speech.py | 4 ++++ 4 files changed, 37 insertions(+) create mode 100644 metagpt/learn/skill_metadata.py diff --git a/metagpt/learn/skill_metadata.py b/metagpt/learn/skill_metadata.py new file mode 100644 index 000000000..6a13d6274 --- /dev/null +++ b/metagpt/learn/skill_metadata.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/20 +@Author : mashenquan +@File : skill_metadata.py +@Desc : Defines metadata for the `skill`. + Depending on the context and specific circumstances, skills may have different effects. + For example: + Proprietor: "Skill of the proprietor entity."（所有者的技能） + Holder: "Skill of the holder entity."（持有者的技能） + Possessor: "Skill of the possessor entity."（拥有者的技能） + Controller: "Skill of the controller entity."（控制者的技能） + Owner: "Skill of the owner entity."（所有者的技能） +""" + + +def skill_metadata(name, description, requisite): + def decorator(func): + func.skill_name = name + func.skill_description = description + func.skill_requisite = requisite + return func + + return decorator diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py index 281815ca6..38fd7c0cb 100644 --- a/metagpt/learn/text_to_embedding.py +++ b/metagpt/learn/text_to_embedding.py @@ -8,10 +8,14 @@ """ import os +from metagpt.learn.skill_metadata import skill_metadata from metagpt.tools.openai_text_to_embedding import oas3_openai_text_to_embedding from metagpt.utils.common import initialize_environment +@skill_metadata(name="Text to Embedding", + description="Convert the text into embeddings.", + requisite="`OPENAI_API_KEY`") def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): """Text to embedding diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 0932dfe07..d123e116a 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -8,11 +8,15 @@ """ import os +from metagpt.learn.skill_metadata import skill_metadata from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image from metagpt.tools.openai_text_to_image import oas3_openai_text_to_image from metagpt.utils.common import initialize_environment +@skill_metadata(name="Text to image", + description="Create a drawing based on the text.", + requisite="`OPENAI_API_KEY` or `METAGPT_TEXT_TO_IMAGE_MODEL`") def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url=""): """Text to image diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 1b81097b8..5631ef45e 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -8,10 +8,14 @@ """ import os +from metagpt.learn.skill_metadata import skill_metadata from metagpt.tools.azure_tts import oas3_azsure_tts from metagpt.utils.common import initialize_environment +@skill_metadata(name="Text to speech", + description="Text-to-speech", + requisite="`AZURE_TTS_SUBSCRIPTION_KEY` and `AZURE_TTS_REGION`") def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", subscription_key="", region=""): """Text to speech From c41f16e7bc58a3df13f04cdf000f4d41c580df76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 19:24:10 +0800 Subject: [PATCH 060/398] feat: +skill meta data decorator --- metagpt/learn/skill_metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/learn/skill_metadata.py b/metagpt/learn/skill_metadata.py index 6a13d6274..dea5fb04d 100644 --- a/metagpt/learn/skill_metadata.py +++ b/metagpt/learn/skill_metadata.py @@ -7,11 +7,11 @@ @Desc : Defines metadata for the `skill`. Depending on the context and specific circumstances, skills may have different effects. For example: - Proprietor: "Skill of the proprietor entity."（所有者的技能） - Holder: "Skill of the holder entity."（持有者的技能） - Possessor: "Skill of the possessor entity."（拥有者的技能） - Controller: "Skill of the controller entity."（控制者的技能） - Owner: "Skill of the owner entity."（所有者的技能） + Proprietor: "Skill of the proprietor entity." + Holder: "Skill of the holder entity." + Possessor: "Skill of the possessor entity." + Controller: "Skill of the controller entity." + Owner: "Skill of the owner entity." """ From ae94b6dff8e10cb65450bd05a8acf14ee24a169d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 20:22:59 +0800 Subject: [PATCH 061/398] feat: merge role_option --- metagpt/actions/write_teaching_plan.py | 4 ++-- metagpt/roles/fork_meta_role.py | 11 +++++++---- metagpt/roles/role.py | 1 + metagpt/roles/teacher.py | 6 +++--- .../metagpt/actions/test_write_teaching_plan.py | 8 +++++--- tests/metagpt/roles/test_fork_meta_role.py | 6 +++++- tests/metagpt/roles/test_teacher.py | 17 +++++++++++------ 7 files changed, 34 insertions(+), 19 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 3718c9801..53371b5a1 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -20,7 +20,7 @@ class TeachingPlanRequirement(Action): class WriteTeachingPlanPart(Action): """Write Teaching Plan Part""" - def __init__(self, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): + def __init__(self, options, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): """ :param name: action name @@ -29,7 +29,7 @@ class WriteTeachingPlanPart(Action): :param topic: topic part of teaching plan :param language: A human language, such as Chinese, English, French, etc. """ - super().__init__(name, context, llm) + super().__init__(options, name, context, llm) self.topic = topic self.language = language self.rsp = None diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py index 555bc8cf3..c21d08e37 100644 --- a/metagpt/roles/fork_meta_role.py +++ b/metagpt/roles/fork_meta_role.py @@ -26,14 +26,16 @@ from metagpt.schema import Message class ForkMetaRole(Role): """A `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file""" - def __init__(self, options, **kwargs): + def __init__(self, runtime_options, cost_manager, role_options, **kwargs): """Initialize a `fork` style meta role - :param options: pattern yaml file data + :param runtime_options: System configuration + :param cost_manager: Cost manager + :param role_options: pattern yaml file data :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` """ - opts = UMLMetaRoleOptions(**options) + opts = UMLMetaRoleOptions(**role_options) global_variables = { "name": Role.format_value(opts.name, kwargs), "profile": Role.format_value(opts.profile, kwargs), @@ -47,6 +49,8 @@ class ForkMetaRole(Role): global_variables[k] = v super(ForkMetaRole, self).__init__( + options=runtime_options, + cost_manager=cost_manager, name=global_variables["name"], profile=global_variables["profile"], goal=global_variables["goal"], @@ -54,7 +58,6 @@ class ForkMetaRole(Role): desc=global_variables["desc"], **kwargs ) - self.options = options actions = [] for m in opts.actions: for k, v in m.items(): diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 5397893eb..00f8ed45f 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -173,6 +173,7 @@ class Role: """Return number of action""" return len(self._actions) + @property def options(self): return self._options diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index 24ede7402..f29f384db 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -20,13 +20,13 @@ import re class Teacher(Role): """Support configurable teacher roles, with native and teaching languages being replaceable through configurations.""" - def __init__(self, name='Lily', profile='{teaching_language} Teacher', + def __init__(self, options, name='Lily', profile='{teaching_language} Teacher', goal='writing a {language} teaching plan part by part', constraints='writing in {language}', desc="", *args, **kwargs): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) + super().__init__(options=options, name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) actions = [] for topic in WriteTeachingPlanPart.TOPICS: - act = WriteTeachingPlanPart(topic=topic, llm=self._llm) + act = WriteTeachingPlanPart(options=options, topic=topic, llm=self._llm) actions.append(act) self._init_actions(actions) self._watch({TeachingPlanRequirement}) diff --git a/tests/metagpt/actions/test_write_teaching_plan.py b/tests/metagpt/actions/test_write_teaching_plan.py index 299a89639..6754fe88c 100644 --- a/tests/metagpt/actions/test_write_teaching_plan.py +++ b/tests/metagpt/actions/test_write_teaching_plan.py @@ -12,12 +12,13 @@ from pydantic import BaseModel from langchain.llms.base import LLM from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart +from metagpt.config import Config from metagpt.schema import Message class MockWriteTeachingPlanPart(WriteTeachingPlanPart): - def __init__(self, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): - super().__init__(name, context, llm, topic, language) + def __init__(self, options, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): + super().__init__(options, name, context, llm, topic, language) async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> str: return f"{WriteTeachingPlanPart.DATA_BEGIN_TAG}\nprompt\n{WriteTeachingPlanPart.DATA_END_TAG}" @@ -47,7 +48,8 @@ async def mock_write_teaching_plan_part(): for i in inputs: seed = Inputs(**i) - act = MockWriteTeachingPlanPart(name=seed.name, topic=seed.topic, language=seed.language) + options = Config().runtime_options + act = MockWriteTeachingPlanPart(options=options, name=seed.name, topic=seed.topic, language=seed.language) await act.run([Message(content="")]) assert act.topic == seed.topic assert str(act) == seed.topic diff --git a/tests/metagpt/roles/test_fork_meta_role.py b/tests/metagpt/roles/test_fork_meta_role.py index b2659330d..355197234 100644 --- a/tests/metagpt/roles/test_fork_meta_role.py +++ b/tests/metagpt/roles/test_fork_meta_role.py @@ -9,6 +9,8 @@ from typing import Dict from pydantic import BaseModel +from metagpt.config import Config +from metagpt.provider.openai_api import CostManager from metagpt.roles.fork_meta_role import ForkMetaRole @@ -79,7 +81,9 @@ def test_creat_role(): "teaching_language": "AA", "language": "BB" } - role = ForkMetaRole(seed.role, **kwargs) + runtime_options = Config().runtime_options + cost_manager = CostManager(options=runtime_options) + role = ForkMetaRole(runtime_options=runtime_options, cost_manager=cost_manager, role_options=seed.role, **kwargs) assert role.action_count == 2 assert "{" not in role.profile assert "{" not in role.goal diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 5faa43455..11c268edb 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -9,6 +9,8 @@ from typing import Dict, Optional from pydantic import BaseModel +from metagpt.config import Config +from metagpt.provider.openai_api import CostManager from metagpt.roles.teacher import Teacher @@ -42,22 +44,25 @@ def test_init(): }, { "name": "Lily{language}", - "expect_name": "LilyChinese", + "expect_name": "Lily{language}", "profile": "X {teaching_language}", - "expect_profile": "X English", + "expect_profile": "X {teaching_language}", "goal": "Do {something_big}, {language}", - "expect_goal": "Do {something_big}, Chinese", + "expect_goal": "Do {something_big}, {language}", "constraints": "Do in {key1}, {language}", - "expect_constraints": "Do in {key1}, Chinese", + "expect_constraints": "Do in {key1}, {language}", "kwargs": {}, "desc": "aaa{language}", - "expect_desc": "aaaChinese" + "expect_desc": "aaa{language}" }, ] for i in inputs: seed = Inputs(**i) - teacher = Teacher(name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, + options = Config().runtime_options + cost_manager = CostManager(options=options) + teacher = Teacher(options=options, cost_manager=cost_manager, name=seed.name, profile=seed.profile, + goal=seed.goal, constraints=seed.constraints, desc=seed.desc, **seed.kwargs) assert teacher.name == seed.expect_name assert teacher.desc == seed.expect_desc From 86e0e706191dc8822d1ed183108fc6546175d16b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 21 Aug 2023 10:44:40 +0800 Subject: [PATCH 062/398] fixbug: teacher role --- .../fork_meta_role_write_teaching_plan.py | 5 +++- examples/write_teaching_plan.py | 2 +- metagpt/actions/meta_action.py | 25 +++++++++++-------- metagpt/roles/fork_meta_role.py | 8 +++--- metagpt/roles/uml_meta_role_factory.py | 2 +- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/examples/fork_meta_role_write_teaching_plan.py b/examples/fork_meta_role_write_teaching_plan.py index d2898605e..e529a9b46 100644 --- a/examples/fork_meta_role_write_teaching_plan.py +++ b/examples/fork_meta_role_write_teaching_plan.py @@ -90,8 +90,11 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * configs = yaml.safe_load(reader) startup_config = ProjectConfig(**configs) - roles = UMLMetaRoleFactory.create_roles(startup_config.roles, **kwargs) company = SoftwareCompany() + roles = UMLMetaRoleFactory.create_roles(role_configs=startup_config.roles, + options=company.options, + cost_manager=company.cost_manager, + **kwargs) company.hire(roles) company.invest(startup_config.startup.investment) company.start_project(lesson, role=startup_config.startup.role, diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index 9874d10a5..6ab5edce4 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -77,7 +77,7 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * lesson = demo_lesson company = SoftwareCompany() - company.hire([Teacher(*args, **kwargs)]) + company.hire([Teacher(options=company.options, cost_manager=company.cost_manager, *args, **kwargs)]) company.invest(investment) company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) await company.run(n_round=1) diff --git a/metagpt/actions/meta_action.py b/metagpt/actions/meta_action.py index 3f01b8c0f..4c52e7cfd 100644 --- a/metagpt/actions/meta_action.py +++ b/metagpt/actions/meta_action.py @@ -21,19 +21,22 @@ from metagpt.schema import Message class MetaAction(Action): - def __init__(self, options: MetaActionOptions, llm=None, **kwargs): - super(MetaAction, self).__init__(options.name, kwargs.get("context"), llm=llm) - self.prompt = options.format_prompt(**kwargs) - self.options = options + def __init__(self, options, action_options: MetaActionOptions, llm=None, **kwargs): + super(MetaAction, self).__init__(options=options, + name=action_options.name, + context=kwargs.get("context"), + llm=llm) + self.prompt = action_options.format_prompt(**kwargs) + self.action_options = action_options self.kwargs = kwargs def __str__(self): """Return `topic` value when str()""" - return self.options.topic + return self.action_options.topic def __repr__(self): """Show `topic` value when debug""" - return self.options.topic + return self.action_options.topic async def run(self, messages, *args, **kwargs): if len(messages) < 1 or not isinstance(messages[0], Message): @@ -46,11 +49,11 @@ class MetaAction(Action): return self.rsp def _set_result(self, rsp): - if self.options.rsp_begin_tag and self.options.rsp_begin_tag in rsp: - ix = rsp.index(self.options.rsp_begin_tag) - rsp = rsp[ix + len(self.options.rsp_begin_tag):] - if self.options.rsp_end_tag and self.options.rsp_end_tag in rsp: - ix = rsp.index(self.options.rsp_end_tag) + if self.action_options.rsp_begin_tag and self.action_options.rsp_begin_tag in rsp: + ix = rsp.index(self.action_options.rsp_begin_tag) + rsp = rsp[ix + len(self.action_options.rsp_begin_tag):] + if self.action_options.rsp_end_tag and self.action_options.rsp_end_tag in rsp: + ix = rsp.index(self.action_options.rsp_end_tag) rsp = rsp[0:ix] self.rsp = rsp.strip() diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py index c21d08e37..5311bc4f0 100644 --- a/metagpt/roles/fork_meta_role.py +++ b/metagpt/roles/fork_meta_role.py @@ -26,10 +26,10 @@ from metagpt.schema import Message class ForkMetaRole(Role): """A `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file""" - def __init__(self, runtime_options, cost_manager, role_options, **kwargs): + def __init__(self, options, cost_manager, role_options, **kwargs): """Initialize a `fork` style meta role - :param runtime_options: System configuration + :param options: System configuration :param cost_manager: Cost manager :param role_options: pattern yaml file data :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` @@ -49,7 +49,7 @@ class ForkMetaRole(Role): global_variables[k] = v super(ForkMetaRole, self).__init__( - options=runtime_options, + options=options, cost_manager=cost_manager, name=global_variables["name"], profile=global_variables["profile"], @@ -70,7 +70,7 @@ class ForkMetaRole(Role): o = MetaActionOptions(**m) o.set_default_template(opts.templates[o.template_ix]) - act = MetaAction(options=o, llm=self._llm, **m) + act = MetaAction(options=options, action_options=o, llm=self._llm, **m) actions.append(act) self._init_actions(actions) requirement_types = set() diff --git a/metagpt/roles/uml_meta_role_factory.py b/metagpt/roles/uml_meta_role_factory.py index 78f9689a2..42071b0a6 100644 --- a/metagpt/roles/uml_meta_role_factory.py +++ b/metagpt/roles/uml_meta_role_factory.py @@ -33,7 +33,7 @@ class UMLMetaRoleFactory: raise NotImplementedError( f"{opt.role_type} is not implemented" ) - r = constructor(m, **kwargs) + r = constructor(role_options=m, **kwargs) roles.append(r) return roles From 58b1acf7b935ad0104fdc65a8133b7131de45dcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 21 Aug 2023 21:30:37 +0800 Subject: [PATCH 063/398] feat: +Message + tags --- metagpt/schema.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 27f5dd10c..4e6cba4ca 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -8,7 +8,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Type, TypedDict +from typing import Type, TypedDict, Set from pydantic import BaseModel @@ -29,6 +29,7 @@ class Message: cause_by: Type["Action"] = field(default="") sent_from: str = field(default="") send_to: str = field(default="") + tags: Set = field(default_factory=Set) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) From cf225320eb69ca2dfeca71730ec48022203f2faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 10:22:19 +0800 Subject: [PATCH 064/398] feat: +Message to __init__ --- metagpt/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metagpt/__init__.py b/metagpt/__init__.py index b9c530d24..7e0247553 100644 --- a/metagpt/__init__.py +++ b/metagpt/__init__.py @@ -3,3 +3,9 @@ # @Time : 2023/4/24 22:26 # @Author : alexanderwu # @File : __init__.py + +from metagpt.schema import Message + +__all__ = [ + "Message", +] From 5121472bd85e9cac565cc08bf5c763a00de522fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 10:23:42 +0800 Subject: [PATCH 065/398] feat: +Message to __init__ --- metagpt/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/__init__.py b/metagpt/__init__.py index 7e0247553..16359ca19 100644 --- a/metagpt/__init__.py +++ b/metagpt/__init__.py @@ -3,6 +3,7 @@ # @Time : 2023/4/24 22:26 # @Author : alexanderwu # @File : __init__.py +# @Desc : mashenquan, 2023/8/22. Add `Message` for importing by external projects. from metagpt.schema import Message From 148279401ee3c10b991df95f0a078e28f51a73ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 10:59:26 +0800 Subject: [PATCH 066/398] feat: Add tags to enable custom message classification --- metagpt/__init__.py | 10 ++++++---- metagpt/schema.py | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/metagpt/__init__.py b/metagpt/__init__.py index 16359ca19..2980109dd 100644 --- a/metagpt/__init__.py +++ b/metagpt/__init__.py @@ -1,9 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Time : 2023/4/24 22:26 -# @Author : alexanderwu -# @File : __init__.py -# @Desc : mashenquan, 2023/8/22. Add `Message` for importing by external projects. +""" +@Time : 2023/4/24 22:26 +@Author : alexanderwu +@File : __init__.py +@Desc : mashenquan, 2023/8/22. Add `Message` for importing by external projects. +""" from metagpt.schema import Message diff --git a/metagpt/schema.py b/metagpt/schema.py index 4e6cba4ca..749e0fd56 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -4,6 +4,7 @@ @Time : 2023/5/8 22:12 @Author : alexanderwu @File : schema.py +@Desc : mashenquan, 2023/8/22. Add tags to enable custom message classification. """ from __future__ import annotations @@ -29,7 +30,7 @@ class Message: cause_by: Type["Action"] = field(default="") sent_from: str = field(default="") send_to: str = field(default="") - tags: Set = field(default_factory=Set) + tags: Set = field(default_factory=set()) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) From 2adcefc298918101d7a50e2a785154ef69b96b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 11:04:29 +0800 Subject: [PATCH 067/398] feat: Add tags to enable custom message classification --- metagpt/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 749e0fd56..2e4a6c62f 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -9,7 +9,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Type, TypedDict, Set +from typing import Type, TypedDict, Set, Optional from pydantic import BaseModel @@ -30,7 +30,7 @@ class Message: cause_by: Type["Action"] = field(default="") sent_from: str = field(default="") send_to: str = field(default="") - tags: Set = field(default_factory=set()) + tags: Optional[Set] = field(default=None) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) From bc97b709bb17e7d25cc48f49632648ff5cb32624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 11:10:08 +0800 Subject: [PATCH 068/398] feat: Add tags to enable custom message classification --- metagpt/schema.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 2e4a6c62f..140f207c8 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -45,6 +45,16 @@ class Message: "content": self.content } + def add_tag(self, tag): + if self.tags is None: + self.tags = set() + self.tags.add(tag) + + def remove_tag(self, tag): + if self.tags is None: + return + self.tags.remove(tag) + @dataclass class UserMessage(Message): From a2e9797d4e7f7af85f43d6f8bf686181a93cc402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 11:13:08 +0800 Subject: [PATCH 069/398] feat: Add tags to enable custom message classification --- metagpt/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 140f207c8..0119f5bbb 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -51,7 +51,7 @@ class Message: self.tags.add(tag) def remove_tag(self, tag): - if self.tags is None: + if self.tags is None or tag not in self.tags: return self.tags.remove(tag) From 8eaf22dd62e47b4cc7611cd1b2fa2338a0af3ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 18:49:39 +0800 Subject: [PATCH 070/398] fixbug: role option, cost_manager argments --- metagpt/roles/customer_service.py | 4 +++- metagpt/roles/researcher.py | 4 +++- metagpt/roles/sales.py | 4 +++- metagpt/roles/seacher.py | 4 ++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/metagpt/roles/customer_service.py b/metagpt/roles/customer_service.py index 4aae7cb03..8550313d4 100644 --- a/metagpt/roles/customer_service.py +++ b/metagpt/roles/customer_service.py @@ -26,9 +26,11 @@ DESC = """ class CustomerService(Sales): def __init__( self, + options, + cost_manager, name="Xiaomei", profile="Human customer service", desc=DESC, store=None ): - super().__init__(name, profile, desc=desc, store=store) + super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, desc=desc, store=store) diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index acb46c718..6d8d072d9 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -22,6 +22,8 @@ class Report(BaseModel): class Researcher(Role): def __init__( self, + options, + cost_manager, name: str = "David", profile: str = "Researcher", goal: str = "Gather information and conduct research", @@ -29,7 +31,7 @@ class Researcher(Role): language: str = "en-us", **kwargs, ): - super().__init__(name, profile, goal, constraints, **kwargs) + super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, **kwargs) self._init_actions([CollectLinks(name), WebBrowseAndSummarize(name), ConductResearch(name)]) self.language = language if language not in ("en-us", "zh-cn"): diff --git a/metagpt/roles/sales.py b/metagpt/roles/sales.py index 51b13f487..35146fdc3 100644 --- a/metagpt/roles/sales.py +++ b/metagpt/roles/sales.py @@ -13,6 +13,8 @@ from metagpt.tools import SearchEngineType class Sales(Role): def __init__( self, + options, + cost_manager, name="Xiaomei", profile="Retail sales guide", desc="I am a sales guide in retail. My name is Xiaomei. I will answer some customer questions next, and I " @@ -23,7 +25,7 @@ class Sales(Role): "professional guide", store=None ): - super().__init__(name, profile, desc=desc) + super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, desc=desc) self._set_store(store) def _set_store(self, store): diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index c116ce98b..7b07ce713 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -13,9 +13,9 @@ from metagpt.tools import SearchEngineType class Searcher(Role): - def __init__(self, name='Alice', profile='Smart Assistant', goal='Provide search services for users', + def __init__(self, options, cost_manager, name='Alice', profile='Smart Assistant', goal='Provide search services for users', constraints='Answer is rich and complete', engine=SearchEngineType.SERPAPI_GOOGLE, **kwargs): - super().__init__(name, profile, goal, constraints, **kwargs) + super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, **kwargs) self._init_actions([SearchAndSummarize(engine=engine)]) def set_search_func(self, search_func): From 9600787d63b7575edac30e505cff503b5c95e424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 18:56:23 +0800 Subject: [PATCH 071/398] fixbug: role option, cost_manager argments --- metagpt/schema.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 0119f5bbb..f45d1e36d 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -9,6 +9,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from enum import StrEnum from typing import Type, TypedDict, Set, Optional from pydantic import BaseModel @@ -16,6 +17,10 @@ from pydantic import BaseModel from metagpt.logs import logger +class MessageTag(StrEnum): + Prerequisite = "prerequisite" + + class RawMessage(TypedDict): content: str role: str @@ -61,6 +66,7 @@ class UserMessage(Message): """便于支持OpenAI的消息 Facilitate support for OpenAI messages """ + def __init__(self, content: str): super().__init__(content, 'user') @@ -70,6 +76,7 @@ class SystemMessage(Message): """便于支持OpenAI的消息 Facilitate support for OpenAI messages """ + def __init__(self, content: str): super().__init__(content, 'system') @@ -79,6 +86,7 @@ class AIMessage(Message): """便于支持OpenAI的消息 Facilitate support for OpenAI messages """ + def __init__(self, content: str): super().__init__(content, 'assistant') From 19767496b16bd05119254c60215093a90c27a6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 19:47:35 +0800 Subject: [PATCH 072/398] =?UTF-8?q?feat:=20CostManager=E6=94=B9pydantic?= =?UTF-8?q?=E7=BB=93=E6=9E=84=EF=BC=8C=E4=BB=A5=E5=A4=87RPC=E4=BC=A0?= =?UTF-8?q?=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/provider/base_gpt_api.py | 7 +++++- metagpt/provider/openai_api.py | 32 ++++++++---------------- metagpt/schema.py | 4 +-- metagpt/software_company.py | 2 +- tests/metagpt/actions/test_write_code.py | 4 +-- tests/metagpt/test_environment.py | 6 ++--- tests/metagpt/test_llm.py | 2 +- 7 files changed, 26 insertions(+), 31 deletions(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index f39e708eb..f1590a77c 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -4,6 +4,7 @@ @Time : 2023/5/5 23:04 @Author : alexanderwu @File : base_gpt_api.py +@Desc : mashenquan, 2023/8/22. + try catch """ from abc import abstractmethod from typing import Optional @@ -41,7 +42,11 @@ class BaseGPTAPI(BaseChatbot): message = self._system_msgs(system_msgs) + [self._user_msg(msg)] else: message = [self._default_system_msg(), self._user_msg(msg)] - rsp = await self.acompletion_text(message, stream=True) + try: + rsp = await self.acompletion_text(message, stream=True) + except Exception as e: + logger.exception(f"{e}") + raise e logger.debug(message) # logger.debug(rsp) return rsp diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 2e951b36f..abfb796f3 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -8,10 +8,11 @@ """ import asyncio import time -from typing import NamedTuple +from typing import NamedTuple, Dict import openai from openai.error import APIConnectionError +from pydantic import BaseModel from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type from metagpt.logs import logger @@ -35,7 +36,7 @@ class RateLimiter: self.rpm = rpm def split_batches(self, batch): - return [batch[i : i + self.rpm] for i in range(0, len(batch), self.rpm)] + return [batch[i: i + self.rpm] for i in range(0, len(batch), self.rpm)] async def wait_if_needed(self, num_requests): current_time = time.time() @@ -56,14 +57,14 @@ class Costs(NamedTuple): total_budget: float -class CostManager: +class CostManager(BaseModel): """计算使用接口的开销""" - def __init__(self, options): - self.total_prompt_tokens = 0 - self.total_completion_tokens = 0 - self.options = options - self.total_budget = 0 + total_prompt_tokens: int = 0 + total_completion_tokens: int = 0 + total_budget: int = 0 + max_budget: int + total_cost: int = 0 def update_cost(self, prompt_tokens, completion_tokens, model): """ @@ -76,7 +77,8 @@ class CostManager: """ self.total_prompt_tokens += prompt_tokens self.total_completion_tokens += completion_tokens - cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model]["completion"]) / 1000 + cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model][ + "completion"]) / 1000 self.total_cost += cost logger.info( f"Total running cost: ${self.total_cost:.3f} | Max budget: ${self.max_budget:.3f} | " @@ -114,18 +116,6 @@ class CostManager: """获得所有开销""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) - @property - def total_cost(self): - return self.options.get("total_cost", 0) - - @total_cost.setter - def total_cost(self, v): - self.options["total_cost"] = v - - @property - def max_budget(self): - return self.options.get("max_budget", 0) - def log_and_reraise(retry_state): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") diff --git a/metagpt/schema.py b/metagpt/schema.py index f45d1e36d..56e9ad95c 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -9,7 +9,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from enum import StrEnum +from enum import Enum from typing import Type, TypedDict, Set, Optional from pydantic import BaseModel @@ -17,7 +17,7 @@ from pydantic import BaseModel from metagpt.logs import logger -class MessageTag(StrEnum): +class MessageTag(Enum): Prerequisite = "prerequisite" diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 3f6f484b4..87b24a1cb 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -30,7 +30,7 @@ class SoftwareCompany(BaseModel): investment: float = Field(default=10.0) idea: str = Field(default="") options: Dict = Field(default=Config().runtime_options) - cost_manager: CostManager = Field(default=CostManager(Config().runtime_options)) + cost_manager: CostManager = Field(default=CostManager(**Config().runtime_options)) class Config: arbitrary_types_allowed = True diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 04216ad7c..9861fd4cd 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -19,7 +19,7 @@ from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE async def test_write_code(): api_design = "设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。" conf = Config() - cost_manager = CostManager(conf.runtime_options) + cost_manager = CostManager(**conf.runtime_options) llm = LLM(options=conf.runtime_options, cost_manager=cost_manager) write_code = WriteCode(options=conf.runtime_options, name="write_code", llm=llm) @@ -35,6 +35,6 @@ async def test_write_code(): async def test_write_code_directly(): prompt = WRITE_CODE_PROMPT_SAMPLE + '\n' + TASKS_2[0] options = Config().runtime_options - llm = LLM(options=options, cost_manager=CostManager(options=options)) + llm = LLM(options=options, cost_manager=CostManager(**options)) rsp = await llm.aask(prompt) logger.info(rsp) diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index d10c93ec0..57650d145 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -26,7 +26,7 @@ def env(): def test_add_role(env: Environment): conf = Config() - cost_manager = CostManager(options=conf.runtime_options) + cost_manager = CostManager(**conf.runtime_options) role = ProductManager(options=conf.runtime_options, cost_manager=cost_manager, name="Alice", @@ -39,7 +39,7 @@ def test_add_role(env: Environment): def test_get_roles(env: Environment): conf = Config() - cost_manager = CostManager(options=conf.runtime_options) + cost_manager = CostManager(**conf.runtime_options) role1 = Role(options=conf.runtime_options, cost_manager=cost_manager, name="Alice", profile="product manager", goal="create a new product", constraints="limited resources") role2 = Role(options=conf.runtime_options, cost_manager=cost_manager, name="Bob", profile="engineer", @@ -53,7 +53,7 @@ def test_get_roles(env: Environment): @pytest.mark.asyncio async def test_publish_and_process_message(env: Environment): conf = Config() - cost_manager = CostManager(options=conf.runtime_options) + cost_manager = CostManager(**conf.runtime_options) product_manager = ProductManager(options=conf.runtime_options, cost_manager=cost_manager, name="Alice", profile="Product Manager", diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 77de6df0c..f61793151 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -16,7 +16,7 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager @pytest.fixture() def llm(): options = Config().runtime_options - return LLM(options=options, cost_manager=CostManager(options)) + return LLM(options=options, cost_manager=CostManager(**options)) @pytest.mark.asyncio From a7157d9e7a0d7c3cbf3a248e32d18ebec2c90fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 19:56:22 +0800 Subject: [PATCH 073/398] =?UTF-8?q?feat:=20CostManager=E6=94=B9pydantic?= =?UTF-8?q?=E7=BB=93=E6=9E=84=EF=BC=8C=E4=BB=A5=E5=A4=87RPC=E4=BC=A0?= =?UTF-8?q?=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/provider/openai_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index abfb796f3..f0b692f46 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -62,9 +62,9 @@ class CostManager(BaseModel): total_prompt_tokens: int = 0 total_completion_tokens: int = 0 - total_budget: int = 0 - max_budget: int - total_cost: int = 0 + total_budget: float = 0 + max_budget: float + total_cost: float = 0 def update_cost(self, prompt_tokens, completion_tokens, model): """ From 6e37e156de17254fccba4ff4dddba6e9e604f899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 21:13:24 +0800 Subject: [PATCH 074/398] fixbug: init action error --- metagpt/roles/researcher.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 6d8d072d9..30545c5c0 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -32,7 +32,10 @@ class Researcher(Role): **kwargs, ): super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, **kwargs) - self._init_actions([CollectLinks(name), WebBrowseAndSummarize(name), ConductResearch(name)]) + self._init_actions([ + CollectLinks(options=options, name=name), + WebBrowseAndSummarize(options=options, name=name), + ConductResearch(options=options, name=name)]) self.language = language if language not in ("en-us", "zh-cn"): logger.warning(f"The language `{language}` has not been tested, it may not work.") From 937bd12a63733d818338f7d3ad8c2d0907fe5c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 23 Aug 2023 13:02:23 +0800 Subject: [PATCH 075/398] feat: memory + tags --- metagpt/memory/memory.py | 8 ++++++++ metagpt/roles/role.py | 9 +++++++-- metagpt/schema.py | 7 +++++++ tests/metagpt/roles/test_teacher.py | 2 +- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 625d98675..1a8003fba 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -91,3 +91,11 @@ class Memory: key = class_names[type(action).__name__] rsp += self.index[key] return rsp + + def get_by_tags(self, tags: list) -> list[Message]: + """Return messages with specified tags""" + result = [] + for m in self.storage: + if m.is_contain_tags(tags): + result.append(m) + return result diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 00f8ed45f..217272b54 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -17,7 +17,7 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as LLM from metagpt.actions import Action, ActionOutput from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory -from metagpt.schema import Message +from metagpt.schema import Message, MessageTag PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -90,6 +90,11 @@ class RoleContext(BaseModel): def history(self) -> list[Message]: return self.memory.get() + @property + def prerequisite(self): + """Retrieve information with `prerequisite` tag""" + return self.memory.get_by_tags([MessageTag.Prerequisite.value]) + class Role: """Role/Proxy""" @@ -209,7 +214,7 @@ class Role: # history=self.history) logger.info(f"{self._setting}: ready to {self._rc.todo}") - requirement = self._rc.important_memory + requirement = self._rc.important_memory or self._rc.prerequisite response = await self._rc.todo.run(requirement, **self._options) # logger.info(response) if isinstance(response, ActionOutput): diff --git a/metagpt/schema.py b/metagpt/schema.py index 56e9ad95c..4c577fd7b 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -60,6 +60,13 @@ class Message: return self.tags.remove(tag) + def is_contain_tags(self, tags: list) -> bool: + """Determine whether the message contains tags.""" + if not tags or not self.tags: + return False + intersection = set(tags) & self.tags + return len(intersection) > 0 + @dataclass class UserMessage(Message): diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 11c268edb..8f673d6e0 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -60,7 +60,7 @@ def test_init(): for i in inputs: seed = Inputs(**i) options = Config().runtime_options - cost_manager = CostManager(options=options) + cost_manager = CostManager(**options) teacher = Teacher(options=options, cost_manager=cost_manager, name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, desc=seed.desc, **seed.kwargs) From 9395d9f7dc5ee0a8b1587ce74afd2798b0e098ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 23 Aug 2023 14:56:53 +0800 Subject: [PATCH 076/398] feat: Add options to Config.__init__ to support externally specified options. --- metagpt/config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 076bc5eb7..d8d772cd0 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -3,8 +3,9 @@ """ @Desc: Provide configuration, singleton. @Modified By: mashenquan, replace `CONFIG` with `os.environ` to support personal config -@Desc: `os.environ` doesn't support personalization, while `Config` does. + `os.environ` doesn't support personalization, while `Config` does. Hence, the parameter reading priority is `Config` first, and if not found, then `os.environ`. +@Modified By: mashenquan, 2023/8/23. Add `options` to `Config.__init__` to support externally specified options. """ import os @@ -43,10 +44,14 @@ class Config: key_yaml_file = PROJECT_ROOT / "config/key.yaml" default_yaml_file = PROJECT_ROOT / "config/config.yaml" - def __init__(self, yaml_file=default_yaml_file): + def __init__(self, yaml_file=default_yaml_file, options=None): self._configs = {} self._init_with_config_files_and_env(self._configs, yaml_file) + if options: + self._configs.update(options) + self._parse() + def _parse(self): logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") From 7dd02ae4b11a5494a470125f785e4acbf7406b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 23 Aug 2023 15:53:33 +0800 Subject: [PATCH 077/398] feat: A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. --- metagpt/roles/fork_meta_role.py | 5 ++++- metagpt/roles/researcher.py | 9 +++++++-- metagpt/roles/role.py | 8 +++++--- metagpt/roles/teacher.py | 7 +++++-- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py index 5311bc4f0..57d467080 100644 --- a/metagpt/roles/fork_meta_role.py +++ b/metagpt/roles/fork_meta_role.py @@ -10,6 +10,8 @@ For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file. +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. + """ import re @@ -82,12 +84,13 @@ class ForkMetaRole(Role): """Everything will be done part by part.""" if self._rc.todo is None: self._set_state(0) - return + return True if self._rc.state + 1 < len(self._states): self._set_state(self._rc.state + 1) else: self._rc.todo = None + return False async def _react(self) -> Message: ret = Message(content="") diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 30545c5c0..f3ff7f8e5 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -1,4 +1,8 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. + +""" import asyncio @@ -40,15 +44,16 @@ class Researcher(Role): if language not in ("en-us", "zh-cn"): logger.warning(f"The language `{language}` has not been tested, it may not work.") - async def _think(self) -> None: + async def _think(self) -> bool: if self._rc.todo is None: self._set_state(0) - return + return True if self._rc.state + 1 < len(self._states): self._set_state(self._rc.state + 1) else: self._rc.todo = None + return False async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 217272b54..493c172ae 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023-8-7, :class:`Role` + properties. @Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; Change cost control from global to company level. +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. """ from __future__ import annotations @@ -192,12 +193,12 @@ class Role: return self._setting.desc return PREFIX_TEMPLATE.format(**self._setting.dict()) - async def _think(self) -> None: - """思考要做什么，决定下一步的action""" + async def _think(self) -> bool: + """Consider what to do and decide on the next course of action. Return false if nothing can be done.""" if len(self._actions) == 1: # 如果只有一个动作，那就只能做这个 self._set_state(0) - return + return True prompt = self._get_prefix() prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1) @@ -207,6 +208,7 @@ class Role: logger.warning(f'Invalid answer of state, {next_state=}') next_state = "0" self._set_state(int(next_state)) + return True async def _act(self) -> Message: # prompt = self.get_prefix() diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index f29f384db..9a68fa9e0 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -4,6 +4,8 @@ @Time : 2023/7/27 @Author : mashenquan @File : teacher.py +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. + """ @@ -31,16 +33,17 @@ class Teacher(Role): self._init_actions(actions) self._watch({TeachingPlanRequirement}) - async def _think(self) -> None: + async def _think(self) -> bool: """Everything will be done part by part.""" if self._rc.todo is None: self._set_state(0) - return + return True if self._rc.state + 1 < len(self._states): self._set_state(self._rc.state + 1) else: self._rc.todo = None + return False async def _react(self) -> Message: ret = Message(content="") From 67f6fe652359f883a7e11281581260e5ffd8f21c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 23 Aug 2023 16:25:47 +0800 Subject: [PATCH 078/398] fixbug: _think return None --- metagpt/roles/teacher.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index 9a68fa9e0..d2a2198f5 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -41,9 +41,10 @@ class Teacher(Role): if self._rc.state + 1 < len(self._states): self._set_state(self._rc.state + 1) - else: - self._rc.todo = None - return False + return True + + self._rc.todo = None + return False async def _react(self) -> Message: ret = Message(content="") From 5f16d6e8534a0b0c2316211374290f9f084ac69a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 24 Aug 2023 15:22:29 +0800 Subject: [PATCH 079/398] feat: +text summarize --- metagpt/provider/openai_api.py | 55 +++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 3baf8d932..48b7991dc 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -9,7 +9,7 @@ import asyncio import time -from typing import NamedTuple +from typing import NamedTuple, List import traceback import openai from openai.error import APIConnectionError @@ -310,3 +310,56 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): @property def openai_api_version(self): return self._options.get("openai_api_version") + + async def get_summary(self, text: str, max_words=20): + """Generate text summary""" + language = self._options.get("language", "English") + command = f"Translate the above content into a {language} summary of less than {max_words} words." + msg = text + "\n\n" + command + logger.info(f"summary ask:{msg}") + response = await self.aask(msg=msg, system_msgs=[]) + logger.info(f"summary rsp: {response}") + return response + + async def get_context_title(self, text: str, max_token_count_per_ask=None, max_words=5) -> str: + """Generate text title""" + max_response_token_count = 50 + max_token_count = max_token_count_per_ask or self._options.get("MAX_TOKENS", 1500) + text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) + + summaries = [] + for ws in text_windows: + response = await self.get_summary(ws) + summaries.append(response) + + language = self._options.get("language", "English") + command = f"Translate the above summary into a {language} title of less than {max_words} words." + summaries.append(command) + msg = "\n".join(summaries) + logger.info(f"title ask:{msg}") + response = await self.aask(msg=msg, system_msgs=[]) + logger.info(f"title rsp: {response}") + return response + + @staticmethod + def split_texts(text: str, window_size) -> List[str]: + """Splitting long text into sliding windows text""" + total_len = len(text) + if total_len <= window_size: + return [text] + + padding_size = 20 if window_size > 20 else 0 + windows = [] + idx = 0 + while idx < total_len: + data_len = window_size - padding_size + if data_len + idx > total_len: + windows.append(text[idx:]) + break + w = text[idx:data_len] + windows.append(w) + for i in range(len(windows)): + if i + 1 == len(windows): + break + windows[i] += windows[i + 1][0:padding_size] + return windows From 799dbd396eeff71e9e5a7ab30935685b2794c9a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 25 Aug 2023 21:10:14 +0800 Subject: [PATCH 080/398] feat: archive --- .well-known/skills.yaml | 17 ++++ metagpt/actions/action.py | 2 +- metagpt/actions/action_output.py | 6 +- metagpt/actions/talk_action.py | 32 +++++++ metagpt/learn/skill_loader.py | 38 ++++++++ metagpt/memory/brain_memory.py | 47 ++++++++++ metagpt/provider/openai_api.py | 2 + metagpt/roles/assistant.py | 143 +++++++++++++++++++++++++++++++ metagpt/roles/role.py | 6 ++ metagpt/schema.py | 3 +- 10 files changed, 291 insertions(+), 5 deletions(-) create mode 100644 .well-known/skills.yaml create mode 100644 metagpt/actions/talk_action.py create mode 100644 metagpt/learn/skill_loader.py create mode 100644 metagpt/memory/brain_memory.py create mode 100644 metagpt/roles/assistant.py diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml new file mode 100644 index 000000000..5ccb8094b --- /dev/null +++ b/.well-known/skills.yaml @@ -0,0 +1,17 @@ +entities: + Assistant: + skills: + - name: text_to_speech + description: Text-to-speech + requisite: + - AZURE_TTS_SUBSCRIPTION_KEY + - AZURE_TTS_REGION + - name: text_to_image + description: Create a drawing based on the text. + requisite: + - OPENAI_API_KEY + - METAGPT_TEXT_TO_IMAGE_MODEL + - name: text_to_embedding + description: Convert the text into embeddings. + requisite: + - OPENAI_API_KEY diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 899c2515c..86a6664ba 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -62,6 +62,6 @@ class Action(ABC): instruct_content = output_class(**parsed_data) return ActionOutput(content, instruct_content) - async def run(self, *args, **kwargs): + async def run(self, *args, **kwargs) -> str | ActionOutput | None: """Run action""" raise NotImplementedError("The run method should be implemented in a subclass.") diff --git a/metagpt/actions/action_output.py b/metagpt/actions/action_output.py index c0b88dcf9..6c812e7fe 100644 --- a/metagpt/actions/action_output.py +++ b/metagpt/actions/action_output.py @@ -6,16 +6,16 @@ @File : action_output """ -from typing import Dict, Type +from typing import Dict, Type, Optional from pydantic import BaseModel, create_model, root_validator, validator class ActionOutput: content: str - instruct_content: BaseModel + instruct_content: Optional[BaseModel] = None - def __init__(self, content: str, instruct_content: BaseModel): + def __init__(self, content: str, instruct_content: BaseModel=None): self.content = content self.instruct_content = instruct_content diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py new file mode 100644 index 000000000..4275a1b9e --- /dev/null +++ b/metagpt/actions/talk_action.py @@ -0,0 +1,32 @@ +from metagpt.actions import Action, ActionOutput +from metagpt.logs import logger + + + +class TalkAction(Action): + def __init__(self, options, name: str = '', talk='', history_summary='', context=None, llm=None): + context = context or {} + context["talk"] = talk + context["history_summery"] = history_summary + super(TalkAction, self).__init__(options=options, name=name, context=context, llm=llm) + self._talk = talk + self._history_summary = history_summary + self._rsp = None + + @property + def prompt(self): + prompt = f"{self._history_summary}\n\n" + if self._history_summary != "": + prompt += "According to the historical conversation above, " + language = self.options.get("language", "Chinese") + prompt += f"Answer in {language}:\n {self._talk}" + return prompt + + async def run(self, *args, **kwargs) -> ActionOutput: + prompt = self.prompt + logger.info(prompt) + rsp = await self.llm.aask(msg=prompt, system_msgs=[]) + logger.info(rsp) + self._rsp = ActionOutput(content=rsp) + return self._rsp + diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py new file mode 100644 index 000000000..eeca12871 --- /dev/null +++ b/metagpt/learn/skill_loader.py @@ -0,0 +1,38 @@ +from pathlib import Path +from typing import List, Dict + +import yaml +from pydantic import BaseModel + + +class Skill(BaseModel): + name: str + description: str + requisite: List[str] + + +class EntitySkills(BaseModel): + skills: List[Skill] + + +class SkillsDeclaration(BaseModel): + entities: Dict[str, EntitySkills] + + +class SkillLoader: + def __init__(self): + skill_file_name = Path(__file__).parent.parent.parent / ".well-known/skills.yaml" + with open(str(skill_file_name), 'r') as file: + skills = yaml.safe_load(file) + self._skills = SkillsDeclaration(**skills) + + def get_skill_list(self, entity_name: str = "Assistant"): + if not self._skills or entity_name not in self._skills.entities: + return {} + entity_skills = self._skills.entities.get(entity_name) + + description_to_name_mappings = {} + for s in entity_skills.skills: + description_to_name_mappings[s.description] = s.name + + return description_to_name_mappings diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py new file mode 100644 index 000000000..97319859a --- /dev/null +++ b/metagpt/memory/brain_memory.py @@ -0,0 +1,47 @@ +from enum import Enum +from typing import List + +import pydantic + +from metagpt import Message + +class MessageType(Enum): + Talk = "TALK" + Solution = "SOLUTION" + Problem = "PROBLEM" + Skill = "SKILL" + Answer = "ANSWER" + + +class BrainMemory(pydantic.BaseModel): + history: List[Message] = [] + stack: List[Message] = [] + solution: List[Message] = [] + + + def add_talk(self, msg: Message): + msg.add_tag(MessageType.Talk.value) + self.history.append(msg) + + def add_answer(self, msg: Message): + msg.add_tag(MessageType.Answer.value) + self.history.append(msg) + + @property + def history_text(self): + if len(self.history) == 0: + return "" + texts = [m.content for m in self.history[:-1]] + return "\n".join(texts) + + def move_to_solution(self): + while len(self.history) > 1: + msg = self.history.pop() + self.solution.append(msg) + + @property + def last_talk(self): + if len(self.history) == 0 or not self.history[-1].is_contain_tags([MessageType.Talk.value]): + return "" + return self.history[-1].content + diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 48b7991dc..06a3154e8 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -313,6 +313,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def get_summary(self, text: str, max_words=20): """Generate text summary""" + if len(text) < max_words: + return text language = self._options.get("language", "English") command = f"Translate the above content into a {language} summary of less than {max_words} words." msg = text + "\n\n" + command diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py new file mode 100644 index 000000000..fde011892 --- /dev/null +++ b/metagpt/roles/assistant.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : fork_meta_role.py +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the + ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to + make these symbols configurable and standardized, making the process of building flows more convenient. + For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` + This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a + configuration file. +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. + +""" +import asyncio +import re + +from metagpt.actions import ActionOutput +from metagpt.actions.talk_action import TalkAction +from metagpt.config import Config +from metagpt.learn.skill_loader import SkillLoader +from metagpt.logs import logger +from metagpt.memory.brain_memory import BrainMemory, MessageType +from metagpt.provider.openai_api import CostManager +from metagpt.roles import Role +from metagpt.schema import Message + +DEFAULT_MAX_TOKENS = 1500 +COMMAND_TOKENS = 500 + + +class Assistant(Role): + """解决通用问题的助手""" + + def __init__(self, options, cost_manager, name="Lily", profile="An assistant", goal="Help to solve problem", + constraints="Talk in {language}", desc="", *args, **kwargs): + super(Assistant, self).__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, + goal=goal, constraints=constraints, desc=desc, *args, **kwargs) + self.memory = BrainMemory() + self.skills = SkillLoader() + + async def think(self) -> bool: + """Everything will be done part by part.""" + if self.memory.history_text != "": + self._refine_memory() + + + prompt = "" + history_text = self.memory.history_text + history_summary = "" + if history_text != "": + max_tokens = self.options.get("MAX_TOKENS", DEFAULT_MAX_TOKENS) + history_summary = await self._llm.get_summary(history_text, max_tokens - COMMAND_TOKENS) + prompt += history_summary + "\n\n" + prompt += "Analyze the conversation history above, in conjunction with the current sentence: \n{self.memory.last_talk}\n\n" + else: + prompt += f"Refer to this sentence:\n {self.memory.last_talk}\n" + skills = self.skills.get_skill_list() + for desc, name in skills.items(): + prompt += f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: text_to_image\n" + if history_text != "": + prompt += "If the last sentence is not related to the conversation history above, return `[SOLUTION]: {title of the history conversation}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" + prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" + prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" + prompt += "Otherwise, rewrite and return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" + logger.info(prompt) + rsp = await self._llm.aask(prompt, []) + logger.info(rsp) + return await self._plan(rsp, history_summary=history_summary) + + async def act(self) -> ActionOutput: + result = await self._rc.todo.run(**self._options) + if not result: + return None + if isinstance(result, str): + msg = Message(content=result) + output = ActionOutput(content=result) + else: + msg = Message(content=result.content, instruct_content=result.instruct_content, + cause_by=type(self._rc.todo)) + output = result + self.memory.add_answer(msg) + return output + + async def talk(self, text): + self.memory.add_talk(Message(content=text, tags=set([MessageType.Talk.value]))) + + async def _plan(self, rsp, **kwargs) -> bool: + skill, text = Assistant.extract_info(rsp) + handlers = { + MessageType.Talk.value: self.talk_handler, + MessageType.Problem.value: self.problem_handler, + MessageType.Solution.value: self.solution_handler, + MessageType.Skill.value: self.skill_handler, + } + handler = handlers.get(skill, self.talk_handler) + return await handler(text, **kwargs) + + @staticmethod + def extract_info(input_string): + pattern = r'\[([A-Z]+)\]:\s*(.+)' + match = re.match(pattern, input_string) + if match: + return match.group(1), match.group(2) + else: + return None, input_string + + async def problem_handler(self, text, **kwargs) -> bool: + action = TalkAction(options=self.options, talk=text, llm=self._llm, **kwargs) + self.add_to_do(action) + return True + + async def solution_handler(self, text, **kwargs) -> bool: + self.memory.move_to_solution() # 问题解决后及时清空内存 + action = TalkAction(options=self.options, talk=text, history_summary="", **kwargs) + self.add_to_do(action) + + async def skill_handler(self, text, **kwargs) -> bool: + pass + + async def _refine_memory(self): + + +async def main(): + options = Config().runtime_options + cost_manager = CostManager(**options) + topic = "dataiku vs. datarobot" + role = Assistant(options=options, cost_manager=cost_manager, language="Chinese") + await role.talk(topic) + while True: + has_action = await role.think() + if not has_action: + break + msg = await role.act() + print(msg) + # 获取用户终端输入 + talk = input("You: ") + await role.talk(talk) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 493c172ae..1bb73f884 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -320,3 +320,9 @@ class Role: for k, v in merged_opts.items(): value = value.replace("{" + f"{k}" + "}", str(v)) return value + + def add_action(self, act): + self._actions.append(act) + + def add_to_do(self, act): + self._rc.todo = act \ No newline at end of file diff --git a/metagpt/schema.py b/metagpt/schema.py index 4c577fd7b..e1cd011c6 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -10,7 +10,7 @@ from __future__ import annotations from dataclasses import dataclass, field from enum import Enum -from typing import Type, TypedDict, Set, Optional +from typing import Type, TypedDict, Set, Optional, List from pydantic import BaseModel @@ -98,6 +98,7 @@ class AIMessage(Message): super().__init__(content, 'assistant') + if __name__ == '__main__': test_content = 'test_message' msgs = [ From 1aeebc85fbba23d96bb8396775636123ac1b929b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 25 Aug 2023 21:54:28 +0800 Subject: [PATCH 081/398] feat: archive --- metagpt/provider/openai_api.py | 23 ++++++++++++ metagpt/roles/assistant.py | 64 ++++++++++++++-------------------- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 06a3154e8..510041e98 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -7,6 +7,7 @@ Change cost control from global to company level. """ import asyncio +import re import time from typing import NamedTuple, List @@ -333,6 +334,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): for ws in text_windows: response = await self.get_summary(ws) summaries.append(response) + if len(summaries) == 1: + return summaries[0] language = self._options.get("language", "English") command = f"Translate the above summary into a {language} title of less than {max_words} words." @@ -343,6 +346,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): logger.info(f"title rsp: {response}") return response + async def is_related(self, text1, text2): + command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + rsp = await self.aask(msg=command, system_msgs=[]) + result, _ = self.extract_info(rsp) + return result == "TRUE" + + async def rewrite(self, sentence: str, context: str): + command = f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + rsp = await self.aask(msg=command, system_msgs=[]) + return rsp + @staticmethod def split_texts(text: str, window_size) -> List[str]: """Splitting long text into sliding windows text""" @@ -365,3 +379,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): break windows[i] += windows[i + 1][0:padding_size] return windows + + @staticmethod + def extract_info(input_string): + pattern = r'\[([A-Z]+)\]:\s*(.+)' + match = re.match(pattern, input_string) + if match: + return match.group(1), match.group(2) + else: + return None, input_string \ No newline at end of file diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index fde011892..dfbd406bc 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -14,7 +14,7 @@ """ import asyncio -import re + from metagpt.actions import ActionOutput from metagpt.actions.talk_action import TalkAction @@ -42,32 +42,18 @@ class Assistant(Role): async def think(self) -> bool: """Everything will be done part by part.""" - if self.memory.history_text != "": - self._refine_memory() - - - prompt = "" - history_text = self.memory.history_text - history_summary = "" - if history_text != "": - max_tokens = self.options.get("MAX_TOKENS", DEFAULT_MAX_TOKENS) - history_summary = await self._llm.get_summary(history_text, max_tokens - COMMAND_TOKENS) - prompt += history_summary + "\n\n" - prompt += "Analyze the conversation history above, in conjunction with the current sentence: \n{self.memory.last_talk}\n\n" - else: - prompt += f"Refer to this sentence:\n {self.memory.last_talk}\n" + last_talk = await self.refine_memory() + prompt = f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): prompt += f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: text_to_image\n" - if history_text != "": - prompt += "If the last sentence is not related to the conversation history above, return `[SOLUTION]: {title of the history conversation}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" prompt += "Otherwise, rewrite and return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(rsp) - return await self._plan(rsp, history_summary=history_summary) + return await self._plan(rsp) async def act(self) -> ActionOutput: result = await self._rc.todo.run(**self._options) @@ -86,40 +72,42 @@ class Assistant(Role): async def talk(self, text): self.memory.add_talk(Message(content=text, tags=set([MessageType.Talk.value]))) - async def _plan(self, rsp, **kwargs) -> bool: - skill, text = Assistant.extract_info(rsp) + async def _plan(self, rsp: str, **kwargs) -> bool: + skill, text = Assistant.extract_info(input_string=rsp) handlers = { MessageType.Talk.value: self.talk_handler, - MessageType.Problem.value: self.problem_handler, - MessageType.Solution.value: self.solution_handler, + MessageType.Problem.value: self.talk_handler, MessageType.Skill.value: self.skill_handler, } handler = handlers.get(skill, self.talk_handler) return await handler(text, **kwargs) - @staticmethod - def extract_info(input_string): - pattern = r'\[([A-Z]+)\]:\s*(.+)' - match = re.match(pattern, input_string) - if match: - return match.group(1), match.group(2) - else: - return None, input_string - - async def problem_handler(self, text, **kwargs) -> bool: + async def talk_handler(self, text, **kwargs) -> bool: action = TalkAction(options=self.options, talk=text, llm=self._llm, **kwargs) self.add_to_do(action) return True - async def solution_handler(self, text, **kwargs) -> bool: - self.memory.move_to_solution() # 问题解决后及时清空内存 - action = TalkAction(options=self.options, talk=text, history_summary="", **kwargs) - self.add_to_do(action) - async def skill_handler(self, text, **kwargs) -> bool: + skill = pass - async def _refine_memory(self): + async def refine_memory(self) -> str: + history_text = self.memory.history_text + last_talk = self.memory.last_talk + if history_text == "": + return last_talk + history_summary = await self._llm.get_context_title(history_text, max_words=20) + if await self._llm.is_related(last_talk, history_summary): # 合并相关内容 + last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) + return last_talk + + self.memory.move_to_solution() # 问题解决后及时清空内存 + return last_talk + + @staticmethod + def extract_info(input_string): + from metagpt.provider.openai_api import OpenAIGPTAPI + return OpenAIGPTAPI.extract_info(input_string) async def main(): From 0821e6d0996d886546d9134f7bc62f35162dddb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 10:21:49 +0800 Subject: [PATCH 082/398] feat: + RateLimitError retry --- metagpt/provider/openai_api.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 510041e98..e98acbd75 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -9,6 +9,7 @@ import asyncio import re import time +import random from typing import NamedTuple, List import traceback @@ -152,15 +153,25 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self.rpm = int(self._options.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: - try: - response = await openai.ChatCompletion.acreate( - **self._cons_kwargs(messages), - stream=True - ) - except Exception as e: - error_str = traceback.format_exc() - logger.error(f"Exception:{e}, stack:{error_str}") - raise e + max_try = 5 + response = None + for i in range(max_try): + try: + response = await openai.ChatCompletion.acreate( + **self._cons_kwargs(messages), + stream=True + ) + break + except openai.error.RateLimitError as e: + random_time = random.uniform(0, 3) # 生成0到5秒之间的随机时间 + rounded_time = round(random_time, 1) # 保留一位小数，以实现0.1秒的精度 + logger.warning(f"Exception:{e}, sleeping for {rounded_time} seconds") + await asyncio.sleep(rounded_time) + continue + except Exception as e: + error_str = traceback.format_exc() + logger.error(f"Exception:{e}, stack:{error_str}") + raise e # create variables to collect the stream of chunks collected_chunks = [] From 4fe3d6e8790f17d01ab059b5fb5d01d02328540a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 16:52:21 +0800 Subject: [PATCH 083/398] fixbug: unit test --- metagpt/actions/skill_action.py | 0 metagpt/tools/metagpt_text_to_image.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 metagpt/actions/skill_action.py diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py new file mode 100644 index 000000000..e69de29bb diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index 393215df0..674ff283a 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -105,7 +105,7 @@ def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): if __name__ == "__main__": initialize_environment() - v = oas3_metagpt_text_2_image("Panda emoji") + v = oas3_metagpt_text_to_image("Panda emoji") data = base64.b64decode(v) with open("tmp.png", mode="wb") as writer: writer.write(data) From 2c593bedea549e5068e1c92ff264908d93add0f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 16:59:12 +0800 Subject: [PATCH 084/398] feat: +common talk role --- .well-known/skills.yaml | 34 ++++++++++-- metagpt/actions/skill_action.py | 88 ++++++++++++++++++++++++++++++ metagpt/actions/talk_action.py | 2 +- metagpt/learn/__init__.py | 8 +++ metagpt/learn/skill_loader.py | 33 +++++++++-- metagpt/learn/text_to_embedding.py | 2 +- metagpt/learn/text_to_image.py | 12 +++- metagpt/learn/text_to_speech.py | 6 +- metagpt/memory/brain_memory.py | 12 +++- metagpt/provider/openai_api.py | 63 ++++++++++++++------- metagpt/roles/assistant.py | 34 +++++++++--- metagpt/roles/role.py | 10 +++- metagpt/schema.py | 3 + 13 files changed, 261 insertions(+), 46 deletions(-) diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 5ccb8094b..7a035910c 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -3,15 +3,41 @@ entities: skills: - name: text_to_speech description: Text-to-speech + id: text_to_speech.text_to_speech requisite: - AZURE_TTS_SUBSCRIPTION_KEY - AZURE_TTS_REGION + arguments: + text: 'The text used for voice conversion. Required.' + lang: 'The value can contain a language code such as en (English), or a locale such as en-US (English - United States). The optional parameter are "English", "Chinese". Default value: "Chinese".' + voice: 'Default value: "zh-CN-XiaomoNeural".' + style: 'Speaking style to express different emotions like cheerfulness, empathy, and calm. The optional parameter values are "affectionate", "angry", "calm", "cheerful", "depressed", "disgruntled", "embarrassed", "envious", "fearful", "gentle", "sad", "serious". Default value: "affectionate".' + role: 'With roles, the same voice can act as a different age and gender. The optional parameter values are "Girl", "Boy", "OlderAdultFemale", "OlderAdultMale", "SeniorFemale", "SeniorMale", "YoungAdultFemale", "YoungAdultMale". Default value: "Girl".' + examples: + - ask: 'A girl says "hello world"' + answer: 'text_to_speech(text="hello world", role="Girl")' + - ask: 'A boy affectionate says "hello world"' + answer: 'text_to_speech(text="hello world", role="Boy", style="affectionate")' + - ask: 'A boy says "你好"' + answer: 'text_to_speech(text="hello world", role="Boy", lang="Chinese")' + returns: + type: string + format: base64 + - name: text_to_image description: Create a drawing based on the text. + id: text_to_image.text_to_image requisite: - OPENAI_API_KEY - METAGPT_TEXT_TO_IMAGE_MODEL - - name: text_to_embedding - description: Convert the text into embeddings. - requisite: - - OPENAI_API_KEY + arguments: + text: 'The text used for image conversion. Required.' + size_type: 'Default value: "512x512".' + examples: + - ask: 'Draw a girl' + answer: 'text_to_image(text="Draw a girl", size_type="512x512")' + - ask: 'Draw an apple' + answer: 'text_to_image(text="Draw an apple", size_type="512x512")' + returns: + type: string + format: base64 diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index e69de29bb..8cc7b6c42 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -0,0 +1,88 @@ +import ast +import importlib + +from metagpt.actions import Action, ActionOutput +from metagpt.learn.skill_loader import Skill +from metagpt.logs import logger + + +class ArgumentsParingAction(Action): + def __init__(self, options, last_talk: str, skill: Skill, context=None, llm=None, **kwargs): + super(ArgumentsParingAction, self).__init__(options=options, name='', context=context, llm=llm) + self.skill = skill + self.ask = last_talk + self.rsp = None + self.args = None + + @property + def prompt(self): + prompt = f"{self.skill.name} function parameters description:\n" + for k, v in self.skill.arguments.items(): + prompt += f"parameter `{k}`: {v}\n" + prompt += "\n" + prompt += "Examples:\n" + for e in self.skill.examples: + prompt += f"If want you to do `{e.ask}`, return `{e.answer}` brief and clear.\n" + prompt += f"\nNow I want you to do `{self.ask}`, return in examples format above, brief and clear." + return prompt + + async def run(self, *args, **kwargs) -> ActionOutput: + prompt = self.prompt + logger.info(prompt) + rsp = await self.llm.aask(msg=prompt, system_msgs=[]) + logger.info(rsp) + self.args = ArgumentsParingAction.parse_arguments(skill_name=self.skill.name, txt=rsp) + self.rsp = ActionOutput(content=rsp) + return self.rsp + + @staticmethod + def parse_arguments(skill_name, txt) -> dict: + prefix = skill_name + "(" + if prefix not in txt: + logger.error(f"{skill_name} not in {txt}") + return None + if ")" not in txt: + logger.error(f"')' not in {txt}") + return None + begin_ix = txt.find(prefix) + end_ix = txt.rfind(")") + args_txt = txt[begin_ix + len(prefix): end_ix] + logger.info(args_txt) + fake_expression = f"dict({args_txt})" + parsed_expression = ast.parse(fake_expression, mode='eval') + args = {} + for keyword in parsed_expression.body.keywords: + key = keyword.arg + value = ast.literal_eval(keyword.value) + args[key] = value + return args + + +class SkillAction(Action): + def __init__(self, options, skill: Skill, args: dict, context=None, llm=None, **kwargs): + super(SkillAction, self).__init__(options=options, name='', context=context, llm=llm) + self._skill = skill + self._args = args + self.rsp = None + + async def run(self, *args, **kwargs) -> str | ActionOutput | None: + """Run action""" + self.rsp = self.find_and_call_function(self._skill.name, args=self._args, **self.options) + return ActionOutput(content=self.rsp, instruct_content=self._skill.json()) + + @staticmethod + def find_and_call_function(function_name, args, **kwargs): + try: + module = importlib.import_module("metagpt.learn") + function = getattr(module, function_name) + # 调用函数并返回结果 + result = function(**args, **kwargs) + return result + except (ModuleNotFoundError, AttributeError): + logger.error(f"{function_name} not found") + return None + + +if __name__ == '__main__': + ArgumentsParingAction.parse_arguments(skill_name="text_to_image", + txt='`text_to_image(text="Draw an apple", size_type="512x512")`') diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 4275a1b9e..5485456c5 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -4,7 +4,7 @@ from metagpt.logs import logger class TalkAction(Action): - def __init__(self, options, name: str = '', talk='', history_summary='', context=None, llm=None): + def __init__(self, options, name: str = '', talk='', history_summary='', context=None, llm=None, **kwargs): context = context or {} context["talk"] = talk context["history_summery"] = history_summary diff --git a/metagpt/learn/__init__.py b/metagpt/learn/__init__.py index 28b8739c3..c8270dbfb 100644 --- a/metagpt/learn/__init__.py +++ b/metagpt/learn/__init__.py @@ -5,3 +5,11 @@ @Author : alexanderwu @File : __init__.py """ + +from metagpt.learn.text_to_image import text_to_image +from metagpt.learn.text_to_speech import text_to_speech + +__all__ = [ + "text_to_image", + "text_to_speech", +] \ No newline at end of file diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index eeca12871..46ead728d 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -1,14 +1,26 @@ from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Optional import yaml from pydantic import BaseModel +class Example(BaseModel): + ask: str + answer: str + +class Returns(BaseModel): + type: str + format: Optional[str] = None + class Skill(BaseModel): name: str description: str + id: str requisite: List[str] + arguments: Dict + examples: List[Example] + returns: Returns class EntitySkills(BaseModel): @@ -26,13 +38,26 @@ class SkillLoader: skills = yaml.safe_load(file) self._skills = SkillsDeclaration(**skills) - def get_skill_list(self, entity_name: str = "Assistant"): - if not self._skills or entity_name not in self._skills.entities: + def get_skill_list(self, entity_name: str = "Assistant") -> Dict: + entity_skills = self.get_entity(entity_name) + if not entity_skills: return {} - entity_skills = self._skills.entities.get(entity_name) description_to_name_mappings = {} for s in entity_skills.skills: description_to_name_mappings[s.description] = s.name return description_to_name_mappings + + def get_skill(self, name, entity_name: str = "Assistant") -> Skill: + entity = self.get_entity(entity_name) + if not entity: + return None + for sk in entity.skills: + if sk.name == name: + return sk + + def get_entity(self, name) -> EntitySkills: + if not self._skills: + return None + return self._skills.entities.get(name) \ No newline at end of file diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py index 38fd7c0cb..6d0cefcdb 100644 --- a/metagpt/learn/text_to_embedding.py +++ b/metagpt/learn/text_to_embedding.py @@ -16,7 +16,7 @@ from metagpt.utils.common import initialize_environment @skill_metadata(name="Text to Embedding", description="Convert the text into embeddings.", requisite="`OPENAI_API_KEY`") -def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): +def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", **kwargs): """Text to embedding :param text: The text used for embedding. diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index d123e116a..2f946e239 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -17,7 +17,7 @@ from metagpt.utils.common import initialize_environment @skill_metadata(name="Text to image", description="Create a drawing based on the text.", requisite="`OPENAI_API_KEY` or `METAGPT_TEXT_TO_IMAGE_MODEL`") -def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url=""): +def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs): """Text to image :param text: The text used for image conversion. @@ -27,8 +27,14 @@ def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url :return: The image data is returned in Base64 encoding. """ initialize_environment() + image_declaration = "data:image/png;base64," if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL") or model_url: - return oas3_metagpt_text_to_image(text, size_type, model_url) + data = oas3_metagpt_text_to_image(text, size_type, model_url) + return image_declaration + data if data else "" if os.environ.get("OPENAI_API_KEY") or openai_api_key: - return oas3_openai_text_to_image(text, size_type, openai_api_key) + data = oas3_openai_text_to_image(text, size_type, openai_api_key) + return image_declaration + data if data else "" + raise EnvironmentError + + diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 5631ef45e..90dd878a1 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -17,7 +17,7 @@ from metagpt.utils.common import initialize_environment description="Text-to-speech", requisite="`AZURE_TTS_SUBSCRIPTION_KEY` and `AZURE_TTS_REGION`") def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", - subscription_key="", region=""): + subscription_key="", region="", **kwargs): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -32,8 +32,10 @@ def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affect """ initialize_environment() + audio_declaration = "data:audio/wav;base64," if (os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") and os.environ.get("AZURE_TTS_REGION")) or \ (subscription_key and region): - return oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + data = oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + return audio_declaration + data if data else data raise EnvironmentError diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 97319859a..68e930144 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -35,9 +35,15 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) def move_to_solution(self): - while len(self.history) > 1: - msg = self.history.pop() - self.solution.append(msg) + if len(self.history) < 2: + return + msgs = self.history[:-1] + self.solution.extend(msgs) + if not self.history[-1].is_contain(MessageType.Talk.value): + self.solution.append(self.history[-1]) + self.history = [] + else: + self.history = self.history[-1:] @property def last_talk(self): diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index e98acbd75..27f22e491 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -153,26 +153,10 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self.rpm = int(self._options.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: - max_try = 5 - response = None - for i in range(max_try): - try: - response = await openai.ChatCompletion.acreate( + response = await self.async_retry_call(openai.ChatCompletion.acreate, **self._cons_kwargs(messages), stream=True ) - break - except openai.error.RateLimitError as e: - random_time = random.uniform(0, 3) # 生成0到5秒之间的随机时间 - rounded_time = round(random_time, 1) # 保留一位小数，以实现0.1秒的精度 - logger.warning(f"Exception:{e}, sleeping for {rounded_time} seconds") - await asyncio.sleep(rounded_time) - continue - except Exception as e: - error_str = traceback.format_exc() - logger.error(f"Exception:{e}, stack:{error_str}") - raise e - # create variables to collect the stream of chunks collected_chunks = [] collected_messages = [] @@ -213,12 +197,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return kwargs async def _achat_completion(self, messages: list[dict]) -> dict: - rsp = await self.llm.ChatCompletion.acreate(**self._cons_kwargs(messages)) + rsp = await self.async_retry_call(self.llm.ChatCompletion.acreate, **self._cons_kwargs(messages)) self._update_costs(rsp.get("usage")) return rsp def _chat_completion(self, messages: list[dict]) -> dict: - rsp = self.llm.ChatCompletion.create(**self._cons_kwargs(messages)) + rsp = self.retry_call(self.llm.ChatCompletion.create, **self._cons_kwargs(messages)) self._update_costs(rsp) return rsp @@ -398,4 +382,43 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if match: return match.group(1), match.group(2) else: - return None, input_string \ No newline at end of file + return None, input_string + + @staticmethod + async def async_retry_call(func, *args, **kwargs): + for i in range(OpenAIGPTAPI.MAX_TRY): + try: + rsp = await func(*args, **kwargs) + return rsp + except openai.error.RateLimitError as e: + random_time = random.uniform(0, 3) # 生成0到5秒之间的随机时间 + rounded_time = round(random_time, 1) # 保留一位小数，以实现0.1秒的精度 + logger.warning(f"Exception:{e}, sleeping for {rounded_time} seconds") + await asyncio.sleep(rounded_time) + continue + except openai.error.APIConnectionError as e: + logger.warning(f"Exception:{e}") + continue + except Exception as e: + error_str = traceback.format_exc() + logger.error(f"Exception:{e}, stack:{error_str}") + raise e + + @staticmethod + def retry_call(func, *args, **kwargs): + for i in range(OpenAIGPTAPI.MAX_TRY): + try: + rsp = func(*args, **kwargs) + return rsp + except openai.error.RateLimitError as e: + logger.warning(f"Exception:{e}") + continue + except openai.error.APIConnectionError as e: + logger.warning(f"Exception:{e}") + continue + except Exception as e: + error_str = traceback.format_exc() + logger.error(f"Exception:{e}, stack:{error_str}") + raise e + + MAX_TRY = 5 diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index dfbd406bc..032d73ca5 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -15,8 +15,8 @@ """ import asyncio - from metagpt.actions import ActionOutput +from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction from metagpt.actions.talk_action import TalkAction from metagpt.config import Config from metagpt.learn.skill_loader import SkillLoader @@ -53,7 +53,7 @@ class Assistant(Role): logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(rsp) - return await self._plan(rsp) + return await self._plan(rsp, last_talk=last_talk) async def act(self) -> ActionOutput: result = await self._rc.todo.run(**self._options) @@ -88,8 +88,18 @@ class Assistant(Role): return True async def skill_handler(self, text, **kwargs) -> bool: - skill = - pass + last_talk = kwargs.get("last_talk") + skill = self.skills.get_skill(text) + logger.info(f"skill not found: {text}") + if not skill: + return await self.talk_handler(text=last_talk, **kwargs) + action = ArgumentsParingAction(options=self.options, skill=skill, llm=self._llm, **kwargs) + await action.run(**kwargs) + if action.args is None: + return await self.talk_handler(text=last_talk, **kwargs) + action = SkillAction(options=self.options, skill=skill, args=action.args, llm=self._llm) + self.add_to_do(action) + return True async def refine_memory(self) -> str: history_text = self.memory.history_text @@ -97,7 +107,7 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_context_title(history_text, max_words=20) - if await self._llm.is_related(last_talk, history_summary): # 合并相关内容 + if last_talk and await self._llm.is_related(last_talk, history_summary): # 合并相关内容 last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk @@ -109,11 +119,20 @@ class Assistant(Role): from metagpt.provider.openai_api import OpenAIGPTAPI return OpenAIGPTAPI.extract_info(input_string) + def get_memory(self) -> str: + return self.memory.json() + + def load_memory(self, jsn): + try: + self.memory = BrainMemory(**jsn) + except Exception as e: + logger.exception(f"load error:{e}, data:{jsn}") + async def main(): options = Config().runtime_options cost_manager = CostManager(**options) - topic = "dataiku vs. datarobot" + topic = "draw an apple" role = Assistant(options=options, cost_manager=cost_manager, language="Chinese") await role.talk(topic) while True: @@ -121,8 +140,9 @@ async def main(): if not has_action: break msg = await role.act() - print(msg) + logger.info(msg) # 获取用户终端输入 + logger.info("Enter prompt") talk = input("You: ") await role.talk(talk) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1bb73f884..47f494c69 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -325,4 +325,12 @@ class Role: self._actions.append(act) def add_to_do(self, act): - self._rc.todo = act \ No newline at end of file + self._rc.todo = act + + async def think(self) -> bool: + return await self._think() + + async def act(self) -> ActionOutput: + msg = await self._act() + return ActionOutput(content=msg.content, + instruct_content=msg.instruct_content) diff --git a/metagpt/schema.py b/metagpt/schema.py index e1cd011c6..909313886 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -67,6 +67,9 @@ class Message: intersection = set(tags) & self.tags return len(intersection) > 0 + def is_contain(self, tag): + return self.is_contain_tags([tag]) + @dataclass class UserMessage(Message): From 644286959152f933bc84815704194080ad86e5e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 17:11:33 +0800 Subject: [PATCH 085/398] feat: +common talk role --- metagpt/roles/assistant.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 032d73ca5..f75c05695 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -43,6 +43,8 @@ class Assistant(Role): async def think(self) -> bool: """Everything will be done part by part.""" last_talk = await self.refine_memory() + if not last_talk: + return False prompt = f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): From 6e459da875896e094826841814714f3fdf9b1911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 17:20:21 +0800 Subject: [PATCH 086/398] feat: +Exceeds the maximum retries exception --- metagpt/provider/openai_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 27f22e491..4fab92fb3 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -403,6 +403,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): error_str = traceback.format_exc() logger.error(f"Exception:{e}, stack:{error_str}") raise e + raise openai.error.OpenAIError("Exceeds the maximum retries") @staticmethod def retry_call(func, *args, **kwargs): @@ -420,5 +421,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): error_str = traceback.format_exc() logger.error(f"Exception:{e}, stack:{error_str}") raise e + raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 From 5dc352bf2fd102732a525f7d1020c91889a5f0be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 19:18:23 +0800 Subject: [PATCH 087/398] feat: fix requirements-test.txt --- requirements-test.txt | 40 +--------------------------------------- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 40 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 7c03dddd9..0a34c35ea 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,41 +1,3 @@ -aiohttp==3.8.4 -azure-cognitiveservices-speech==1.30.0 -channels==4.0.0 -chromadb==0.3.22 -# Django==4.1.5 -# docx==0.2.4 -duckduckgo_search==2.9.4 -#faiss==1.5.3 -faiss_cpu==1.7.4 -fire==0.4.0 -# godot==0.1.1 -# google_api_python_client==2.93.0 -langchain==0.0.231 -loguru==0.6.0 -meilisearch==0.21.0 -numpy==1.24.3 -openai==0.27.8 -openpyxl -pandas==1.4.1 -pydantic==1.10.7 -#pygame==2.1.3 -pymilvus==2.2.8 -pytest==7.2.2 -python_docx==0.8.11 -PyYAML==6.0 -# sentence_transformers==2.2.2 -setuptools==65.6.3 -tenacity==8.2.2 -tiktoken==0.3.3 -tqdm==4.64.0 -#unstructured[local-inference] -playwright -selenium>4 -webdriver_manager<3.9 -anthropic==0.3.6 -typing-inspect==0.8.0 -typing_extensions==4.5.0 -bs4 -aiofiles +-r requirements.txt pytest pytest-asyncio \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4bfab1f3b..70f2a3809 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,4 +40,4 @@ libcst==1.0.1 qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 - +azure-cognitiveservices-speech==1.30.0 From 2c83921aee8231696947c6dacfc66f340a739648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 19:54:49 +0800 Subject: [PATCH 088/398] feat: +brain memory --- metagpt/roles/assistant.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index f75c05695..e02005f31 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -28,6 +28,7 @@ from metagpt.schema import Message DEFAULT_MAX_TOKENS = 1500 COMMAND_TOKENS = 500 +BRAIN_MEMORY = "BRAIN_MEMORY" class Assistant(Role): @@ -37,7 +38,8 @@ class Assistant(Role): constraints="Talk in {language}", desc="", *args, **kwargs): super(Assistant, self).__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) - self.memory = BrainMemory() + brain_memory = options.get(BRAIN_MEMORY) + self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory() self.skills = SkillLoader() async def think(self) -> bool: From 6e10cbb73bd19b01cf70146c06fa63effc3db4d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 20:18:47 +0800 Subject: [PATCH 089/398] feat: +knowledge --- metagpt/actions/talk_action.py | 7 +++++-- metagpt/memory/brain_memory.py | 5 +++++ metagpt/roles/assistant.py | 5 +++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 5485456c5..dab4873fb 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -4,18 +4,21 @@ from metagpt.logs import logger class TalkAction(Action): - def __init__(self, options, name: str = '', talk='', history_summary='', context=None, llm=None, **kwargs): + def __init__(self, options, name: str = '', talk='', history_summary='', knowledge='', context=None, llm=None, **kwargs): context = context or {} context["talk"] = talk context["history_summery"] = history_summary + context["knowledge"] = knowledge super(TalkAction, self).__init__(options=options, name=name, context=context, llm=llm) self._talk = talk self._history_summary = history_summary + self._knowledge = knowledge self._rsp = None @property def prompt(self): - prompt = f"{self._history_summary}\n\n" + prompt = f"{self._knowledge}\n\n" + prompt += f"{self._history_summary}\n\n" if self._history_summary != "": prompt += "According to the historical conversation above, " language = self.options.get("language", "Chinese") diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 68e930144..422c096f3 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -17,6 +17,7 @@ class BrainMemory(pydantic.BaseModel): history: List[Message] = [] stack: List[Message] = [] solution: List[Message] = [] + knowledge: List[Message] = [] def add_talk(self, msg: Message): @@ -27,6 +28,10 @@ class BrainMemory(pydantic.BaseModel): msg.add_tag(MessageType.Answer.value) self.history.append(msg) + def get_knowledge(self) -> str: + texts = [k.content for k in self.knowledge] + return "\n".join(texts) + @property def history_text(self): if len(self.history) == 0: diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index e02005f31..c001d69f0 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -87,7 +87,8 @@ class Assistant(Role): return await handler(text, **kwargs) async def talk_handler(self, text, **kwargs) -> bool: - action = TalkAction(options=self.options, talk=text, llm=self._llm, **kwargs) + action = TalkAction(options=self.options, talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, + **kwargs) self.add_to_do(action) return True @@ -136,7 +137,7 @@ class Assistant(Role): async def main(): options = Config().runtime_options cost_manager = CostManager(**options) - topic = "draw an apple" + topic = "what's apple" role = Assistant(options=options, cost_manager=cost_manager, language="Chinese") await role.talk(topic) while True: From d35dc8bfefd250c57a306fba8bce725bb4578aa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 20:23:50 +0800 Subject: [PATCH 090/398] feat: +knowledge --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index dab4873fb..b1410d34f 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -17,7 +17,7 @@ class TalkAction(Action): @property def prompt(self): - prompt = f"{self._knowledge}\n\n" + prompt = f"Knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" prompt += f"{self._history_summary}\n\n" if self._history_summary != "": prompt += "According to the historical conversation above, " From 9ff489b6c68f7e668496dede44d9f6c1bff86cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 20:24:57 +0800 Subject: [PATCH 091/398] feat: +knowledge --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index b1410d34f..5692cf4f4 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -17,7 +17,7 @@ class TalkAction(Action): @property def prompt(self): - prompt = f"Knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" + prompt = f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" prompt += f"{self._history_summary}\n\n" if self._history_summary != "": prompt += "According to the historical conversation above, " From cc89f3b7263e24a445f42fec92282480029a1660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 21:55:34 +0800 Subject: [PATCH 092/398] feat: revert --- metagpt/memory/memory.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 1a8003fba..a96aaf1be 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -4,8 +4,6 @@ @Time : 2023/5/20 12:15 @Author : alexanderwu @File : memory.py -@Modified By: mashenquan, 2023-8-7. Modified get_by_actions() to support for dynamically generated Action classes - at runtime. """ from collections import defaultdict from typing import Iterable, Type @@ -82,20 +80,8 @@ class Memory: def get_by_actions(self, actions: Iterable[Type[Action]]) -> list[Message]: """Return all messages triggered by specified Actions""" rsp = [] - # Using the `type(obj).__name__` approach to support the runtime creation of requirement classes. - # See `MetaAction.get_action_type()` for more. - class_names = {type(k).__name__: k for k in self.index.keys()} for action in actions: - if type(action).__name__ not in class_names: + if action not in self.index: continue - key = class_names[type(action).__name__] - rsp += self.index[key] + rsp += self.index[action] return rsp - - def get_by_tags(self, tags: list) -> list[Message]: - """Return messages with specified tags""" - result = [] - for m in self.storage: - if m.is_contain_tags(tags): - result.append(m) - return result From 2574ecaecfb4054da2e42b81573f5e52ba8ac73f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 22:08:45 +0800 Subject: [PATCH 093/398] =?UTF-8?q?feat:=20=E5=88=A0=E6=8E=89meta=20role?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fork_meta_role_write_teaching_plan.py | 126 ----------------- metagpt/provider/openai_api.py | 6 - metagpt/roles/fork_meta_role.py | 133 ------------------ metagpt/roles/uml_meta_role_factory.py | 43 ------ metagpt/roles/uml_meta_role_options.py | 69 --------- tests/metagpt/actions/test_meta_action.py | 51 ------- tests/metagpt/roles/test_fork_meta_role.py | 94 ------------- .../roles/test_uml_meta_role_factory.py | 61 -------- .../roles/test_uml_meta_role_options.py | 40 ------ 9 files changed, 623 deletions(-) delete mode 100644 examples/fork_meta_role_write_teaching_plan.py delete mode 100644 metagpt/roles/fork_meta_role.py delete mode 100644 metagpt/roles/uml_meta_role_factory.py delete mode 100644 metagpt/roles/uml_meta_role_options.py delete mode 100644 tests/metagpt/actions/test_meta_action.py delete mode 100644 tests/metagpt/roles/test_fork_meta_role.py delete mode 100644 tests/metagpt/roles/test_uml_meta_role_factory.py delete mode 100644 tests/metagpt/roles/test_uml_meta_role_options.py diff --git a/examples/fork_meta_role_write_teaching_plan.py b/examples/fork_meta_role_write_teaching_plan.py deleted file mode 100644 index e529a9b46..000000000 --- a/examples/fork_meta_role_write_teaching_plan.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : fork_meta_role.py -@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to possess the - ability to construct flows freely by concatenating symbols. Simultaneously, I am also striving to make - these symbols configurable and standardized, making the process of building flow structures more - convenient. This is a fork meta-role demo that implements the functionality of - `examples/write_teaching_plan.py`. -""" - -import asyncio -from pathlib import Path -import sys - -sys.path.append(str(Path(__file__).resolve().parent.parent)) -import aiofiles -import fire -import yaml - -from metagpt.actions.meta_action import MetaAction -from metagpt.logs import logger -from metagpt.roles.uml_meta_role_factory import UMLMetaRoleFactory -from metagpt.roles.uml_meta_role_options import ProjectConfig -from metagpt.software_company import SoftwareCompany - - -async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): - """Run a startup. Be a teacher in education industry.""" - - demo_lesson = """ - UNIT 1 Making New Friends - TOPIC 1 Welcome to China! - Section A - - 1a Listen and number the following names. - Jane Mari Kangkang Michael - Look, listen and understand. Then practice the conversation. - Work in groups. Introduce yourself using - I ’m ... Then practice 1a - with your own hometown or the following places. - - 1b Listen and number the following names - Jane Michael Maria Kangkang - 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. - China the USA the UK Hong Kong Beijing - - 2a Look, listen and understand. Then practice the conversation - Hello! - Hello! - Hello! - Hello! Are you Maria? - No, I’m not. I’m Jane. - Oh, nice to meet you, Jane - Nice to meet you, too. - Hi, Maria! - Hi, Kangkang! - Welcome to China! - Thanks. - - 2b Work in groups. Make up a conversation with your own name and the - following structures. - A: Hello! / Good morning! / Hi! I’m ... Are you ... ? - B: ... - - 3a Listen, say and trace - Aa Bb Cc Dd Ee Ff Gg - - 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. - Aa Bb Cc Dd Ee Ff Gg - - 3c Match the big letters with the small ones. Then write them on the lines. - """ - - lesson = "" - if lesson_file and Path(lesson_file).exists(): - async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: - lesson = await reader.read() - logger.info(f"Course content: {lesson}") - if not lesson: - logger.info("No course content provided, using the demo course.") - lesson = demo_lesson - - yaml_filename = kwargs["config"] - kwargs["lesson"] = lesson - - with open(yaml_filename, "r") as reader: - configs = yaml.safe_load(reader) - - startup_config = ProjectConfig(**configs) - company = SoftwareCompany() - roles = UMLMetaRoleFactory.create_roles(role_configs=startup_config.roles, - options=company.options, - cost_manager=company.cost_manager, - **kwargs) - company.hire(roles) - company.invest(startup_config.startup.investment) - company.start_project(lesson, role=startup_config.startup.role, - cause_by=MetaAction.get_action_type(startup_config.startup.requirement)) - await company.run(n_round=startup_config.startup.n_round) - - -def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): - """ - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - :param idea: lesson filename. - :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. - :param n_round: Reserved. - :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` - :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` - :return: - """ - asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) - - -if __name__ == '__main__': - """ - Formats: - ``` - python write_teaching_plan.py lesson_filename --teaching_language= --language= - ``` - If `lesson_filename` is not available, a demo lesson content will be used. - """ - fire.Fire(main) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 4fab92fb3..098388a7c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -396,9 +396,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): logger.warning(f"Exception:{e}, sleeping for {rounded_time} seconds") await asyncio.sleep(rounded_time) continue - except openai.error.APIConnectionError as e: - logger.warning(f"Exception:{e}") - continue except Exception as e: error_str = traceback.format_exc() logger.error(f"Exception:{e}, stack:{error_str}") @@ -414,9 +411,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): except openai.error.RateLimitError as e: logger.warning(f"Exception:{e}") continue - except openai.error.APIConnectionError as e: - logger.warning(f"Exception:{e}") - continue except Exception as e: error_str = traceback.format_exc() logger.error(f"Exception:{e}, stack:{error_str}") diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py deleted file mode 100644 index 57d467080..000000000 --- a/metagpt/roles/fork_meta_role.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : fork_meta_role.py -@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the - ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to - make these symbols configurable and standardized, making the process of building flows more convenient. - For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` - This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a - configuration file. -@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. - -""" - -import re - -import aiofiles - -from metagpt.actions.meta_action import MetaAction -from metagpt.const import WORKSPACE_ROOT -from metagpt.logs import logger -from metagpt.roles import Role -from metagpt.roles.uml_meta_role_options import MetaActionOptions, UMLMetaRoleOptions -from metagpt.schema import Message - - -class ForkMetaRole(Role): - """A `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file""" - def __init__(self, options, cost_manager, role_options, **kwargs): - """Initialize a `fork` style meta role - - :param options: System configuration - :param cost_manager: Cost manager - :param role_options: pattern yaml file data - :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` - :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` - """ - opts = UMLMetaRoleOptions(**role_options) - global_variables = { - "name": Role.format_value(opts.name, kwargs), - "profile": Role.format_value(opts.profile, kwargs), - "goal": Role.format_value(opts.goal, kwargs), - "constraints": Role.format_value(opts.constraints, kwargs), - "desc": Role.format_value(opts.desc, kwargs), - "role": Role.format_value(opts.role, kwargs) - } - for k, v in kwargs.items(): - if k not in global_variables: - global_variables[k] = v - - super(ForkMetaRole, self).__init__( - options=options, - cost_manager=cost_manager, - name=global_variables["name"], - profile=global_variables["profile"], - goal=global_variables["goal"], - constraints=global_variables["constraints"], - desc=global_variables["desc"], - **kwargs - ) - actions = [] - for m in opts.actions: - for k, v in m.items(): - v = Role.format_value(v, kwargs) - m[k] = v - for k, v in global_variables.items(): - if k not in m: - m[k] = v - - o = MetaActionOptions(**m) - o.set_default_template(opts.templates[o.template_ix]) - - act = MetaAction(options=options, action_options=o, llm=self._llm, **m) - actions.append(act) - self._init_actions(actions) - requirement_types = set() - for v in opts.requirement: - requirement_types.add(MetaAction.get_action_type(v)) - self._watch(requirement_types) - - async def _think(self) -> None: - """Everything will be done part by part.""" - if self._rc.todo is None: - self._set_state(0) - return True - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None - return False - - async def _react(self) -> Message: - ret = Message(content="") - while True: - await self._think() - if self._rc.todo is None: - break - logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") - msg = await self._act() - if ret.content != '': - ret.content += "\n\n\n" - ret.content += msg.content - logger.info(ret.content) - await self.save(ret.content) - return ret - - async def save(self, content): - """Save teaching plan""" - output_filename = self.options.get("output_filename") - if not output_filename: - return - filename = ForkMetaRole.new_file_name(output_filename) - pathname = WORKSPACE_ROOT / "teaching_plan" - pathname.mkdir(exist_ok=True) - pathname = pathname / filename - try: - async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer: - await writer.write(content) - except Exception as e: - logger.error(f'Save failed：{e}') - logger.info(f"Save to:{pathname}") - - @staticmethod - def new_file_name(lesson_title, ext=".md"): - """Create a related file name based on `lesson_title` and `ext`.""" - # Define the special characters that need to be replaced. - illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' - # Replace the special characters with underscores. - filename = re.sub(illegal_chars, '_', lesson_title) + ext - return re.sub(r'_+', '_', filename) \ No newline at end of file diff --git a/metagpt/roles/uml_meta_role_factory.py b/metagpt/roles/uml_meta_role_factory.py deleted file mode 100644 index 42071b0a6..000000000 --- a/metagpt/roles/uml_meta_role_factory.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : uml_meta_role_factory.py -@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the - ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to - make these symbols configurable and standardized, making the process of building flows more convenient. - For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` -""" - -from metagpt.roles.fork_meta_role import ForkMetaRole -from metagpt.roles.uml_meta_role_options import UMLMetaRoleOptions - - -class UMLMetaRoleFactory: - """Factory of UML activity role classes""" - - @classmethod - def create_roles(cls, role_configs, **kwargs): - """Generate the flow of the project based on the configuration in the format of config/pattern/template.yaml. - - :param role_configs: `roles` field of template.yaml - :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` - - """ - roles = [] - for m in role_configs: - opt = UMLMetaRoleOptions(**m) - constructor = cls.CONSTRUCTORS.get(opt.role_type) - if constructor is None: - raise NotImplementedError( - f"{opt.role_type} is not implemented" - ) - r = constructor(role_options=m, **kwargs) - roles.append(r) - return roles - - CONSTRUCTORS = { - "fork": ForkMetaRole, - # TODO: add more activity node constructor here.. - } diff --git a/metagpt/roles/uml_meta_role_options.py b/metagpt/roles/uml_meta_role_options.py deleted file mode 100644 index 1d0fb322e..000000000 --- a/metagpt/roles/uml_meta_role_options.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : uml_meta_role_options.py -@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the - ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to - make these symbols configurable and standardized, making the process of building flows more convenient. - For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` -""" - -from typing import List, Dict - -from pydantic import BaseModel - - -# `startup` field of config/pattern/template.yaml -class StartupConfig(BaseModel): - requirement: str - role: str - investment: float = 3.0 - n_round: int = 3 - - -# config/pattern/template.yaml -class ProjectConfig(BaseModel): - startup: StartupConfig - roles: List[Dict] - - -# element of `actions` field of config/pattern/template.yaml -class MetaActionOptions(BaseModel): - topic: str - name: str = "" - language: str = "Chinese" - template_ix: int = 0 - statements: List[str] = [] - template: str = "" - rsp_begin_tag: str = "" - rsp_end_tag: str = "" - - def set_default_template(self, v): - if not self.template: - self.template = v - - def format_prompt(self, **kwargs): - statements = "\n".join(self.statements) - opts = kwargs.copy() - opts["statements"] = statements - - from metagpt.roles import Role - prompt = Role.format_value(self.template, opts) - return prompt - - -# element of `roles` field of config/pattern/template.yaml -class UMLMetaRoleOptions(BaseModel): - role_type: str - name: str = "" - profile: str = "" - goal: str = "" - role: str = "" - constraints: str = "" - desc: str = "" - templates: List[str] = [] - output_filename: str = "" - actions: List - requirement: List diff --git a/tests/metagpt/actions/test_meta_action.py b/tests/metagpt/actions/test_meta_action.py deleted file mode 100644 index cbaf3456c..000000000 --- a/tests/metagpt/actions/test_meta_action.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/8 -@Author : mashenquan -@File : test_meta_action.py -""" -from typing import Dict - -from pydantic import BaseModel - -from metagpt.actions.meta_action import MetaAction -from metagpt.roles.uml_meta_role_options import MetaActionOptions - - -def test_meta_action_create(): - class Inputs(BaseModel): - options: Dict - kwargs: Dict - expect_class_name: str - expect_prompt: str - - inputs = [ - { - "options": { - "topic": "TOPIC_A", - "name": "A", - "language": "XX", - "template_ix": 0, - "statements": ["Statement A", "Statement B"], - "template": "{statements}", - "rsp_begin_tag": "", - "rsp_end_tag": "" - }, - "kwargs": {}, - "expect_class_name": "TOPIC_A", - "expect_prompt": "\n".join(["Statement A", "Statement B"]), - } - ] - - for i in inputs: - seed = Inputs(**i) - opt = MetaActionOptions(**seed.options) - act = MetaAction(opt, **seed.kwargs) - assert seed.expect_prompt == act.prompt - t = MetaAction.get_action_type(seed.expect_class_name) - assert t.__name__ == seed.expect_class_name - - -if __name__ == '__main__': - test_meta_action_create() diff --git a/tests/metagpt/roles/test_fork_meta_role.py b/tests/metagpt/roles/test_fork_meta_role.py deleted file mode 100644 index 355197234..000000000 --- a/tests/metagpt/roles/test_fork_meta_role.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/8 -@Author : mashenquan -@File : test_fork_meta_role.py -""" -from typing import Dict - -from pydantic import BaseModel - -from metagpt.config import Config -from metagpt.provider.openai_api import CostManager -from metagpt.roles.fork_meta_role import ForkMetaRole - - -def test_creat_role(): - class Inputs(BaseModel): - role: Dict - action_count: int - - inputs = [ - { - "role": { - "role_type": "fork", - "name": "Lily", - "profile": "{teaching_language} Teacher", - "goal": "writing a {language} teaching plan part by part", - "constraints": "writing in {language}", - "role": "You are a {teaching_language} Teacher, named Lily, your goal is writing a {" - "teaching_language} teaching plan part by part, and the constraint is writing in {language}.", - "desc": "", - "output_filename": "teaching_plan_demo.md", - "requirement": ["TeachingPlanRequirement"], - "templates": [ - "Do not refer to the context of the previous conversation records, start the conversation " - "anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[" - "LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" " - "defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the " - "format requirements for your responses;\n\t\"Constraint\" defines the conditions that your " - "responses must comply with.\n\n{statements}\nConstraint: Writing in {language}.\nAnswer options: " - "Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\n[" - "LESSON_BEGIN]\n{lesson}\n[LESSON_END]", - "Do not refer to the context of the previous conversation records, start the conversation " - "anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[" - "LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" " - "defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the " - "format requirements for your responses;\n\t\"Constraint\" defines the conditions that your " - "responses must comply with.\n\nCapacity and role: {role}\nStatement: Write the \"{topic}\" part " - "of teaching plan, WITHOUT ANY content unrelated to \"{topic}\"!!\n{statements}\nAnswer options: " - "Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" " - "tags.\nAnswer options: Using proper markdown format from second-level header " - "format.\nConstraint: Writing in {language}.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END] " - ], - "actions": [ - { - "name": "", - "topic": "Title", - "language": "Chinese", - "statements": [ - "Statement: Find and return the title of the lesson only with \"# \" prefixed, without " - "anything else."], - "template_ix": 0}, - { - "name": "", - "topic": "Teaching Hours", - "language": "Chinese", - "statements": [], - "template_ix": 1, - "rsp_begin_tag": "[TEACHING_PLAN_BEGIN]", - "rsp_end_tag": "[TEACHING_PLAN_END]"} - ] - }, - "action_count": 2 - } - ] - - for i in inputs: - seed = Inputs(**i) - kwargs = { - "teaching_language": "AA", - "language": "BB" - } - runtime_options = Config().runtime_options - cost_manager = CostManager(options=runtime_options) - role = ForkMetaRole(runtime_options=runtime_options, cost_manager=cost_manager, role_options=seed.role, **kwargs) - assert role.action_count == 2 - assert "{" not in role.profile - assert "{" not in role.goal - assert "{" not in role.constraints - - -if __name__ == '__main__': - test_creat_role() diff --git a/tests/metagpt/roles/test_uml_meta_role_factory.py b/tests/metagpt/roles/test_uml_meta_role_factory.py deleted file mode 100644 index f59a30611..000000000 --- a/tests/metagpt/roles/test_uml_meta_role_factory.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/8 -@Author : mashenquan -@File : test_uml_meta_role_factory.py -""" -from typing import List, Dict - -from pydantic import BaseModel - -from metagpt.roles.uml_meta_role_factory import UMLMetaRoleFactory - - -def test_create_roles(): - class Inputs(BaseModel): - roles: List - kwargs: Dict - - inputs = [ - { - "roles": [ - { - "role_type": "fork", - "name": "Lily", - "profile": "{teaching_language} Teacher", - "goal": "writing a {language} teaching plan part by part", - "constraints": "writing in {language}", - "role": "You are a {teaching_language} Teacher, named Lily.", - "desc": "", - "output_filename": "teaching_plan_demo.md", - "requirement": ["TeachingPlanRequirement"], - "templates": ["Do 1 {statements}", "Do 2 {statements}"], - "actions": [ - { - "name": "", - "topic": "Title", - "language": "Chinese", - "statements": ["statement 1", "statement 2"]} - ], - "template_ix": 0 - } - ], - "kwargs": { - "teaching_language": "AA", - "language": "BB", - } - } - ] - - for i in inputs: - seed = Inputs(**i) - roles = UMLMetaRoleFactory.create_roles(seed.roles, **seed.kwargs) - assert len(roles) == 1 - assert "{" not in roles[0].profile - assert "{" not in roles[0].goal - assert roles[0].action_count == 1 - - -if __name__ == '__main__': - test_create_roles() diff --git a/tests/metagpt/roles/test_uml_meta_role_options.py b/tests/metagpt/roles/test_uml_meta_role_options.py deleted file mode 100644 index 1eb66c50e..000000000 --- a/tests/metagpt/roles/test_uml_meta_role_options.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/8 -@Author : mashenquan -@File : test_uml_meta_role_options.py -""" -from typing import List - -from pydantic import BaseModel - -from metagpt.roles.uml_meta_role_options import MetaActionOptions - - -def test_set_default_template(): - class Inputs(BaseModel): - statements: List - template: str - expect_prompt: str - - inputs = [ - { - "statements": ["Statement: 1", "Statement: 2"], - "template": "{statements}", - "expect_prompt": "Statement: 1\nStatement: 2" - } - ] - - for i in inputs: - seed = Inputs(**i) - opt = MetaActionOptions(topic="", statements=seed.statements) - assert opt.template == "" - opt.set_default_template(seed.template) - assert opt.template == seed.template - kwargs = {} - assert opt.format_prompt(**kwargs) == seed.expect_prompt - - -if __name__ == '__main__': - test_set_default_template() From f33af9dbc9d7af71aafacb6aa51b936eaf3e56c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 22:24:25 +0800 Subject: [PATCH 094/398] fixbug: skill_yaml_file_name --- metagpt/learn/skill_loader.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index 46ead728d..71535f310 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -32,9 +32,10 @@ class SkillsDeclaration(BaseModel): class SkillLoader: - def __init__(self): - skill_file_name = Path(__file__).parent.parent.parent / ".well-known/skills.yaml" - with open(str(skill_file_name), 'r') as file: + def __init__(self, skill_yaml_file_name: Path = None): + if not skill_yaml_file_name: + skill_yaml_file_name = Path(__file__).parent.parent.parent / ".well-known/skills.yaml" + with open(str(skill_yaml_file_name), 'r') as file: skills = yaml.safe_load(file) self._skills = SkillsDeclaration(**skills) From 1545a702ccd4610c14acd818ae8fc6a19fd8d84d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 22:28:41 +0800 Subject: [PATCH 095/398] fixbug: skill_yaml_file_name --- metagpt/actions/meta_action.py | 64 ---------------------------------- metagpt/roles/assistant.py | 5 ++- 2 files changed, 4 insertions(+), 65 deletions(-) delete mode 100644 metagpt/actions/meta_action.py diff --git a/metagpt/actions/meta_action.py b/metagpt/actions/meta_action.py deleted file mode 100644 index 4c52e7cfd..000000000 --- a/metagpt/actions/meta_action.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : meta_action.py -@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the - ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to - make these symbols configurable and standardized, making the process of building flows more convenient. - For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` - This file defines a meta action capable of generating arbitrary actions at runtime based on a - configuration file. -""" - -from typing import Type - -from metagpt.actions import Action -from metagpt.logs import logger -from metagpt.roles.uml_meta_role_options import MetaActionOptions -from metagpt.schema import Message - - -class MetaAction(Action): - def __init__(self, options, action_options: MetaActionOptions, llm=None, **kwargs): - super(MetaAction, self).__init__(options=options, - name=action_options.name, - context=kwargs.get("context"), - llm=llm) - self.prompt = action_options.format_prompt(**kwargs) - self.action_options = action_options - self.kwargs = kwargs - - def __str__(self): - """Return `topic` value when str()""" - return self.action_options.topic - - def __repr__(self): - """Show `topic` value when debug""" - return self.action_options.topic - - async def run(self, messages, *args, **kwargs): - if len(messages) < 1 or not isinstance(messages[0], Message): - raise ValueError("Invalid args, a tuple of List[Message] is expected") - - logger.debug(self.prompt) - rsp = await self._aask(prompt=self.prompt) - logger.debug(rsp) - self._set_result(rsp) - return self.rsp - - def _set_result(self, rsp): - if self.action_options.rsp_begin_tag and self.action_options.rsp_begin_tag in rsp: - ix = rsp.index(self.action_options.rsp_begin_tag) - rsp = rsp[ix + len(self.action_options.rsp_begin_tag):] - if self.action_options.rsp_end_tag and self.action_options.rsp_end_tag in rsp: - ix = rsp.index(self.action_options.rsp_end_tag) - rsp = rsp[0:ix] - self.rsp = rsp.strip() - - @staticmethod - def get_action_type(topic: str): - """Create a runtime :class:`Action` subclass""" - action_type: Type["Action"] = type(topic, (Action,), {"name": topic}) - return action_type diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index c001d69f0..a3af715e3 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -14,6 +14,7 @@ """ import asyncio +from pathlib import Path from metagpt.actions import ActionOutput from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction @@ -29,6 +30,7 @@ from metagpt.schema import Message DEFAULT_MAX_TOKENS = 1500 COMMAND_TOKENS = 500 BRAIN_MEMORY = "BRAIN_MEMORY" +SKILL_PATH = "SKILL_PATH" class Assistant(Role): @@ -40,7 +42,8 @@ class Assistant(Role): goal=goal, constraints=constraints, desc=desc, *args, **kwargs) brain_memory = options.get(BRAIN_MEMORY) self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory() - self.skills = SkillLoader() + skill_path = Path(options.get(SKILL_PATH)) if options.get(SKILL_PATH) else None + self.skills = SkillLoader(skill_yaml_file_name=skill_path) async def think(self) -> bool: """Everything will be done part by part.""" From ee77d4b0fb2ce865b59de2a6095d01c9ab695f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 10:41:34 +0800 Subject: [PATCH 096/398] feat: +exported function --- metagpt/roles/role.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 47f494c69..286c87eb1 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -328,9 +328,16 @@ class Role: self._rc.todo = act async def think(self) -> bool: - return await self._think() + """The exported `think` function""" + has_action = await self._think() + if not has_action: + return False + if not self._rc.todo: + return False + return True async def act(self) -> ActionOutput: + """The exported `act` function""" msg = await self._act() return ActionOutput(content=msg.content, instruct_content=msg.instruct_content) From 93d6bc6569e4e011d7823a21bea018d7b05ef57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 11:13:32 +0800 Subject: [PATCH 097/398] feat: +todo_description --- metagpt/roles/role.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 286c87eb1..c57bf4f43 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -341,3 +341,11 @@ class Role: msg = await self._act() return ActionOutput(content=msg.content, instruct_content=msg.instruct_content) + + @property + def todo_description(self): + if not self._rc or not self._rc.todo: + return "" + if self._rc.todo.desc: + return self._rc.todo.desc + return f"{self._rc.todo.__class__}" From 6d3f2acddbcb7a68004dbcb7a228d19918ce22ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 11:19:59 +0800 Subject: [PATCH 098/398] feat: +todo_description --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index c57bf4f43..ed02575db 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -348,4 +348,4 @@ class Role: return "" if self._rc.todo.desc: return self._rc.todo.desc - return f"{self._rc.todo.__class__}" + return f"{type(self._rc.todo).__name__}" From 5a03ff20ce65e353955a5209ac65e91edd007fdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 11:58:32 +0800 Subject: [PATCH 099/398] fixbug: call skill in api --- metagpt/roles/assistant.py | 5 +++-- metagpt/utils/common.py | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index a3af715e3..3924039b5 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -98,14 +98,15 @@ class Assistant(Role): async def skill_handler(self, text, **kwargs) -> bool: last_talk = kwargs.get("last_talk") skill = self.skills.get_skill(text) - logger.info(f"skill not found: {text}") if not skill: + logger.info(f"skill not found: {text}") return await self.talk_handler(text=last_talk, **kwargs) action = ArgumentsParingAction(options=self.options, skill=skill, llm=self._llm, **kwargs) await action.run(**kwargs) if action.args is None: return await self.talk_handler(text=last_talk, **kwargs) - action = SkillAction(options=self.options, skill=skill, args=action.args, llm=self._llm) + action = SkillAction(options=self.options, skill=skill, args=action.args, llm=self._llm, name=skill.name, + desc=skill.description) self.add_to_do(action) return True diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index ea6af7e7c..a6e4dc20d 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -260,9 +260,16 @@ def parse_recipient(text): return recipient.group(1) if recipient else "" -def initialize_environment(): +def initialize_environment(options=None): """Load `config/config.yaml` to `os.environ`""" + if options: + for k, v in options.items(): + os.environ[k] = str(v) + return + yaml_file_path = Path(__file__).resolve().parent.parent.parent / "config/config.yaml" + if not yaml_file_path.exists(): + return with open(str(yaml_file_path), "r") as yaml_file: data = yaml.safe_load(yaml_file) for k, v in data.items(): From 4fddfbab581b28736ef851f99bee14f5d1385179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 13:22:34 +0800 Subject: [PATCH 100/398] fixbug: No user feedback, unsure if past conversation is finished. --- metagpt/memory/brain_memory.py | 2 +- metagpt/roles/assistant.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 422c096f3..9d1b038bb 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -53,6 +53,6 @@ class BrainMemory(pydantic.BaseModel): @property def last_talk(self): if len(self.history) == 0 or not self.history[-1].is_contain_tags([MessageType.Talk.value]): - return "" + return None return self.history[-1].content diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 3924039b5..1e503857a 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -113,6 +113,8 @@ class Assistant(Role): async def refine_memory(self) -> str: history_text = self.memory.history_text last_talk = self.memory.last_talk + if last_talk is None: # No user feedback, unsure if past conversation is finished. + return None if history_text == "": return last_talk history_summary = await self._llm.get_context_title(history_text, max_words=20) From 903e89cec36b7d07e1bd52b9894e7c6b7131ca6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 13:27:13 +0800 Subject: [PATCH 101/398] fixbug: No user feedback, unsure if past conversation is finished. --- metagpt/roles/assistant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 1e503857a..199cdcafd 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -130,8 +130,8 @@ class Assistant(Role): from metagpt.provider.openai_api import OpenAIGPTAPI return OpenAIGPTAPI.extract_info(input_string) - def get_memory(self) -> str: - return self.memory.json() + def get_memory(self, exclude=None) -> str: + return self.memory.json(exclude=exclude) def load_memory(self, jsn): try: From 3e9151e52e331978548ba6a9a6527e5569991a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 15:11:28 +0800 Subject: [PATCH 102/398] fixbug: brain memory serialize --- metagpt/memory/brain_memory.py | 30 ++++++------ metagpt/roles/assistant.py | 2 +- metagpt/schema.py | 17 ++++++- tests/metagpt/memory/test_brain_memory.py | 57 +++++++++++++++++++++++ 4 files changed, 90 insertions(+), 16 deletions(-) create mode 100644 tests/metagpt/memory/test_brain_memory.py diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 9d1b038bb..cb67fea8e 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -1,10 +1,11 @@ from enum import Enum -from typing import List +from typing import List, Dict import pydantic from metagpt import Message + class MessageType(Enum): Talk = "TALK" Solution = "SOLUTION" @@ -14,29 +15,28 @@ class MessageType(Enum): class BrainMemory(pydantic.BaseModel): - history: List[Message] = [] - stack: List[Message] = [] - solution: List[Message] = [] - knowledge: List[Message] = [] - + history: List[Dict] = [] + stack: List[Dict] = [] + solution: List[Dict] = [] + knowledge: List[Dict] = [] def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) - self.history.append(msg) + self.history.append(msg.dict()) def add_answer(self, msg: Message): msg.add_tag(MessageType.Answer.value) - self.history.append(msg) + self.history.append(msg.dict()) def get_knowledge(self) -> str: - texts = [k.content for k in self.knowledge] + texts = [Message(**m).content for m in self.knowledge] return "\n".join(texts) @property def history_text(self): if len(self.history) == 0: return "" - texts = [m.content for m in self.history[:-1]] + texts = [Message(**m).content for m in self.history[:-1]] return "\n".join(texts) def move_to_solution(self): @@ -44,7 +44,7 @@ class BrainMemory(pydantic.BaseModel): return msgs = self.history[:-1] self.solution.extend(msgs) - if not self.history[-1].is_contain(MessageType.Talk.value): + if not Message(**self.history[-1]).is_contain(MessageType.Talk.value): self.solution.append(self.history[-1]) self.history = [] else: @@ -52,7 +52,9 @@ class BrainMemory(pydantic.BaseModel): @property def last_talk(self): - if len(self.history) == 0 or not self.history[-1].is_contain_tags([MessageType.Talk.value]): + if len(self.history) == 0: return None - return self.history[-1].content - + last_msg = Message(**self.history[-1]) + if not last_msg.is_contain(MessageType.Talk.value): + return None + return last_msg.content diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 199cdcafd..4519fcdb8 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -77,7 +77,7 @@ class Assistant(Role): return output async def talk(self, text): - self.memory.add_talk(Message(content=text, tags=set([MessageType.Talk.value]))) + self.memory.add_talk(Message(content=text)) async def _plan(self, rsp: str, **kwargs) -> bool: skill, text = Assistant.extract_info(input_string=rsp) diff --git a/metagpt/schema.py b/metagpt/schema.py index 909313886..4d7f0cc21 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -70,6 +70,22 @@ class Message: def is_contain(self, tag): return self.is_contain_tags([tag]) + def dict(self): + """pydantic-like `dict` function""" + full = { + "instruct_content": self.instruct_content, + "cause_by": self.cause_by, + "sent_from": self.sent_from, + "send_to": self.send_to, + "tags": self.tags + } + + m = {"content": self.content} + for k, v in full.items(): + if v: + m[k] = v + return m + @dataclass class UserMessage(Message): @@ -101,7 +117,6 @@ class AIMessage(Message): super().__init__(content, 'assistant') - if __name__ == '__main__': test_content = 'test_message' msgs = [ diff --git a/tests/metagpt/memory/test_brain_memory.py b/tests/metagpt/memory/test_brain_memory.py new file mode 100644 index 000000000..b5fc942ca --- /dev/null +++ b/tests/metagpt/memory/test_brain_memory.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/27 +@Author : mashenquan +@File : test_brain_memory.py +""" +import json +from typing import List + +import pydantic + +from metagpt.memory.brain_memory import BrainMemory +from metagpt.schema import Message + + +def test_json(): + class Input(pydantic.BaseModel): + history: List[str] + solution: List[str] + knowledge: List[str] + stack: List[str] + + inputs = [ + { + "history": ["a", "b"], + "solution": ["c"], + "knowledge": ["d", "e"], + "stack": ["f"] + } + ] + + for i in inputs: + v = Input(**i) + bm = BrainMemory() + for h in v.history: + msg = Message(content=h) + bm.history.append(msg.dict()) + for h in v.solution: + msg = Message(content=h) + bm.solution.append(msg.dict()) + for h in v.knowledge: + msg = Message(content=h) + bm.knowledge.append(msg.dict()) + for h in v.stack: + msg = Message(content=h) + bm.stack.append(msg.dict()) + s = bm.json() + m = json.loads(s) + bm = BrainMemory(**m) + assert bm + for v in bm.history: + msg = Message(**v) + assert msg + +if __name__ == '__main__': + test_json() \ No newline at end of file From 9b890275c4c4a2e1580556113b0928ca871da838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 10:36:13 +0800 Subject: [PATCH 103/398] feat: +x-prerequisite --- .well-known/metagpt_oas3_api.yaml | 14 ++++++++++++++ metagpt/tools/metagpt_text_to_image.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index a226181a5..56c6f42d5 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -12,6 +12,11 @@ servers: paths: /tts/azsure: + x-prerequisite: + - name: AZURE_TTS_SUBSCRIPTION_KEY + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + - name: AZURE_TTS_REGION + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" post: summary: "Convert Text to Base64-encoded .wav File Stream" description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" @@ -69,6 +74,9 @@ paths: description: "Internal Server Error" /txt2img/openai: + x-prerequisite: + - name: OPENAI_API_KEY + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" post: summary: "Convert Text to Base64-encoded Image Data Stream" operationId: openai_text_to_image.oas3_openai_text_to_image @@ -107,6 +115,9 @@ paths: '500': description: "Internal Server Error" /txt2embedding/openai: + x-prerequisite: + - name: OPENAI_API_KEY + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" post: summary: Text to embedding operationId: openai_text_to_embedding.oas3_openai_text_to_embedding @@ -146,6 +157,9 @@ paths: $ref: "#/components/schemas/Error" /txt2image/metagpt: + x-prerequisite: + - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL + description: "Model url." post: summary: "Text to Image" description: "Generate an image from the provided text using the MetaGPT Text-to-Image API." diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index 674ff283a..8588462d3 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -98,7 +98,7 @@ def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): if not text: return "" if not model_url: - model_url = os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL') + model_url = os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL_URL') return MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type) From 13eddeae2fab883106be8547b1b84f8c42903775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 10:42:54 +0800 Subject: [PATCH 104/398] feat: +x-prerequisite --- .well-known/skills.yaml | 16 ++++++++++------ metagpt/learn/text_to_image.py | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 7a035910c..06b9ffd0c 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -4,9 +4,11 @@ entities: - name: text_to_speech description: Text-to-speech id: text_to_speech.text_to_speech - requisite: - - AZURE_TTS_SUBSCRIPTION_KEY - - AZURE_TTS_REGION + x-prerequisite: + - name: AZURE_TTS_SUBSCRIPTION_KEY + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + - name: AZURE_TTS_REGION + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" arguments: text: 'The text used for voice conversion. Required.' lang: 'The value can contain a language code such as en (English), or a locale such as en-US (English - United States). The optional parameter are "English", "Chinese". Default value: "Chinese".' @@ -27,9 +29,11 @@ entities: - name: text_to_image description: Create a drawing based on the text. id: text_to_image.text_to_image - requisite: - - OPENAI_API_KEY - - METAGPT_TEXT_TO_IMAGE_MODEL + x-prerequisite: + - name: OPENAI_API_KEY + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL + description: "Model url." arguments: text: 'The text used for image conversion. Required.' size_type: 'Default value: "512x512".' diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 2f946e239..d245b06db 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -28,7 +28,7 @@ def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url """ initialize_environment() image_declaration = "data:image/png;base64," - if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL") or model_url: + if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL_URL") or model_url: data = oas3_metagpt_text_to_image(text, size_type, model_url) return image_declaration + data if data else "" if os.environ.get("OPENAI_API_KEY") or openai_api_key: From aaf18d2641113bb410a91776948b7fcf810ef2ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 11:04:28 +0800 Subject: [PATCH 105/398] feat: +x-prerequisite --- .well-known/ai-plugin.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.well-known/ai-plugin.json b/.well-known/ai-plugin.json index 44e8435f2..ac0178fd0 100644 --- a/.well-known/ai-plugin.json +++ b/.well-known/ai-plugin.json @@ -9,10 +9,10 @@ }, "api": { "type": "openapi", - "url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/.well-known/metagpt_oas3_api.yaml", + "url": "https://github.com/iorisa/MetaGPT/blob/feature/assistant_role/.well-known/metagpt_oas3_api.yaml", "has_user_authentication": false }, - "logo_url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/docs/resources/MetaGPT-logo.png", + "logo_url": "https://github.com/geekan/MetaGPT/blob/main/docs/resources/MetaGPT-logo.png", "contact_email": "mashenquan@fuzhi.cn", - "legal_info_url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/docs/README_CN.md" + "legal_info_url": "https://github.com/geekan/MetaGPT/blob/main/docs/README_CN.md" } \ No newline at end of file From c67789756147d84d785f09b0e3a33497442d91cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 11:48:38 +0800 Subject: [PATCH 106/398] fixbug: runtime options --- metagpt/roles/role.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index ed02575db..f605f5010 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -14,7 +14,9 @@ from __future__ import annotations from typing import Iterable, Type, Dict from pydantic import BaseModel, Field -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM + +from metagpt.config import Config +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager from metagpt.actions import Action, ActionOutput from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory @@ -100,7 +102,11 @@ class RoleContext(BaseModel): class Role: """Role/Proxy""" - def __init__(self, options, cost_manager, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): + def __init__(self, options=None, cost_manager=None, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): + if not options: + options = Config().runtime_options + if not cost_manager: + cost_manager = CostManager(*options) self._options = Role.supply_options(options=kwargs, default_options=options) name = Role.format_value(name, self._options) From ac744062609d6218b5cac72be5916067667f9d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 11:55:09 +0800 Subject: [PATCH 107/398] =?UTF-8?q?fixbug:=20+=E7=BC=BA=E7=9C=81=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/action.py | 5 +++-- metagpt/roles/role.py | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 86a6664ba..10579d4f4 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -12,13 +12,14 @@ from typing import Optional from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput +from metagpt.config import Config from metagpt.utils.common import OutputParser from metagpt.logs import logger class Action(ABC): - def __init__(self, options, name: str = '', context=None, llm=None): - self.options = options + def __init__(self, options=None, name: str = '', context=None, llm=None): + self.options = options or Config().runtime_options self.name: str = name self.llm = llm self.context = context diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index f605f5010..4f46bb973 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -103,10 +103,9 @@ class Role: """Role/Proxy""" def __init__(self, options=None, cost_manager=None, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): - if not options: - options = Config().runtime_options - if not cost_manager: - cost_manager = CostManager(*options) + options = options or Config().runtime_options + cost_manager = cost_manager or CostManager(*options) + self._options = Role.supply_options(options=kwargs, default_options=options) name = Role.format_value(name, self._options) From b410b9352078cbcf35df8690f241c38311df4840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 12:01:34 +0800 Subject: [PATCH 108/398] =?UTF-8?q?fixbug:=20+=E7=BC=BA=E7=9C=81=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 4519fcdb8..dae516795 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -3,7 +3,7 @@ """ @Time : 2023/8/7 @Author : mashenquan -@File : fork_meta_role.py +@File : assistant.py @Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to make these symbols configurable and standardized, making the process of building flows more convenient. From f17660b12251ddb362d4f3233589093cb61c8cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 12:16:01 +0800 Subject: [PATCH 109/398] fixbug: get_memory --- metagpt/roles/assistant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index dae516795..d6f52e4e4 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -130,8 +130,8 @@ class Assistant(Role): from metagpt.provider.openai_api import OpenAIGPTAPI return OpenAIGPTAPI.extract_info(input_string) - def get_memory(self, exclude=None) -> str: - return self.memory.json(exclude=exclude) + def get_memory(self) -> str: + return self.memory.json() def load_memory(self, jsn): try: From 6acf3f628238f960d8771efcd6ad9f035c3af7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 12:36:31 +0800 Subject: [PATCH 110/398] fixbug: get_memory --- metagpt/schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 4d7f0cc21..ce08455fc 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -74,7 +74,6 @@ class Message: """pydantic-like `dict` function""" full = { "instruct_content": self.instruct_content, - "cause_by": self.cause_by, "sent_from": self.sent_from, "send_to": self.send_to, "tags": self.tags From 6794645ff63e6b28f17492907c60755c447d2c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 15:42:47 +0800 Subject: [PATCH 111/398] =?UTF-8?q?feat:=20=E6=94=B9=E5=BC=82=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/learn/text_to_embedding.py | 4 ++-- metagpt/learn/text_to_image.py | 7 +++--- metagpt/learn/text_to_speech.py | 4 ++-- metagpt/tools/azure_tts.py | 17 +++++++------ metagpt/tools/hello.py | 2 +- metagpt/tools/metagpt_oas3_api_svc.py | 2 +- metagpt/tools/metagpt_text_to_image.py | 13 +++++----- metagpt/tools/openai_text_to_embedding.py | 19 ++++++++------- metagpt/tools/openai_text_to_image.py | 24 ++++++++++--------- requirements.txt | 1 + tests/metagpt/learn/test_text_to_embedding.py | 4 ++-- tests/metagpt/learn/test_text_to_image.py | 11 +++++++-- tests/metagpt/learn/test_text_to_speech.py | 11 +++++++-- tests/metagpt/tools/test_azure_tts.py | 9 ++++--- 14 files changed, 78 insertions(+), 50 deletions(-) diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py index 6d0cefcdb..5c08ef0b9 100644 --- a/metagpt/learn/text_to_embedding.py +++ b/metagpt/learn/text_to_embedding.py @@ -16,7 +16,7 @@ from metagpt.utils.common import initialize_environment @skill_metadata(name="Text to Embedding", description="Convert the text into embeddings.", requisite="`OPENAI_API_KEY`") -def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", **kwargs): +async def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", **kwargs): """Text to embedding :param text: The text used for embedding. @@ -26,5 +26,5 @@ def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", * """ initialize_environment() if os.environ.get("OPENAI_API_KEY") or openai_api_key: - return oas3_openai_text_to_embedding(text, model=model, openai_api_key=openai_api_key) + return await oas3_openai_text_to_embedding(text, model=model, openai_api_key=openai_api_key) raise EnvironmentError diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index d245b06db..db9844c71 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -17,7 +17,7 @@ from metagpt.utils.common import initialize_environment @skill_metadata(name="Text to image", description="Create a drawing based on the text.", requisite="`OPENAI_API_KEY` or `METAGPT_TEXT_TO_IMAGE_MODEL`") -def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs): +async def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs): """Text to image :param text: The text used for image conversion. @@ -29,10 +29,11 @@ def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url initialize_environment() image_declaration = "data:image/png;base64," if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL_URL") or model_url: - data = oas3_metagpt_text_to_image(text, size_type, model_url) + data = await oas3_metagpt_text_to_image(text, size_type, model_url) return image_declaration + data if data else "" + if os.environ.get("OPENAI_API_KEY") or openai_api_key: - data = oas3_openai_text_to_image(text, size_type, openai_api_key) + data = await oas3_openai_text_to_image(text, size_type, openai_api_key) return image_declaration + data if data else "" raise EnvironmentError diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 90dd878a1..e5eb3d488 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -16,7 +16,7 @@ from metagpt.utils.common import initialize_environment @skill_metadata(name="Text to speech", description="Text-to-speech", requisite="`AZURE_TTS_SUBSCRIPTION_KEY` and `AZURE_TTS_REGION`") -def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", +async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", subscription_key="", region="", **kwargs): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -35,7 +35,7 @@ def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affect audio_declaration = "data:audio/wav;base64," if (os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") and os.environ.get("AZURE_TTS_REGION")) or \ (subscription_key and region): - data = oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) return audio_declaration + data if data else data raise EnvironmentError diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 21e8f1b6c..1fd36e78c 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -6,6 +6,7 @@ @File : azure_tts.py @Desc : azure TTS OAS3 api, which provides text-to-speech functionality """ +import asyncio from pathlib import Path from uuid import uuid4 import base64 @@ -14,7 +15,7 @@ import sys sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.utils.common import initialize_environment from metagpt.logs import logger - +from aiofile import async_open from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer import os @@ -31,7 +32,7 @@ class AzureTTS: self.region = region if region else os.environ.get('AZURE_TTS_REGION') # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles - def synthesize_speech(self, lang, voice, text, output_file): + async def synthesize_speech(self, lang, voice, text, output_file): speech_config = SpeechConfig( subscription=self.subscription_key, region=self.region) speech_config.speech_synthesis_voice_name = voice @@ -61,7 +62,7 @@ class AzureTTS: # Export -def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): +async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -95,9 +96,9 @@ def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key tts = AzureTTS(subscription_key=subscription_key, region=region) filename = Path(__file__).resolve().parent / (str(uuid4()).replace("-", "") + ".wav") try: - tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename)) - with open(str(filename), mode="rb") as reader: - data = reader.read() + await tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename)) + async with async_open(filename, mode="rb") as reader: + data = await reader.read() base64_string = base64.b64encode(data).decode('utf-8') filename.unlink() except Exception as e: @@ -110,5 +111,7 @@ def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key if __name__ == "__main__": initialize_environment() - v = oas3_azsure_tts("测试，test") + loop = asyncio.new_event_loop() + v = loop.create_task(oas3_azsure_tts("测试，test")) + loop.run_until_complete(v) print(v) diff --git a/metagpt/tools/hello.py b/metagpt/tools/hello.py index e1bad6456..2eb4c31f0 100644 --- a/metagpt/tools/hello.py +++ b/metagpt/tools/hello.py @@ -17,7 +17,7 @@ import connexion # openapi implement -def post_greeting(name: str) -> str: +async def post_greeting(name: str) -> str: return f"Hello {name}\n" diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index 277d41dfb..624bb7d93 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -20,7 +20,7 @@ def oas_http_svc(): """Start the OAS 3.0 OpenAPI HTTP service""" initialize_environment() - app = connexion.FlaskApp(__name__, specification_dir='../../.well-known/') + app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') app.add_api("metagpt_oas3_api.yaml") app.add_api("openapi.yaml") app.run(port=8080) diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index 8588462d3..bc551134a 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -12,6 +12,7 @@ import sys from pathlib import Path from typing import List, Dict +import aiohttp import requests from pydantic import BaseModel @@ -27,7 +28,7 @@ class MetaGPTText2Image: """ self.model_url = model_url if model_url else os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL') - def text_2_image(self, text, size_type="512x512"): + async def text_2_image(self, text, size_type="512x512"): """Text to image :param text: The text used for image conversion. @@ -75,9 +76,9 @@ class MetaGPTText2Image: parameters: Dict try: - response = requests.post(self.model_url, headers=headers, json=data) - response.raise_for_status() # Raise an exception for 4xx or 5xx responses - result = ImageResult(**response.json()) + async with aiohttp.ClientSession() as session: + async with session.post(self.model_url, headers=headers, json=data) as response: + result = ImageResult(**await response.json()) if len(result.images) == 0: return "" return result.images[0] @@ -87,7 +88,7 @@ class MetaGPTText2Image: # Export -def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): +async def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): """Text to image :param text: The text used for image conversion. @@ -99,7 +100,7 @@ def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): return "" if not model_url: model_url = os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL_URL') - return MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type) + return await MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type) if __name__ == "__main__": diff --git a/metagpt/tools/openai_text_to_embedding.py b/metagpt/tools/openai_text_to_embedding.py index 9eddd5bc1..119eb35b6 100644 --- a/metagpt/tools/openai_text_to_embedding.py +++ b/metagpt/tools/openai_text_to_embedding.py @@ -7,10 +7,12 @@ @Desc : OpenAI Text-to-Embedding OAS3 api, which provides text-to-embedding functionality. For more details, checkout: `https://platform.openai.com/docs/api-reference/embeddings/object` """ +import asyncio import os from pathlib import Path from typing import List +import aiohttp import requests from pydantic import BaseModel import sys @@ -47,7 +49,7 @@ class OpenAIText2Embedding: """ self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') - def text_2_embedding(self, text, model="text-embedding-ada-002"): + async def text_2_embedding(self, text, model="text-embedding-ada-002"): """Text to embedding :param text: The text used for embedding. @@ -61,16 +63,16 @@ class OpenAIText2Embedding: } data = {"input": text, "model": model} try: - response = requests.post("https://api.openai.com/v1/embeddings", headers=headers, json=data) - response.raise_for_status() # Raise an exception for 4xx or 5xx responses - return response.json() + async with aiohttp.ClientSession() as session: + async with session.post("https://api.openai.com/v1/embeddings", headers=headers, json=data) as response: + return await response.json() except requests.exceptions.RequestException as e: logger.error(f"An error occurred:{e}") return {} # Export -def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): +async def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): """Text to embedding :param text: The text used for embedding. @@ -82,11 +84,12 @@ def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", openai_a return "" if not openai_api_key: openai_api_key = os.environ.get("OPENAI_API_KEY") - return OpenAIText2Embedding(openai_api_key).text_2_embedding(text, model=model) + return await OpenAIText2Embedding(openai_api_key).text_2_embedding(text, model=model) if __name__ == "__main__": initialize_environment() - - v = oas3_openai_text_to_embedding("Panda emoji") + loop = asyncio.new_event_loop() + v = loop.create_task(oas3_openai_text_to_embedding("Panda emoji")) + loop.run_until_complete(v) print(v) diff --git a/metagpt/tools/openai_text_to_image.py b/metagpt/tools/openai_text_to_image.py index 6ec96d166..cd48c62af 100644 --- a/metagpt/tools/openai_text_to_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -12,6 +12,7 @@ import sys from pathlib import Path from typing import List +import aiohttp import requests from pydantic import BaseModel @@ -27,7 +28,7 @@ class OpenAIText2Image: """ self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') - def text_2_image(self, text, size_type="1024x1024"): + async def text_2_image(self, text, size_type="1024x1024"): """Text to image :param text: The text used for image conversion. @@ -48,27 +49,28 @@ class OpenAIText2Image: } data = {"prompt": text, "n": 1, "size": size_type} try: - response = requests.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) - response.raise_for_status() # Raise an exception for 4xx or 5xx responses - result = ImageResult(**response.json()) + async with aiohttp.ClientSession() as session: + async with session.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) as response: + result = ImageResult(** await response.json()) except requests.exceptions.RequestException as e: logger.error(f"An error occurred:{e}") return "" if len(result.data) > 0: - return OpenAIText2Image.get_image_data(result.data[0].url) + return await OpenAIText2Image.get_image_data(result.data[0].url) return "" @staticmethod - def get_image_data(url): + async def get_image_data(url): """Fetch image data from a URL and encode it as Base64 :param url: Image url :return: Base64-encoded image data. """ try: - response = requests.get(url) - response.raise_for_status() # Raise an exception for 4xx or 5xx responses - image_data = response.content + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + response.raise_for_status() # 如果是 4xx 或 5xx 响应，会引发异常 + image_data = await response.read() base64_image = base64.b64encode(image_data).decode("utf-8") return base64_image @@ -78,7 +80,7 @@ class OpenAIText2Image: # Export -def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_api_key=""): +async def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_api_key=""): """Text to image :param text: The text used for image conversion. @@ -90,7 +92,7 @@ def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_api_key return "" if not openai_api_key: openai_api_key = os.environ.get("OPENAI_API_KEY") - return OpenAIText2Image(openai_api_key).text_2_image(text, size_type=size_type) + return await OpenAIText2Image(openai_api_key).text_2_image(text, size_type=size_type) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 70f2a3809..ed3f755c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,3 +41,4 @@ qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 azure-cognitiveservices-speech==1.30.0 +aiofile \ No newline at end of file diff --git a/tests/metagpt/learn/test_text_to_embedding.py b/tests/metagpt/learn/test_text_to_embedding.py index c85e5dde8..d81a8ac1c 100644 --- a/tests/metagpt/learn/test_text_to_embedding.py +++ b/tests/metagpt/learn/test_text_to_embedding.py @@ -8,11 +8,11 @@ """ import asyncio -import base64 from pydantic import BaseModel from metagpt.learn.text_to_embedding import text_to_embedding +from metagpt.tools.openai_text_to_embedding import ResultEmbedding async def mock_text_to_embedding(): @@ -25,7 +25,7 @@ async def mock_text_to_embedding(): for i in inputs: seed = Input(**i) - data = text_to_embedding(seed.input) + data = await text_to_embedding(seed.input) v = ResultEmbedding(**data) assert len(v.data) > 0 diff --git a/tests/metagpt/learn/test_text_to_image.py b/tests/metagpt/learn/test_text_to_image.py index 545c8a3ef..c359797de 100644 --- a/tests/metagpt/learn/test_text_to_image.py +++ b/tests/metagpt/learn/test_text_to_image.py @@ -25,10 +25,17 @@ async def mock_text_to_image(): for i in inputs: seed = Input(**i) - base64_data = text_to_image(seed.input) + base64_data = await text_to_image(seed.input) assert base64_data != "" print(f"{seed.input} -> {base64_data}") - assert base64.b64decode(base64_data, validate=True) + flags = ";base64," + assert flags in base64_data + ix = base64_data.find(flags) + len(flags) + declaration = base64_data[0: ix] + assert declaration + data = base64_data[ix:] + assert data + assert base64.b64decode(data, validate=True) def test_suite(): diff --git a/tests/metagpt/learn/test_text_to_speech.py b/tests/metagpt/learn/test_text_to_speech.py index dbb599e38..68de5a3b2 100644 --- a/tests/metagpt/learn/test_text_to_speech.py +++ b/tests/metagpt/learn/test_text_to_speech.py @@ -24,10 +24,17 @@ async def mock_text_to_speech(): for i in inputs: seed = Input(**i) - base64_data = text_to_speech(seed.input) + base64_data = await text_to_speech(seed.input) assert base64_data != "" print(f"{seed.input} -> {base64_data}") - assert base64.b64decode(base64_data, validate=True) + flags = ";base64," + assert flags in base64_data + ix = base64_data.find(flags) + len(flags) + declaration = base64_data[0: ix] + assert declaration + data = base64_data[ix:] + assert data + assert base64.b64decode(data, validate=True) def test_suite(): diff --git a/tests/metagpt/tools/test_azure_tts.py b/tests/metagpt/tools/test_azure_tts.py index 49dd7eed1..41d429109 100644 --- a/tests/metagpt/tools/test_azure_tts.py +++ b/tests/metagpt/tools/test_azure_tts.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023-8-9, add more text formatting options @Modified By: mashenquan, 2023-8-17, move to `tools` folder. """ +import asyncio import sys from pathlib import Path @@ -19,7 +20,7 @@ from metagpt.utils.common import initialize_environment def test_azure_tts(): initialize_environment() - azure_tts = AzureTTS() + azure_tts = AzureTTS(subscription_key="", region="") text = """ 女儿看见父亲走了进来，问道： @@ -33,11 +34,13 @@ def test_azure_tts(): path = WORKSPACE_ROOT / "tts" path.mkdir(exist_ok=True, parents=True) filename = path / "girl.wav" - result = azure_tts.synthesize_speech( + loop = asyncio.new_event_loop() + v = loop.create_task(azure_tts.synthesize_speech( lang="zh-CN", voice="zh-CN-XiaomoNeural", text=text, - output_file=str(filename)) + output_file=str(filename))) + result = loop.run_until_complete(v) print(result) From 3a1ebf19b7858f3d3156a7d29767200b23db5199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 16:48:59 +0800 Subject: [PATCH 112/398] feat: +OPTIONS --- metagpt/config.py | 73 ++++++++++++++++++++++------------------------- metagpt/const.py | 3 ++ 2 files changed, 37 insertions(+), 39 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 31488b466..ceaa582e2 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -1,18 +1,17 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Desc: Provide configuration, singleton. -@Modified By: mashenquan, replace `CONFIG` with `os.environ` to support personal config - `os.environ` doesn't support personalization, while `Config` does. - Hence, the parameter reading priority is `Config` first, and if not found, then `os.environ`. -@Modified By: mashenquan, 2023/8/23. Add `options` to `Config.__init__` to support externally specified options. +Provide configuration, singleton. +@Modified BY: mashenquan, 2023/8/28. Replace the global variable `CONFIG` with `ContextVar`. """ import os +from copy import deepcopy +from typing import Any import openai import yaml -from metagpt.const import PROJECT_ROOT +from metagpt.const import PROJECT_ROOT, OPTIONS from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.singleton import Singleton @@ -30,34 +29,26 @@ class NotConfiguredException(Exception): super().__init__(self.message) -class Config: +class Config(metaclass=Singleton): """ - For example: - - ```python + Usual Usage: config = Config("config.yaml") secret_key = config.get_key("MY_SECRET_KEY") print("Secret key:", secret_key) - ``` """ + _instance = None key_yaml_file = PROJECT_ROOT / "config/key.yaml" default_yaml_file = PROJECT_ROOT / "config/config.yaml" - def __init__(self, yaml_file=default_yaml_file, options=None): - self._configs = {} - self._init_with_config_files_and_env(self._configs, yaml_file) - if options: - self._configs.update(options) - self._parse() - - def _parse(self): + def __init__(self, yaml_file=default_yaml_file): + self._init_with_config_files_and_env(yaml_file) logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -94,41 +85,45 @@ class Config: self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") - def _init_with_config_files_and_env(self, configs: dict, yaml_file): - """Load in decreasing priority from `config/key.yaml`, `config/config.yaml`, and environment variables.""" - configs.update(os.environ) + def _init_with_config_files_and_env(self, yaml_file): + """从config/key.yaml / config/config.yaml / env三处按优先级递减加载""" + configs = dict(os.environ) for _yaml_file in [yaml_file, self.key_yaml_file]: if not _yaml_file.exists(): continue - # Load local YAML file. + # 加载本地 YAML 文件 with open(_yaml_file, "r", encoding="utf-8") as file: yaml_data = yaml.safe_load(file) if not yaml_data: continue configs.update(yaml_data) + OPTIONS.set(configs) - def _get(self, *args, **kwargs): - return self._configs.get(*args, **kwargs) + @staticmethod + def _get(*args, **kwargs): + m = OPTIONS.get() + return m.get(*args, **kwargs) def get(self, key, *args, **kwargs): - """Retrieve value from `config/key.yaml`, `config/config.yaml`, and environment variables. - Raise an error if not found.""" + """Retrieve values from config/key.yaml, config/config.yaml, and environment variables. Throw an error if not found.""" value = self._get(key, *args, **kwargs) if value is None: raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") return value - @property - def runtime_options(self): - """Runtime key-value configuration parameters.""" - opts = {} - for k, v in self._configs.items(): - opts[k] = v - for attribute, value in vars(self).items(): - if attribute == "_configs": - continue - opts[attribute] = value - return opts + def __setattr__(self, name: str, value: Any) -> None: + OPTIONS.get()[name] = value + def __getattr__(self, name: str) -> Any: + m = OPTIONS.get() + return m.get(name) + + def set_context(self, options: dict): + """Update current config""" + opts = deepcopy(OPTIONS.get()) + opts.update(options) + OPTIONS.set(opts) + +CONFIG = Config() diff --git a/metagpt/const.py b/metagpt/const.py index 505eebd46..20513461a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -5,6 +5,7 @@ @Author : alexanderwu @File : const.py """ +import contextvars from pathlib import Path @@ -35,3 +36,5 @@ TMP = PROJECT_ROOT / 'tmp' RESEARCH_PATH = DATA_PATH / "research" MEM_TTL = 24 * 30 * 3600 + +OPTIONS = contextvars.ContextVar("OPTIONS") From 143ffb0c2cecde75d56d3098044b1cbe1ae5bbe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 17:45:45 +0800 Subject: [PATCH 113/398] feat: replaced with OPTIONS --- metagpt/actions/action.py | 9 ++-- metagpt/actions/action_output.py | 1 + metagpt/actions/analyze_dep_libs.py | 5 +- metagpt/actions/debug_error.py | 5 +- metagpt/actions/design_api.py | 11 ++--- metagpt/actions/design_api_review.py | 5 +- metagpt/actions/design_filenames.py | 5 +- metagpt/actions/project_management.py | 5 +- metagpt/actions/research.py | 24 ++++----- metagpt/actions/run_code.py | 5 +- metagpt/actions/search_and_summarize.py | 8 +-- metagpt/actions/skill_action.py | 19 ++++++-- metagpt/actions/write_code.py | 5 +- metagpt/actions/write_code_review.py | 5 +- metagpt/actions/write_prd.py | 7 ++- metagpt/actions/write_prd_review.py | 5 +- metagpt/actions/write_teaching_plan.py | 2 +- metagpt/actions/write_test.py | 5 +- metagpt/learn/skill_metadata.py | 25 ---------- metagpt/learn/text_to_embedding.py | 10 +--- metagpt/learn/text_to_image.py | 11 ++--- metagpt/learn/text_to_speech.py | 13 ++--- metagpt/llm.py | 20 ++++++++ metagpt/manager.py | 5 +- metagpt/roles/architect.py | 6 +-- metagpt/roles/customer_service.py | 4 +- metagpt/roles/engineer.py | 6 +-- metagpt/roles/product_manager.py | 6 +-- metagpt/roles/project_manager.py | 6 +-- metagpt/roles/qa_engineer.py | 4 +- metagpt/roles/researcher.py | 9 +--- metagpt/roles/role.py | 59 +++++++---------------- metagpt/roles/sales.py | 4 +- metagpt/roles/seacher.py | 4 +- metagpt/roles/teacher.py | 6 +-- metagpt/software_company.py | 30 +++--------- metagpt/tools/openai_text_to_embedding.py | 6 +-- metagpt/utils/common.py | 15 ------ startup.py | 16 ++---- 39 files changed, 144 insertions(+), 252 deletions(-) delete mode 100644 metagpt/learn/skill_metadata.py create mode 100644 metagpt/llm.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 10579d4f4..5cf4f3d81 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -4,7 +4,7 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : action.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +@Modified By: mashenquan, 2023/8/20. Add function return annotations. """ from abc import ABC from typing import Optional @@ -12,15 +12,16 @@ from typing import Optional from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput -from metagpt.config import Config +from metagpt.llm import LLM from metagpt.utils.common import OutputParser from metagpt.logs import logger class Action(ABC): - def __init__(self, options=None, name: str = '', context=None, llm=None): - self.options = options or Config().runtime_options + def __init__(self, name: str = '', context=None, llm: LLM = None): self.name: str = name + if llm is None: + llm = LLM() self.llm = llm self.context = context self.prefix = "" diff --git a/metagpt/actions/action_output.py b/metagpt/actions/action_output.py index 6c812e7fe..917368798 100644 --- a/metagpt/actions/action_output.py +++ b/metagpt/actions/action_output.py @@ -4,6 +4,7 @@ @Time : 2023/7/11 10:03 @Author : chengmaoyu @File : action_output +@Modified By: mashenquan, 2023/8/20. Allow 'instruct_content' to be blank. """ from typing import Dict, Type, Optional diff --git a/metagpt/actions/analyze_dep_libs.py b/metagpt/actions/analyze_dep_libs.py index d7b251ead..23c35cdf8 100644 --- a/metagpt/actions/analyze_dep_libs.py +++ b/metagpt/actions/analyze_dep_libs.py @@ -4,7 +4,6 @@ @Time : 2023/5/19 12:01 @Author : alexanderwu @File : analyze_dep_libs.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import Action @@ -27,8 +26,8 @@ Focus only on the names of shared dependencies, do not add any other explanation class AnalyzeDepLibs(Action): - def __init__(self, options, name, context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) self.desc = "根据上下文，分析程序运行依赖库" async def run(self, requirement, filepaths_string): diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 78c970337..d69a22dba 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : debug_error.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import re @@ -26,8 +25,8 @@ Now you should start rewriting the code: ## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. """ class DebugError(Action): - def __init__(self, options, name="DebugError", context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="DebugError", context=None, llm=None): + super().__init__(name, context, llm) # async def run(self, code, error): # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index a01e1c753..cf23e6ad1 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -5,7 +5,6 @@ @Author : alexanderwu @File : design_api.py @Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import shutil from pathlib import Path @@ -92,8 +91,8 @@ OUTPUT_MAPPING = { class WriteDesign(Action): - def __init__(self, options, name, context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) self.desc = "Based on the PRD, think about the system design, and design the corresponding APIs, " \ "data structures, library tables, processes, and paths. Please provide your design, feedback " \ "clearly and in detail." @@ -108,15 +107,15 @@ class WriteDesign(Action): def _save_prd(self, docs_path, resources_path, prd): prd_file = docs_path / 'prd.md' quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) - mermaid_to_file(options=self.options, mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / 'competitive_analysis') + mermaid_to_file(quadrant_chart, resources_path / 'competitive_analysis') logger.info(f"Saving PRD to {prd_file}") prd_file.write_text(prd) def _save_system_design(self, docs_path, resources_path, content): data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content) seq_flow = CodeParser.parse_code(block="Program call flow", text=content) - mermaid_to_file(options=self.options, mermaid_code=data_api_design, output_file_without_suffix=resources_path / 'data_api_design') - mermaid_to_file(options=self.options, mermaid_code=seq_flow, output_file_without_suffix=resources_path / 'seq_flow') + mermaid_to_file(data_api_design, resources_path / 'data_api_design') + mermaid_to_file(seq_flow, resources_path / 'seq_flow') system_design_file = docs_path / 'system_design.md' logger.info(f"Saving System Designs to {system_design_file}") system_design_file.write_text(content) diff --git a/metagpt/actions/design_api_review.py b/metagpt/actions/design_api_review.py index ca4147cca..687a33652 100644 --- a/metagpt/actions/design_api_review.py +++ b/metagpt/actions/design_api_review.py @@ -4,14 +4,13 @@ @Time : 2023/5/11 19:31 @Author : alexanderwu @File : design_api_review.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action class DesignReview(Action): - def __init__(self, options, name, context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) async def run(self, prd, api_design): prompt = f"Here is the Product Requirement Document (PRD):\n\n{prd}\n\nHere is the list of APIs designed " \ diff --git a/metagpt/actions/design_filenames.py b/metagpt/actions/design_filenames.py index 1f71e9530..6c3d8e803 100644 --- a/metagpt/actions/design_filenames.py +++ b/metagpt/actions/design_filenames.py @@ -4,7 +4,6 @@ @Time : 2023/5/19 11:50 @Author : alexanderwu @File : design_filenames.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import Action from metagpt.logs import logger @@ -16,8 +15,8 @@ Do not add any other explanations, just return a Python string list.""" class DesignFilenames(Action): - def __init__(self, options, name, context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) self.desc = "Based on the PRD, consider system design, and carry out the basic design of the corresponding " \ "APIs, data structures, and database tables. Please give your design, feedback clearly and in detail." diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index d17bf6b03..16473ff01 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -5,7 +5,6 @@ @Author : alexanderwu @File : project_management.py @Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import List, Tuple @@ -105,8 +104,8 @@ OUTPUT_MAPPING = { class WriteTasks(Action): - def __init__(self, options, name="CreateTasks", context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="CreateTasks", context=None, llm=None): + super().__init__(name, context, llm) def _save(self, context, rsp): ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) diff --git a/metagpt/actions/research.py b/metagpt/actions/research.py index 22b0eaa1d..81eb876dd 100644 --- a/metagpt/actions/research.py +++ b/metagpt/actions/research.py @@ -1,9 +1,5 @@ #!/usr/bin/env python -""" -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. -""" - from __future__ import annotations import asyncio @@ -13,6 +9,7 @@ from typing import Callable from pydantic import parse_obj_as from metagpt.actions import Action +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.tools.search_engine import SearchEngine from metagpt.tools.web_browser_engine import WebBrowserEngine, WebBrowserEngineType @@ -82,15 +79,14 @@ class CollectLinks(Action): """Action class to collect links from a search engine.""" def __init__( self, - options, name: str = "", *args, rank_func: Callable[[list[str]], None] | None = None, **kwargs, ): - super().__init__(options=options, name=name, *args, **kwargs) + super().__init__(name, *args, **kwargs) self.desc = "Collect links from a search engine." - self.search_engine = SearchEngine(options=options) + self.search_engine = SearchEngine() self.rank_func = rank_func async def run( @@ -130,7 +126,7 @@ class CollectLinks(Action): remove.pop() if len(remove) == 0: break - prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, self.options.get("max_tokens_rsp")) + prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, CONFIG.max_tokens_rsp) logger.debug(prompt) queries = await self._aask(prompt, [system_text]) try: @@ -182,10 +178,9 @@ class WebBrowseAndSummarize(Action): **kwargs, ): super().__init__(*args, **kwargs) - if self.options.get("model_for_researcher_summary"): - self.llm.model = self.options.get("model_for_researcher_summary") + if CONFIG.model_for_researcher_summary: + self.llm.model = CONFIG.model_for_researcher_summary self.web_browser_engine = WebBrowserEngine( - options=self.options, engine=WebBrowserEngineType.CUSTOM if browse_func else None, run_func=browse_func, ) @@ -218,8 +213,7 @@ class WebBrowseAndSummarize(Action): for u, content in zip([url, *urls], contents): content = content.inner_text chunk_summaries = [] - for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, - self.options.get("max_tokens_rsp")): + for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, CONFIG.max_tokens_rsp): logger.debug(prompt) summary = await self._aask(prompt, [system_text]) if summary == "Not relevant.": @@ -245,8 +239,8 @@ class ConductResearch(Action): """Action class to conduct research and generate a research report.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if self.options.get("model_for_researcher_report"): - self.llm.model = self.options.get("model_for_researcher_report") + if CONFIG.model_for_researcher_report: + self.llm.model = CONFIG.model_for_researcher_report async def run( self, diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 824ed83fa..f69d2cd1a 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : run_code.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import os import subprocess @@ -58,8 +57,8 @@ standard errors: {errs}; class RunCode(Action): - def __init__(self, options, name="RunCode", context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="RunCode", context=None, llm=None): + super().__init__(name, context, llm) @classmethod async def run_text(cls, code) -> Tuple[str, str]: diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 80d1c52e4..9f54587fa 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -101,16 +101,16 @@ You are a member of a professional butler team and will provide helpful suggesti class SearchAndSummarize(Action): - def __init__(self, options, name="", context=None, llm=None, engine=None, search_func=None): - self.engine = engine or options.get("search_engine") + def __init__(self, name="", context=None, llm=None, engine=None, search_func=None): + self.engine = engine or CONFIG.search_engine try: - self.search_engine = SearchEngine(options=options, engine=self.engine, run_func=search_func) + self.search_engine = SearchEngine(self.engine, run_func=search_func) except pydantic.ValidationError: self.search_engine = None self.result = "" - super().__init__(options=options, name=name, context=context, llm=llm) + super().__init__(name, context, llm) async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYSTEM) -> str: if self.search_engine is None: diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index 8cc7b6c42..c921a5f17 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/28 +@Author : mashenquan +@File : skill_action.py +@Desc : Call learned skill +""" + import ast import importlib @@ -7,8 +16,8 @@ from metagpt.logs import logger class ArgumentsParingAction(Action): - def __init__(self, options, last_talk: str, skill: Skill, context=None, llm=None, **kwargs): - super(ArgumentsParingAction, self).__init__(options=options, name='', context=context, llm=llm) + def __init__(self, last_talk: str, skill: Skill, context=None, llm=None, **kwargs): + super(ArgumentsParingAction, self).__init__(name='', context=context, llm=llm) self.skill = skill self.ask = last_talk self.rsp = None @@ -59,15 +68,15 @@ class ArgumentsParingAction(Action): class SkillAction(Action): - def __init__(self, options, skill: Skill, args: dict, context=None, llm=None, **kwargs): - super(SkillAction, self).__init__(options=options, name='', context=context, llm=llm) + def __init__(self, skill: Skill, args: dict, context=None, llm=None, **kwargs): + super(SkillAction, self).__init__(name='', context=context, llm=llm) self._skill = skill self._args = args self.rsp = None async def run(self, *args, **kwargs) -> str | ActionOutput | None: """Run action""" - self.rsp = self.find_and_call_function(self._skill.name, args=self._args, **self.options) + self.rsp = self.find_and_call_function(self._skill.name, args=self._args, **kwargs) return ActionOutput(content=self.rsp, instruct_content=self._skill.json()) @staticmethod diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 9a2a2f81a..cc122ef7a 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import WriteDesign from metagpt.actions.action import Action @@ -44,8 +43,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc class WriteCode(Action): - def __init__(self, options, name="WriteCode", context: list[Message] = None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="WriteCode", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) def _is_invalid(self, filename): return any(i in filename for i in ["mp3", "wav"]) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index d256c6bcb..7f6a7a38e 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code_review.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action @@ -63,8 +62,8 @@ FORMAT_EXAMPLE = """ class WriteCodeReview(Action): - def __init__(self, options, name="WriteCodeReview", context: list[Message] = None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="WriteCodeReview", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt): diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 794d3ee9d..0edd24d55 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import List, Tuple @@ -128,11 +127,11 @@ OUTPUT_MAPPING = { class WritePRD(Action): - def __init__(self, options, name="", context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="", context=None, llm=None): + super().__init__(name, context, llm) async def run(self, requirements, *args, **kwargs) -> ActionOutput: - sas = SearchAndSummarize(options=self.options, llm=self.llm) + sas = SearchAndSummarize() # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) rsp = "" info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" diff --git a/metagpt/actions/write_prd_review.py b/metagpt/actions/write_prd_review.py index 8c22f9c0a..5ff9624c5 100644 --- a/metagpt/actions/write_prd_review.py +++ b/metagpt/actions/write_prd_review.py @@ -4,14 +4,13 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd_review.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action class WritePRDReview(Action): - def __init__(self, options, name, context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) self.prd = None self.desc = "Based on the PRD, conduct a PRD Review, providing clear and detailed feedback" self.prd_review_prompt_template = """ diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 53371b5a1..bd8507350 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -42,7 +42,7 @@ class WriteTeachingPlanPart(Action): statements = [] from metagpt.roles import Role for p in statement_patterns: - s = Role.format_value(p, kwargs) + s = Role.format_value(p) statements.append(s) formatter = self.PROMPT_TITLE_TEMPLATE if self.topic == self.COURSE_TITLE else self.PROMPT_TEMPLATE prompt = formatter.format(formation=self.FORMATION, diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 94006005f..5e50fdb55 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_test.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action from metagpt.utils.common import CodeParser @@ -31,8 +30,8 @@ you should correctly import the necessary classes based on these file locations! class WriteTest(Action): - def __init__(self, options, name="WriteTest", context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="WriteTest", context=None, llm=None): + super().__init__(name, context, llm) async def write_code(self, prompt): code_rsp = await self._aask(prompt) diff --git a/metagpt/learn/skill_metadata.py b/metagpt/learn/skill_metadata.py deleted file mode 100644 index dea5fb04d..000000000 --- a/metagpt/learn/skill_metadata.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/20 -@Author : mashenquan -@File : skill_metadata.py -@Desc : Defines metadata for the `skill`. - Depending on the context and specific circumstances, skills may have different effects. - For example: - Proprietor: "Skill of the proprietor entity." - Holder: "Skill of the holder entity." - Possessor: "Skill of the possessor entity." - Controller: "Skill of the controller entity." - Owner: "Skill of the owner entity." -""" - - -def skill_metadata(name, description, requisite): - def decorator(func): - func.skill_name = name - func.skill_description = description - func.skill_requisite = requisite - return func - - return decorator diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py index 5c08ef0b9..26dab0419 100644 --- a/metagpt/learn/text_to_embedding.py +++ b/metagpt/learn/text_to_embedding.py @@ -6,16 +6,11 @@ @File : text_to_embedding.py @Desc : Text-to-Embedding skill, which provides text-to-embedding functionality. """ -import os -from metagpt.learn.skill_metadata import skill_metadata +from metagpt.config import CONFIG from metagpt.tools.openai_text_to_embedding import oas3_openai_text_to_embedding -from metagpt.utils.common import initialize_environment -@skill_metadata(name="Text to Embedding", - description="Convert the text into embeddings.", - requisite="`OPENAI_API_KEY`") async def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", **kwargs): """Text to embedding @@ -24,7 +19,6 @@ async def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. """ - initialize_environment() - if os.environ.get("OPENAI_API_KEY") or openai_api_key: + if CONFIG.OPENAI_API_KEY or openai_api_key: return await oas3_openai_text_to_embedding(text, model=model, openai_api_key=openai_api_key) raise EnvironmentError diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index db9844c71..2762c2f18 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -8,15 +8,11 @@ """ import os -from metagpt.learn.skill_metadata import skill_metadata +from metagpt.config import CONFIG from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image from metagpt.tools.openai_text_to_image import oas3_openai_text_to_image -from metagpt.utils.common import initialize_environment -@skill_metadata(name="Text to image", - description="Create a drawing based on the text.", - requisite="`OPENAI_API_KEY` or `METAGPT_TEXT_TO_IMAGE_MODEL`") async def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs): """Text to image @@ -26,13 +22,12 @@ async def text_to_image(text, size_type: str = "512x512", openai_api_key="", mod :param model_url: MetaGPT model url :return: The image data is returned in Base64 encoding. """ - initialize_environment() image_declaration = "data:image/png;base64," - if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL_URL") or model_url: + if CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL_URL or model_url: data = await oas3_metagpt_text_to_image(text, size_type, model_url) return image_declaration + data if data else "" - if os.environ.get("OPENAI_API_KEY") or openai_api_key: + if CONFIG.OPENAI_API_KEY or openai_api_key: data = await oas3_openai_text_to_image(text, size_type, openai_api_key) return image_declaration + data if data else "" diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index e5eb3d488..ba73de04c 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -6,16 +6,14 @@ @File : text_to_speech.py @Desc : Text-to-Speech skill, which provides text-to-speech functionality """ -import os -from metagpt.learn.skill_metadata import skill_metadata + +from metagpt.config import CONFIG + from metagpt.tools.azure_tts import oas3_azsure_tts -from metagpt.utils.common import initialize_environment -@skill_metadata(name="Text to speech", - description="Text-to-speech", - requisite="`AZURE_TTS_SUBSCRIPTION_KEY` and `AZURE_TTS_REGION`") + async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", subscription_key="", region="", **kwargs): """Text to speech @@ -31,9 +29,8 @@ async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style=" :return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string. """ - initialize_environment() audio_declaration = "data:audio/wav;base64," - if (os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") and os.environ.get("AZURE_TTS_REGION")) or \ + if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or \ (subscription_key and region): data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) return audio_declaration + data if data else data diff --git a/metagpt/llm.py b/metagpt/llm.py new file mode 100644 index 000000000..6a9a9132f --- /dev/null +++ b/metagpt/llm.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/11 14:45 +@Author : alexanderwu +@File : llm.py +""" + +from metagpt.provider.anthropic_api import Claude2 as Claude +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM + +DEFAULT_LLM = LLM() +CLAUDE_LLM = Claude() + + +async def ai_func(prompt): + """使用LLM进行QA + QA with LLMs + """ + return await DEFAULT_LLM.aask(prompt) diff --git a/metagpt/manager.py b/metagpt/manager.py index c4565808e..9d238c621 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -4,15 +4,14 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : manager.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ - +from metagpt.llm import LLM from metagpt.logs import logger from metagpt.schema import Message class Manager: - def __init__(self, llm): + def __init__(self, llm: LLM = LLM()): self.llm = llm # Large Language Model self.role_directions = { "BOSS": "Product Manager", diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 5a498c50b..00b6cb2eb 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -4,8 +4,6 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : architect.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. """ from metagpt.actions import WriteDesign, WritePRD @@ -14,8 +12,8 @@ from metagpt.roles import Role class Architect(Role): """Architect: Listen to PRD, responsible for designing API, designing code files""" - def __init__(self, options, cost_manager, name="Bob", profile="Architect", goal="Design a concise, usable, complete python system", + def __init__(self, name="Bob", profile="Architect", goal="Design a concise, usable, complete python system", constraints="Try to specify good open source tools as much as possible"): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) + super().__init__(name, profile, goal, constraints) self._init_actions([WriteDesign]) self._watch({WritePRD}) diff --git a/metagpt/roles/customer_service.py b/metagpt/roles/customer_service.py index 8550313d4..4aae7cb03 100644 --- a/metagpt/roles/customer_service.py +++ b/metagpt/roles/customer_service.py @@ -26,11 +26,9 @@ DESC = """ class CustomerService(Sales): def __init__( self, - options, - cost_manager, name="Xiaomei", profile="Human customer service", desc=DESC, store=None ): - super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, desc=desc, store=store) + super().__init__(name, profile, desc=desc, store=store) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 9da2b5a09..072e53998 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -47,10 +47,10 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): - def __init__(self, options, cost_manager, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", + def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", n_borg=1, use_code_review=False): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) + super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review if self.use_code_review: @@ -131,7 +131,7 @@ class Engineer(Role): async def _act_sp(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later for todo in self.todos: - code = await WriteCode(options=self.options, llm=self._llm).run( + code = await WriteCode().run( context=self._rc.history, filename=todo ) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index bb69c8dfd..b42e9bb29 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -4,16 +4,14 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : product_manager.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. """ from metagpt.actions import BossRequirement, WritePRD from metagpt.roles import Role class ProductManager(Role): - def __init__(self, options, cost_manager, name="Alice", profile="Product Manager", goal="Efficiently create a successful product", + def __init__(self, name="Alice", profile="Product Manager", goal="Efficiently create a successful product", constraints=""): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) + super().__init__(name, profile, goal, constraints) self._init_actions([WritePRD]) self._watch([BossRequirement]) diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 3e8b36550..ff374de13 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -4,16 +4,14 @@ @Time : 2023/5/11 15:04 @Author : alexanderwu @File : project_manager.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. """ from metagpt.actions import WriteDesign, WriteTasks from metagpt.roles import Role class ProjectManager(Role): - def __init__(self, options, cost_manager, name="Eve", profile="Project Manager", + def __init__(self, name="Eve", profile="Project Manager", goal="Improve team efficiency and deliver with quality and quantity", constraints=""): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) + super().__init__(name, profile, goal, constraints) self._init_actions([WriteTasks]) self._watch([WriteDesign]) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index ac5df0dbd..65bf2cc5b 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -20,15 +20,13 @@ from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP class QaEngineer(Role): def __init__( self, - options, - cost_manager, name="Edward", profile="QaEngineer", goal="Write comprehensive and robust tests to ensure codes will work as expected without bugs", constraints="The test code you write should conform to code standard like PEP8, be modular, easy to read and maintain", test_round_allowed=5, ): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) + super().__init__(name, profile, goal, constraints) self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index f3ff7f8e5..cb4d28c33 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -26,8 +26,6 @@ class Report(BaseModel): class Researcher(Role): def __init__( self, - options, - cost_manager, name: str = "David", profile: str = "Researcher", goal: str = "Gather information and conduct research", @@ -35,11 +33,8 @@ class Researcher(Role): language: str = "en-us", **kwargs, ): - super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, **kwargs) - self._init_actions([ - CollectLinks(options=options, name=name), - WebBrowseAndSummarize(options=options, name=name), - ConductResearch(options=options, name=name)]) + super().__init__(name, profile, goal, constraints, **kwargs) + self._init_actions([CollectLinks(name), WebBrowseAndSummarize(name), ConductResearch(name)]) self.language = language if language not in ("en-us", "zh-cn"): logger.warning(f"The language `{language}` has not been tested, it may not work.") diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 4f46bb973..a1ac0d9e7 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,9 +4,7 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py -@Modified By: mashenquan, 2023-8-7, :class:`Role` + properties. -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. +@Modified By: mashenquan, 2023-8-7, Support template-style variables, such as '{teaching_language} Teacher'. @Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. """ from __future__ import annotations @@ -15,7 +13,8 @@ from typing import Iterable, Type, Dict from pydantic import BaseModel, Field -from metagpt.config import Config +from metagpt.config import Config, CONFIG +from metagpt.const import OPTIONS from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager from metagpt.actions import Action, ActionOutput from metagpt.logs import logger @@ -74,13 +73,12 @@ class RoleContext(BaseModel): todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) - options: Dict class Config: arbitrary_types_allowed = True def check(self, role_id: str): - if self.options.get("long_term_memory"): + if CONFIG.long_term_memory: self.long_term_memory.recover_memory(role_id, self) self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation @@ -102,26 +100,20 @@ class RoleContext(BaseModel): class Role: """Role/Proxy""" - def __init__(self, options=None, cost_manager=None, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): - options = options or Config().runtime_options - cost_manager = cost_manager or CostManager(*options) - - self._options = Role.supply_options(options=kwargs, default_options=options) - - name = Role.format_value(name, self._options) - profile = Role.format_value(profile, self._options) - goal = Role.format_value(goal, self._options) - constraints = Role.format_value(constraints, self._options) - desc = Role.format_value(desc, self._options) - - self._cost_manager = cost_manager - self._llm = LLM(options=self._options, cost_manager=cost_manager) + def __init__(self, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): + # Replace template-style variables, such as '{teaching_language} Teacher'. + name = Role.format_value(name) + profile = Role.format_value(profile) + goal = Role.format_value(goal) + constraints = Role.format_value(constraints) + desc = Role.format_value(desc) + self._llm = LLM() self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) self._states = [] self._actions = [] self._role_id = str(self._setting) - self._rc = RoleContext(options=self._options) + self._rc = RoleContext() def _reset(self): self._states = [] @@ -131,7 +123,7 @@ class Role: self._reset() for idx, action in enumerate(actions): if not isinstance(action, Action): - i = action(options=self._options, name="", llm=self._llm) + i = action("", llm=self._llm) else: i = action i.set_prefix(self._get_prefix(), self.profile) @@ -184,14 +176,6 @@ class Role: """Return number of action""" return len(self._actions) - @property - def options(self): - return self._options - - @options.setter - def options(self, opts): - self._options.update(opts) - def _get_prefix(self): """获取角色前缀""" if self._setting.desc: @@ -222,7 +206,7 @@ class Role: logger.info(f"{self._setting}: ready to {self._rc.todo}") requirement = self._rc.important_memory or self._rc.prerequisite - response = await self._rc.todo.run(requirement, **self._options) + response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): msg = Message(content=response.content, instruct_content=response.instruct_content, @@ -300,23 +284,14 @@ class Role: return rsp @staticmethod - def supply_options(options, default_options=None): - """Supply missing options""" - ret = default_options.copy() if default_options else {} - if not options: - return ret - ret.update(options) - return ret - - @staticmethod - def format_value(value, opts, default_opts=None): + def format_value(value): """Fill parameters inside `value` with `options`.""" if not isinstance(value, str): return value if "{" not in value: return value - merged_opts = Role.supply_options(opts, default_opts) + merged_opts = OPTIONS.get() or {} try: return value.format(**merged_opts) except KeyError as e: diff --git a/metagpt/roles/sales.py b/metagpt/roles/sales.py index 35146fdc3..51b13f487 100644 --- a/metagpt/roles/sales.py +++ b/metagpt/roles/sales.py @@ -13,8 +13,6 @@ from metagpt.tools import SearchEngineType class Sales(Role): def __init__( self, - options, - cost_manager, name="Xiaomei", profile="Retail sales guide", desc="I am a sales guide in retail. My name is Xiaomei. I will answer some customer questions next, and I " @@ -25,7 +23,7 @@ class Sales(Role): "professional guide", store=None ): - super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, desc=desc) + super().__init__(name, profile, desc=desc) self._set_store(store) def _set_store(self, store): diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index 7b07ce713..c116ce98b 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -13,9 +13,9 @@ from metagpt.tools import SearchEngineType class Searcher(Role): - def __init__(self, options, cost_manager, name='Alice', profile='Smart Assistant', goal='Provide search services for users', + def __init__(self, name='Alice', profile='Smart Assistant', goal='Provide search services for users', constraints='Answer is rich and complete', engine=SearchEngineType.SERPAPI_GOOGLE, **kwargs): - super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, **kwargs) + super().__init__(name, profile, goal, constraints, **kwargs) self._init_actions([SearchAndSummarize(engine=engine)]) def set_search_func(self, search_func): diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index d2a2198f5..ca88fd681 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -22,13 +22,13 @@ import re class Teacher(Role): """Support configurable teacher roles, with native and teaching languages being replaceable through configurations.""" - def __init__(self, options, name='Lily', profile='{teaching_language} Teacher', + def __init__(self, name='Lily', profile='{teaching_language} Teacher', goal='writing a {language} teaching plan part by part', constraints='writing in {language}', desc="", *args, **kwargs): - super().__init__(options=options, name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) actions = [] for topic in WriteTeachingPlanPart.TOPICS: - act = WriteTeachingPlanPart(options=options, topic=topic, llm=self._llm) + act = WriteTeachingPlanPart(topic=topic, llm=self._llm) actions.append(act) self._init_actions(actions) self._watch({TeachingPlanRequirement}) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 529dc0fe7..8f173ebf3 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,22 +4,16 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py -@Modified By: mashenquan, 2023-07-27, Add `role` & `cause_by` parameters to `start_project()`. -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. """ -from typing import Dict - from pydantic import BaseModel, Field from metagpt.actions import BossRequirement +from metagpt.config import CONFIG from metagpt.environment import Environment from metagpt.logs import logger -from metagpt.provider.openai_api import CostManager from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import NoMoneyException -from metagpt.config import Config class SoftwareCompany(BaseModel): @@ -30,8 +24,6 @@ class SoftwareCompany(BaseModel): environment: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") - options: Dict = Field(default=Config().runtime_options) - cost_manager: CostManager = Field(default=CostManager(**Config().runtime_options)) class Config: arbitrary_types_allowed = True @@ -43,17 +35,17 @@ class SoftwareCompany(BaseModel): def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" self.investment = investment - self.options["max_budget"] = investment + CONFIG.max_budget = investment logger.info(f'Investment: ${investment}.') def _check_balance(self): - if self.total_cost > self.max_budget: - raise NoMoneyException(self.total_cost, f'Insufficient funds: {self.max_budget}') + if CONFIG.total_cost > CONFIG.max_budget: + raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea, role="BOSS", cause_by=BossRequirement): + def start_project(self, idea): """Start a project from publishing boss requirement.""" self.idea = idea - self.environment.publish_message(Message(role=role, content=idea, cause_by=cause_by)) + self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) def _save(self): logger.info(self.json()) @@ -67,13 +59,3 @@ class SoftwareCompany(BaseModel): self._check_balance() await self.environment.run() return self.environment.history - - @property - def max_budget(self): - return self.options.get("max_budget", 0) - - @property - def total_cost(self): - return self.options.get("total_cost", 0) - - diff --git a/metagpt/tools/openai_text_to_embedding.py b/metagpt/tools/openai_text_to_embedding.py index 119eb35b6..73984aff6 100644 --- a/metagpt/tools/openai_text_to_embedding.py +++ b/metagpt/tools/openai_text_to_embedding.py @@ -17,8 +17,9 @@ import requests from pydantic import BaseModel import sys +from metagpt.config import CONFIG + sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initialize_environment from metagpt.logs import logger @@ -83,12 +84,11 @@ async def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", op if not text: return "" if not openai_api_key: - openai_api_key = os.environ.get("OPENAI_API_KEY") + openai_api_key = CONFIG.OPENAI_API_KEY return await OpenAIText2Embedding(openai_api_key).text_2_embedding(text, model=model) if __name__ == "__main__": - initialize_environment() loop = asyncio.new_event_loop() v = loop.create_task(oas3_openai_text_to_embedding("Panda emoji")) loop.run_until_complete(v) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index a6e4dc20d..791bb2767 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -259,18 +259,3 @@ def parse_recipient(text): recipient = re.search(pattern, text) return recipient.group(1) if recipient else "" - -def initialize_environment(options=None): - """Load `config/config.yaml` to `os.environ`""" - if options: - for k, v in options.items(): - os.environ[k] = str(v) - return - - yaml_file_path = Path(__file__).resolve().parent.parent.parent / "config/config.yaml" - if not yaml_file_path.exists(): - return - with open(str(yaml_file_path), "r") as yaml_file: - data = yaml.safe_load(yaml_file) - for k, v in data.items(): - os.environ[k] = str(v) diff --git a/startup.py b/startup.py index 84cd43956..03b2149c4 100644 --- a/startup.py +++ b/startup.py @@ -1,10 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. -""" - import asyncio import platform import fire @@ -16,15 +11,14 @@ from metagpt.software_company import SoftwareCompany async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False, run_tests: bool = False): """Run a startup. Be a boss.""" - company = SoftwareCompany() - company.hire([ProductManager(options=company.options, cost_manager=company.cost_manager), - Architect(options=company.options, cost_manager=company.cost_manager), - ProjectManager(options=company.options, cost_manager=company.cost_manager), - Engineer(n_borg=5, use_code_review=code_review, options=company.options, cost_manager=company.cost_manager)]) + company.hire([ProductManager(), + Architect(), + ProjectManager(), + Engineer(n_borg=5, use_code_review=code_review)]) if run_tests: # developing features: run tests on the spot and identify bugs (bug fixing capability comes soon!) - company.hire([QaEngineer(options=company.options, cost_manager=company.cost_manager)]) + company.hire([QaEngineer()]) company.invest(investment) company.start_project(idea) await company.run(n_round=n_round) From 23ba0f3540c90f0e3336741c98ad50debcd0d6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 17:56:50 +0800 Subject: [PATCH 114/398] feat: replaced with OPTIONS --- metagpt/provider/anthropic_api.py | 15 ++----- metagpt/provider/openai_api.py | 65 +++++++++++-------------------- 2 files changed, 26 insertions(+), 54 deletions(-) diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 326d23a5c..03802a716 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -4,22 +4,17 @@ @Time : 2023/7/21 11:15 @Author : Leo Xiao @File : anthropic_api.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. """ import anthropic from anthropic import Anthropic -from metagpt.config import Config +from metagpt.config import CONFIG class Claude2: - def __init__(self, options=None): - self.options = options or Config().runtime_options - def ask(self, prompt): - client = Anthropic(api_key=self.claude_api_key) + client = Anthropic(api_key=CONFIG.claude_api_key) res = client.completions.create( model="claude-2", @@ -29,7 +24,7 @@ class Claude2: return res.completion async def aask(self, prompt): - client = Anthropic(api_key=self.claude_api_key) + client = Anthropic(api_key=CONFIG.claude_api_key) res = client.completions.create( model="claude-2", @@ -37,7 +32,3 @@ class Claude2: max_tokens_to_sample=1000, ) return res.completion - - @property - def claude_api_key(self): - return self.options.get("claude_api_key") diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 098388a7c..640694b67 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -18,6 +18,7 @@ from openai.error import APIConnectionError from pydantic import BaseModel from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.token_counter import ( @@ -134,23 +135,22 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): Check https://platform.openai.com/examples for examples """ - def __init__(self, options, cost_manager): - self._options = options - self.__init_openai() + def __init__(self, cost_manager): + self.__init_openai(CONFIG) self.llm = openai - self.model = self.openai_api_model + self.model = CONFIG.openai_api_model self.auto_max_tokens = False - self._cost_manager = cost_manager + self._cost_manager = cost_manager or CostManager() RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self): - openai.api_key = self.openai_api_key - if self.openai_api_base: - openai.api_base = self.openai_api_base - if self.openai_api_type: - openai.api_type = self.openai_api_type - openai.api_version = self.openai_api_version - self.rpm = int(self._options.get("RPM", 10)) + def __init_openai(self, config): + openai.api_key = config.openai_api_key + if config.openai_api_base: + openai.api_base = config.openai_api_base + if config.openai_api_type: + openai.api_type = config.openai_api_type + openai.api_version = config.openai_api_version + self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: response = await self.async_retry_call(openai.ChatCompletion.acreate, @@ -175,9 +175,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return full_reply_content def _cons_kwargs(self, messages: list[dict]) -> dict: - if self._options.get("openai_api_type") == "azure": + if CONFIG.openai_api_type == "azure": kwargs = { - "deployment_id": self._options.get("deployment_id"), + "deployment_id": CONFIG.deployment_id, "messages": messages, "max_tokens": self.get_max_tokens(messages), "n": 1, @@ -232,7 +232,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def _calc_usage(self, messages: list[dict], rsp: str) -> dict: usage = {} - if self._options.get("calc_usage"): + if CONFIG.calc_usage: try: prompt_tokens = count_message_tokens(messages, self.model) completion_tokens = count_string_tokens(rsp, self.model) @@ -271,7 +271,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return results def _update_costs(self, usage: dict): - if self._options.get("calc_usage"): + if CONFIG.calc_usage: try: prompt_tokens = int(usage['prompt_tokens']) completion_tokens = int(usage['completion_tokens']) @@ -284,34 +284,14 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def get_max_tokens(self, messages: list[dict]): if not self.auto_max_tokens: - return self._options.get("max_tokens_rsp") - return get_max_completion_tokens(messages, self.model, self._options.get("max_tokens_rsp")) - - @property - def openai_api_model(self): - return self._options.get("openai_api_model") - - @property - def openai_api_key(self): - return self._options.get("openai_api_key") - - @property - def openai_api_base(self): - return self._options.get("openai_api_base") - - @property - def openai_api_type(self): - return self._options.get("openai_api_type") - - @property - def openai_api_version(self): - return self._options.get("openai_api_version") + return CONFIG.max_tokens_rsp + return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) async def get_summary(self, text: str, max_words=20): """Generate text summary""" if len(text) < max_words: return text - language = self._options.get("language", "English") + language = CONFIG.language or self.DEFAULT_LANGUAGE command = f"Translate the above content into a {language} summary of less than {max_words} words." msg = text + "\n\n" + command logger.info(f"summary ask:{msg}") @@ -322,7 +302,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def get_context_title(self, text: str, max_token_count_per_ask=None, max_words=5) -> str: """Generate text title""" max_response_token_count = 50 - max_token_count = max_token_count_per_ask or self._options.get("MAX_TOKENS", 1500) + max_token_count = max_token_count_per_ask or CONFIG.MAX_TOKENS or 1500 text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) summaries = [] @@ -332,7 +312,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if len(summaries) == 1: return summaries[0] - language = self._options.get("language", "English") + language = CONFIG.language or self.DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." summaries.append(command) msg = "\n".join(summaries) @@ -418,3 +398,4 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 + DEFAULT_LANGUAGE = "Engilish" From 7895af2c5a59511c3ba01e50420890a1cd85460b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:07:59 +0800 Subject: [PATCH 115/398] feat: replace CONFIG with OPTIONS --- examples/write_teaching_plan.py | 4 ++-- metagpt/actions/search_and_summarize.py | 1 + metagpt/actions/write_teaching_plan.py | 4 ++-- metagpt/config.py | 5 +++++ metagpt/provider/openai_api.py | 4 ++-- metagpt/roles/assistant.py | 7 ++++--- metagpt/software_company.py | 4 ++-- metagpt/tools/search_engine.py | 12 ++++++------ 8 files changed, 24 insertions(+), 17 deletions(-) diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index 6ab5edce4..2a9c4c0e5 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -77,9 +77,9 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * lesson = demo_lesson company = SoftwareCompany() - company.hire([Teacher(options=company.options, cost_manager=company.cost_manager, *args, **kwargs)]) + company.hire([Teacher(*args, **kwargs)]) company.invest(investment) - company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) + company.start_project(lesson, cause_by=TeachingPlanRequirement, role="Teacher", **kwargs) await company.run(n_round=1) diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 9f54587fa..5c7577e17 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -9,6 +9,7 @@ import pydantic from metagpt.actions import Action +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Message from metagpt.tools.search_engine import SearchEngine diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index bd8507350..7c959ce85 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -20,7 +20,7 @@ class TeachingPlanRequirement(Action): class WriteTeachingPlanPart(Action): """Write Teaching Plan Part""" - def __init__(self, options, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): + def __init__(self, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): """ :param name: action name @@ -29,7 +29,7 @@ class WriteTeachingPlanPart(Action): :param topic: topic part of teaching plan :param language: A human language, such as Chinese, English, French, etc. """ - super().__init__(options, name, context, llm) + super().__init__(name, context, llm) self.topic = topic self.language = language self.rsp = None diff --git a/metagpt/config.py b/metagpt/config.py index ceaa582e2..a3edc22b6 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -126,4 +126,9 @@ class Config(metaclass=Singleton): opts.update(options) OPTIONS.set(opts) + @property + def options(self): + """Return all key-values""" + return OPTIONS.get() + CONFIG = Config() diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 640694b67..02bf5126c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -67,7 +67,7 @@ class CostManager(BaseModel): total_prompt_tokens: int = 0 total_completion_tokens: int = 0 total_budget: float = 0 - max_budget: float + max_budget: float = CONFIG.max_budget total_cost: float = 0 def update_cost(self, prompt_tokens, completion_tokens, model): @@ -135,7 +135,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): Check https://platform.openai.com/examples for examples """ - def __init__(self, cost_manager): + def __init__(self, cost_manager=None): self.__init_openai(CONFIG) self.llm = openai self.model = CONFIG.openai_api_model diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index d6f52e4e4..c8a786b41 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -10,7 +10,8 @@ For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file. -@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false + indicates that further reasoning cannot continue. """ import asyncio @@ -34,7 +35,7 @@ SKILL_PATH = "SKILL_PATH" class Assistant(Role): - """解决通用问题的助手""" + """Assistant for solving common issues.""" def __init__(self, options, cost_manager, name="Lily", profile="An assistant", goal="Help to solve problem", constraints="Talk in {language}", desc="", *args, **kwargs): @@ -152,7 +153,7 @@ async def main(): break msg = await role.act() logger.info(msg) - # 获取用户终端输入 + # Retrieve user terminal input. logger.info("Enter prompt") talk = input("You: ") await role.talk(talk) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 8f173ebf3..8d9c990ee 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -42,10 +42,10 @@ class SoftwareCompany(BaseModel): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea): + def start_project(self, idea, role="BOSS", cause_by=BossRequirement, **kwargs): """Start a project from publishing boss requirement.""" self.idea = idea - self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) + self.environment.publish_message(Message(content=idea, role=role, cause_by=cause_by)) def _save(self): logger.info(self.json()) diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index c82ae6595..5b8b7f046 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -11,6 +11,7 @@ from __future__ import annotations import importlib from typing import Callable, Coroutine, Literal, overload, Dict +from metagpt.config import CONFIG from metagpt.tools import SearchEngineType @@ -28,23 +29,22 @@ class SearchEngine: def __init__( self, - options: Dict, engine: SearchEngineType | None = None, run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None ): - engine = engine or options.get("search_engine") + engine = engine or CONFIG.search_engine if engine == SearchEngineType.SERPAPI_GOOGLE: module = "metagpt.tools.search_engine_serpapi" - run_func = importlib.import_module(module).SerpAPIWrapper(**options).run + run_func = importlib.import_module(module).SerpAPIWrapper(**CONFIG.options).run elif engine == SearchEngineType.SERPER_GOOGLE: module = "metagpt.tools.search_engine_serper" - run_func = importlib.import_module(module).SerperWrapper(**options).run + run_func = importlib.import_module(module).SerperWrapper(**CONFIG.options).run elif engine == SearchEngineType.DIRECT_GOOGLE: module = "metagpt.tools.search_engine_googleapi" - run_func = importlib.import_module(module).GoogleAPIWrapper(**options).run + run_func = importlib.import_module(module).GoogleAPIWrapper(**CONFIG.options).run elif engine == SearchEngineType.DUCK_DUCK_GO: module = "metagpt.tools.search_engine_ddg" - run_func = importlib.import_module(module).DDGAPIWrapper(**options).run + run_func = importlib.import_module(module).DDGAPIWrapper(**CONFIG.options).run elif engine == SearchEngineType.CUSTOM_ENGINE: pass # run_func = run_func else: From 00f1e1882036261b16f1fb682153bc71f0059edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:13:41 +0800 Subject: [PATCH 116/398] feat: replace CONFIG with OPTIONS --- examples/write_teaching_plan.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index 2a9c4c0e5..191547193 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -11,6 +11,8 @@ import asyncio from pathlib import Path import sys +from metagpt.config import CONFIG + sys.path.append(str(Path(__file__).resolve().parent.parent)) import aiofiles import fire @@ -66,6 +68,7 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * 3c Match the big letters with the small ones. Then write them on the lines. """ + CONFIG.set_context(kwargs) lesson = "" if lesson_file and Path(lesson_file).exists(): From 3a96405a692efdd7ca96b104c73983744ce48a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:15:48 +0800 Subject: [PATCH 117/398] feat: delete useless config --- config/pattern/template.yaml | 40 -------- config/pattern/write_teaching_plan.yaml | 126 ------------------------ 2 files changed, 166 deletions(-) delete mode 100644 config/pattern/template.yaml delete mode 100644 config/pattern/write_teaching_plan.yaml diff --git a/config/pattern/template.yaml b/config/pattern/template.yaml deleted file mode 100644 index d148804f0..000000000 --- a/config/pattern/template.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Pattern Configuration Template -# Created By: mashenquan, 2023-8-7 -# File Name: template.yaml -# This template defines a set of structural standards for generating roles and action flows based on configurations. -# For more about UML 2.0 activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` - -# project settings -startup: - requirement: "TeachingPlanRequirement" # Defines project initial requirement action - role: "Teacher" # Defines project role - investment: 3.0 # Defines the max project investment - n_round: 1 # Defines the max project round count - -# roles settings -roles: # A project can involve multiple roles. -- role_type: "fork" # `fork` type role corresponds to the functional positioning of the `fork` node in UML 2.0 activity diagrams. - name: "Lily" - profile: "{teaching_language} Teacher" - goal: "writing a {language} teaching plan part by part" - constraints: "writing in {language}" - role: "You are a {teaching_language} Teacher, named Lily, your goal is ..." - desc: "" - output_filename: "teaching_plan_demo.md" - requirement: ["TeachingPlanRequirement"] - templates: # The template provides a convenient way to generate prompts. After each action selects its respective template, you only need to provide the corresponding variable values. Variable replacement is automatically handled by the framework. - - "Do ..." - - "Do ..." - # role's action settings - actions: # A role can have multiple actions. - - name: "" - topic: "Title" - language: "Chinese" - statements: # When replacing template variables, multiple statements will be joined into a single string using line breaks. - - "Statement: Find and return ..." - template_ix: 0 - rsp_begin_tag: "[..._BEGIN]" # When asking, request the LLM to include the tag in the response. It's optional. - rsp_end_tag: "[..._END]" # When asking, request the LLM to include the tag in the response. It's optional. - - - diff --git a/config/pattern/write_teaching_plan.yaml b/config/pattern/write_teaching_plan.yaml deleted file mode 100644 index 5b5f2af77..000000000 --- a/config/pattern/write_teaching_plan.yaml +++ /dev/null @@ -1,126 +0,0 @@ -# The `fork` role demo implements the flow of the code in `examples/write_teaching_plan.py`. - -# project settings -startup: - requirement: "TeachingPlanRequirement" # Defines project initial requirement action - role: "Teacher" - investment: 3.0 - n_round: 1 - -# roles settings -roles: # A project can involve multiple roles. -- role_type: "fork" # `fork` type role corresponds to the functional positioning of the `fork` node in UML 2.0 activity diagrams. - name: "Lily" - profile: "{teaching_language} Teacher" - goal: "writing a {language} teaching plan part by part" - constraints: "writing in {language}" - role: "You are a {teaching_language} Teacher, named Lily, your goal is writing a {teaching_language} teaching plan part by part, and the constraint is writing in {language}." - desc: "" - output_filename: "teaching_plan_demo" - requirement: ["TeachingPlanRequirement"] - templates: # The template provides a convenient way to generate prompts. After each action selects its respective template, you only need to provide the corresponding variable values. Variable replacement is automatically handled by the framework. - - "Do not refer to the context of the previous conversation records, start the conversation anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the format requirements for your responses;\n\t\"Constraint\" defines the conditions that your responses must comply with.\n\n{statements}\nConstraint: Writing in {language}.\nAnswer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END]" - - "Do not refer to the context of the previous conversation records, start the conversation anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the format requirements for your responses;\n\t\"Constraint\" defines the conditions that your responses must comply with.\n\nCapacity and role: {role}\nStatement: Write the \"{topic}\" part of teaching plan, WITHOUT ANY content unrelated to \"{topic}\"!!\n{statements}\nAnswer options: Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\nAnswer options: Using proper markdown format from second-level header format.\nConstraint: Writing in {language}.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END]" - actions: # 一个role可以有多个action - - name: "" - topic: "Title" - language: "Chinese" - statements: # When replacing template variables, multiple statements will be joined into a single string using line breaks. - - "Statement: Find and return the title of the lesson only with \"# \" string prefixed, without anything else." - template_ix: 0 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Teaching Hours" - language: "Chinese" - statements: [] - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" # When asking, request the LLM to include the tag in the response. It's optional. - rsp_end_tag: "[TEACHING_PLAN_END]" # When asking, request the LLM to include the tag in the response. It's optional. - - name: "" - topic: "Teaching Objectives" - language: "Chinese" - statements: [] - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Teaching Content" - language: "Chinese" - statements: - - "Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar structures that appear in the textbook, as well as the listening materials and key points." - - "Statement: \"Teaching Content\" must include more examples." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Teaching Methods and Strategies" - language: "Chinese" - statements: - - "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, procedures, in detail." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Learning Activities" - language: "Chinese" - statements: [] - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Teaching Time Allocation" - language: "Chinese" - statements: - - "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each part of the textbook content." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Assessment and Feedback" - language: "Chinese" - statements: [] - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Teaching Summary and Improvement" - language: "Chinese" - statements: [] - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Vocabulary Cloze" - language: "Chinese" - statements: - - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers. The key-related vocabulary and phrases in the textbook content must all be included in the exercises." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Choice Questions" - language: "Chinese" - statements: - - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create choice questions. 10 questions." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Grammar Questions" - language: "Chinese" - statements: - - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create grammar questions. 10 questions." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Translation Questions" - language: "Chinese" - statements: - - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create translation questions. The translation should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - From 3243078b77d15874a2fde38a2833005ebe0d143e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:21:50 +0800 Subject: [PATCH 118/398] feat: replace CONFIG with OPTIONS --- metagpt/actions/talk_action.py | 19 ++++++++++++++----- metagpt/const.py | 1 + metagpt/provider/openai_api.py | 7 ++++--- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 5692cf4f4..555b202d1 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -1,15 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/28 +@Author : mashenquan +@File : talk_action.py +@Desc : Act as it’s a talk +""" + from metagpt.actions import Action, ActionOutput +from metagpt.config import CONFIG +from metagpt.const import DEFAULT_LANGUAGE from metagpt.logs import logger - class TalkAction(Action): - def __init__(self, options, name: str = '', talk='', history_summary='', knowledge='', context=None, llm=None, **kwargs): + def __init__(self, name: str = '', talk='', history_summary='', knowledge='', context=None, llm=None, **kwargs): context = context or {} context["talk"] = talk context["history_summery"] = history_summary context["knowledge"] = knowledge - super(TalkAction, self).__init__(options=options, name=name, context=context, llm=llm) + super(TalkAction, self).__init__(name=name, context=context, llm=llm) self._talk = talk self._history_summary = history_summary self._knowledge = knowledge @@ -21,7 +31,7 @@ class TalkAction(Action): prompt += f"{self._history_summary}\n\n" if self._history_summary != "": prompt += "According to the historical conversation above, " - language = self.options.get("language", "Chinese") + language = CONFIG.language or DEFAULT_LANGUAGE prompt += f"Answer in {language}:\n {self._talk}" return prompt @@ -32,4 +42,3 @@ class TalkAction(Action): logger.info(rsp) self._rsp = ActionOutput(content=rsp) return self._rsp - diff --git a/metagpt/const.py b/metagpt/const.py index 20513461a..0e50f2c39 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -38,3 +38,4 @@ RESEARCH_PATH = DATA_PATH / "research" MEM_TTL = 24 * 30 * 3600 OPTIONS = contextvars.ContextVar("OPTIONS") +DEFAULT_LANGUAGE = "Engilish" \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 02bf5126c..45e67739b 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -19,6 +19,7 @@ from pydantic import BaseModel from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type from metagpt.config import CONFIG +from metagpt.const import DEFAULT_LANGUAGE from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.token_counter import ( @@ -291,7 +292,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """Generate text summary""" if len(text) < max_words: return text - language = CONFIG.language or self.DEFAULT_LANGUAGE + language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above content into a {language} summary of less than {max_words} words." msg = text + "\n\n" + command logger.info(f"summary ask:{msg}") @@ -312,7 +313,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if len(summaries) == 1: return summaries[0] - language = CONFIG.language or self.DEFAULT_LANGUAGE + language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." summaries.append(command) msg = "\n".join(summaries) @@ -398,4 +399,4 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 - DEFAULT_LANGUAGE = "Engilish" + From 7c4b5b40828918d3084ac622bb4293d1ac8c0a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:26:21 +0800 Subject: [PATCH 119/398] feat: fix coding --- metagpt/const.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index 0e50f2c39..a14dbc5b8 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -3,7 +3,8 @@ """ @Time : 2023/5/1 11:59 @Author : alexanderwu -@File : const.py +@File : const.py' +@Modified By: mashenquan, 2023/8/28. Add 'OPTIONS', 'DEFAULT_LANGUAGE' """ import contextvars from pathlib import Path @@ -38,4 +39,4 @@ RESEARCH_PATH = DATA_PATH / "research" MEM_TTL = 24 * 30 * 3600 OPTIONS = contextvars.ContextVar("OPTIONS") -DEFAULT_LANGUAGE = "Engilish" \ No newline at end of file +DEFAULT_LANGUAGE = "English" From 1c2b14b46df1f28f7131acc71ab5887ff69e690b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:31:07 +0800 Subject: [PATCH 120/398] feat: + annotations --- metagpt/learn/skill_loader.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index 71535f310..cbf63c60a 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : skill_loader.py +@Desc : Skill YAML Configuration Loader. +""" + from pathlib import Path from typing import List, Dict, Optional @@ -9,10 +18,12 @@ class Example(BaseModel): ask: str answer: str + class Returns(BaseModel): type: str format: Optional[str] = None + class Skill(BaseModel): name: str description: str @@ -40,6 +51,7 @@ class SkillLoader: self._skills = SkillsDeclaration(**skills) def get_skill_list(self, entity_name: str = "Assistant") -> Dict: + """Return the skill name based on the skill description.""" entity_skills = self.get_entity(entity_name) if not entity_skills: return {} @@ -51,6 +63,7 @@ class SkillLoader: return description_to_name_mappings def get_skill(self, name, entity_name: str = "Assistant") -> Skill: + """Return a skill by name.""" entity = self.get_entity(entity_name) if not entity: return None @@ -59,6 +72,7 @@ class SkillLoader: return sk def get_entity(self, name) -> EntitySkills: + """Return a list of skills for the entity.""" if not self._skills: return None - return self._skills.entities.get(name) \ No newline at end of file + return self._skills.entities.get(name) From deccb9fde272312c0e5de2fe5262bc7db5d8f802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:35:30 +0800 Subject: [PATCH 121/398] feat: + annotations --- metagpt/config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index a3edc22b6..f1c869b6c 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -52,10 +52,12 @@ class Config(metaclass=Singleton): ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") - openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy - if openai_proxy: - openai.proxy = openai_proxy - openai.api_base = self.openai_api_base + if not self.openai_api_base or "YOUR_API_BASE" == self.openai_api_base: + openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy + if openai_proxy: + openai.proxy = openai_proxy + else: + logger.info("Set OPENAI_API_BASE in case of network issues") self.openai_api_type = self._get("OPENAI_API_TYPE") self.openai_api_version = self._get("OPENAI_API_VERSION") self.openai_api_rpm = self._get("RPM", 3) From 8738831e0fdfcfd2a6f60c30bf419b3130241232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:38:27 +0800 Subject: [PATCH 122/398] feat: + annotations --- metagpt/learn/text_to_image.py | 1 - metagpt/learn/text_to_speech.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 2762c2f18..620e58180 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -6,7 +6,6 @@ @File : text_to_image.py @Desc : Text-to-Image skill, which provides text-to-image functionality. """ -import os from metagpt.config import CONFIG from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index ba73de04c..66fbba5be 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -7,15 +7,13 @@ @Desc : Text-to-Speech skill, which provides text-to-speech functionality """ - from metagpt.config import CONFIG from metagpt.tools.azure_tts import oas3_azsure_tts - async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", - subscription_key="", region="", **kwargs): + subscription_key="", region="", **kwargs): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` From 455c59d8c4af7abeb8c080bdc167e2369e00c6f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:41:32 +0800 Subject: [PATCH 123/398] feat: + annotations --- metagpt/memory/brain_memory.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index cb67fea8e..b3445a1f2 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : brain_memory.py +@Desc : Support memory for multiple tasks and multiple mainlines. +""" + from enum import Enum from typing import List, Dict From 27561765cf49c421147fd4e4bf2d76a37672aa5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:45:56 +0800 Subject: [PATCH 124/398] feat: + annotations --- metagpt/const.py | 3 ++- metagpt/provider/openai_api.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index a14dbc5b8..8c1460a02 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -4,7 +4,7 @@ @Time : 2023/5/1 11:59 @Author : alexanderwu @File : const.py' -@Modified By: mashenquan, 2023/8/28. Add 'OPTIONS', 'DEFAULT_LANGUAGE' +@Modified By: mashenquan, 2023/8/28. Add 'OPTIONS', 'DEFAULT_LANGUAGE', 'DEFAULT_MAX_TOKENS' """ import contextvars from pathlib import Path @@ -40,3 +40,4 @@ MEM_TTL = 24 * 30 * 3600 OPTIONS = contextvars.ContextVar("OPTIONS") DEFAULT_LANGUAGE = "English" +DEFAULT_MAX_TOKENS = 1500 \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 45e67739b..7dba00530 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -19,7 +19,7 @@ from pydantic import BaseModel from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type from metagpt.config import CONFIG -from metagpt.const import DEFAULT_LANGUAGE +from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.token_counter import ( @@ -303,7 +303,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def get_context_title(self, text: str, max_token_count_per_ask=None, max_words=5) -> str: """Generate text title""" max_response_token_count = 50 - max_token_count = max_token_count_per_ask or CONFIG.MAX_TOKENS or 1500 + max_token_count = max_token_count_per_ask or CONFIG.MAX_TOKENS or DEFAULT_MAX_TOKENS text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) summaries = [] From 946e6fa8b39d82f5f688c01bdcd4f3a1e20d1464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:47:57 +0800 Subject: [PATCH 125/398] feat: + annotations --- metagpt/const.py | 7 +++++-- metagpt/roles/assistant.py | 6 +----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index 8c1460a02..9e7462da6 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -4,7 +4,7 @@ @Time : 2023/5/1 11:59 @Author : alexanderwu @File : const.py' -@Modified By: mashenquan, 2023/8/28. Add 'OPTIONS', 'DEFAULT_LANGUAGE', 'DEFAULT_MAX_TOKENS' +@Modified By: mashenquan, 2023/8/28. Add 'OPTIONS', 'DEFAULT_LANGUAGE', 'DEFAULT_MAX_TOKENS'... """ import contextvars from pathlib import Path @@ -40,4 +40,7 @@ MEM_TTL = 24 * 30 * 3600 OPTIONS = contextvars.ContextVar("OPTIONS") DEFAULT_LANGUAGE = "English" -DEFAULT_MAX_TOKENS = 1500 \ No newline at end of file +DEFAULT_MAX_TOKENS = 1500 +COMMAND_TOKENS = 500 +BRAIN_MEMORY = "BRAIN_MEMORY" +SKILL_PATH = "SKILL_PATH" \ No newline at end of file diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index c8a786b41..7d1517d7e 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -21,6 +21,7 @@ from metagpt.actions import ActionOutput from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction from metagpt.actions.talk_action import TalkAction from metagpt.config import Config +from metagpt.const import BRAIN_MEMORY, SKILL_PATH from metagpt.learn.skill_loader import SkillLoader from metagpt.logs import logger from metagpt.memory.brain_memory import BrainMemory, MessageType @@ -28,11 +29,6 @@ from metagpt.provider.openai_api import CostManager from metagpt.roles import Role from metagpt.schema import Message -DEFAULT_MAX_TOKENS = 1500 -COMMAND_TOKENS = 500 -BRAIN_MEMORY = "BRAIN_MEMORY" -SKILL_PATH = "SKILL_PATH" - class Assistant(Role): """Assistant for solving common issues.""" From 71b4922f554a0dc411bde745066ecc286ad0fc5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 20:17:55 +0800 Subject: [PATCH 126/398] feat: fix coding --- metagpt/learn/skill_loader.py | 12 ++++++++--- metagpt/roles/assistant.py | 29 +++++++++++++------------- metagpt/tools/azure_tts.py | 3 --- metagpt/tools/metagpt_oas3_api_svc.py | 3 --- metagpt/tools/metagpt_text_to_image.py | 3 --- metagpt/tools/openai_text_to_image.py | 3 --- tests/metagpt/tools/test_azure_tts.py | 3 --- 7 files changed, 23 insertions(+), 33 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index cbf63c60a..1cd83240d 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -6,12 +6,11 @@ @File : skill_loader.py @Desc : Skill YAML Configuration Loader. """ - from pathlib import Path from typing import List, Dict, Optional import yaml -from pydantic import BaseModel +from pydantic import BaseModel, Field class Example(BaseModel): @@ -24,11 +23,18 @@ class Returns(BaseModel): format: Optional[str] = None +class Prerequisite(BaseModel): + name: str + type: Optional[str] = None + description: Optional[str] = None + default: Optional[str] = None + + class Skill(BaseModel): name: str description: str id: str - requisite: List[str] + x_prerequisite: Optional[List[Prerequisite]] = Field(default=None, alias="x-prerequisite") arguments: Dict examples: List[Example] returns: Returns diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 7d1517d7e..944b250f1 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -20,7 +20,7 @@ from pathlib import Path from metagpt.actions import ActionOutput from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction from metagpt.actions.talk_action import TalkAction -from metagpt.config import Config +from metagpt.config import Config, CONFIG from metagpt.const import BRAIN_MEMORY, SKILL_PATH from metagpt.learn.skill_loader import SkillLoader from metagpt.logs import logger @@ -33,13 +33,13 @@ from metagpt.schema import Message class Assistant(Role): """Assistant for solving common issues.""" - def __init__(self, options, cost_manager, name="Lily", profile="An assistant", goal="Help to solve problem", + def __init__(self, name="Lily", profile="An assistant", goal="Help to solve problem", constraints="Talk in {language}", desc="", *args, **kwargs): - super(Assistant, self).__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, + super(Assistant, self).__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) - brain_memory = options.get(BRAIN_MEMORY) + brain_memory = CONFIG.BRAIN_MEMORY self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory() - skill_path = Path(options.get(SKILL_PATH)) if options.get(SKILL_PATH) else None + skill_path = Path(CONFIG.SKILL_PATH) if CONFIG.SKILL_PATH else None self.skills = SkillLoader(skill_yaml_file_name=skill_path) async def think(self) -> bool: @@ -60,7 +60,7 @@ class Assistant(Role): return await self._plan(rsp, last_talk=last_talk) async def act(self) -> ActionOutput: - result = await self._rc.todo.run(**self._options) + result = await self._rc.todo.run(**CONFIG.options) if not result: return None if isinstance(result, str): @@ -87,7 +87,7 @@ class Assistant(Role): return await handler(text, **kwargs) async def talk_handler(self, text, **kwargs) -> bool: - action = TalkAction(options=self.options, talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, + action = TalkAction(talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, **kwargs) self.add_to_do(action) return True @@ -98,12 +98,11 @@ class Assistant(Role): if not skill: logger.info(f"skill not found: {text}") return await self.talk_handler(text=last_talk, **kwargs) - action = ArgumentsParingAction(options=self.options, skill=skill, llm=self._llm, **kwargs) + action = ArgumentsParingAction(skill=skill, llm=self._llm, **kwargs) await action.run(**kwargs) if action.args is None: return await self.talk_handler(text=last_talk, **kwargs) - action = SkillAction(options=self.options, skill=skill, args=action.args, llm=self._llm, name=skill.name, - desc=skill.description) + action = SkillAction(skill=skill, args=action.args, llm=self._llm, name=skill.name, desc=skill.description) self.add_to_do(action) return True @@ -115,11 +114,11 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_context_title(history_text, max_words=20) - if last_talk and await self._llm.is_related(last_talk, history_summary): # 合并相关内容 + if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk - self.memory.move_to_solution() # 问题解决后及时清空内存 + self.memory.move_to_solution() # Promptly clear memory after the issue is resolved. return last_talk @staticmethod @@ -138,10 +137,9 @@ class Assistant(Role): async def main(): - options = Config().runtime_options - cost_manager = CostManager(**options) + cost_manager = CostManager() topic = "what's apple" - role = Assistant(options=options, cost_manager=cost_manager, language="Chinese") + role = Assistant(cost_manager=cost_manager, language="Chinese") await role.talk(topic) while True: has_action = await role.think() @@ -156,4 +154,5 @@ async def main(): if __name__ == '__main__': + CONFIG.language = "Chinese" asyncio.run(main()) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 1fd36e78c..e9bb55bed 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -13,7 +13,6 @@ import base64 import sys sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initialize_environment from metagpt.logs import logger from aiofile import async_open from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer @@ -109,8 +108,6 @@ async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscripti if __name__ == "__main__": - initialize_environment() - loop = asyncio.new_event_loop() v = loop.create_task(oas3_azsure_tts("测试，test")) loop.run_until_complete(v) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index 624bb7d93..5c23f6566 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -13,13 +13,10 @@ import sys import connexion sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initialize_environment def oas_http_svc(): """Start the OAS 3.0 OpenAPI HTTP service""" - initialize_environment() - app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') app.add_api("metagpt_oas3_api.yaml") app.add_api("openapi.yaml") diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index bc551134a..43d22961b 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -17,7 +17,6 @@ import requests from pydantic import BaseModel sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initialize_environment from metagpt.logs import logger @@ -104,8 +103,6 @@ async def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url if __name__ == "__main__": - initialize_environment() - v = oas3_metagpt_text_to_image("Panda emoji") data = base64.b64decode(v) with open("tmp.png", mode="wb") as writer: diff --git a/metagpt/tools/openai_text_to_image.py b/metagpt/tools/openai_text_to_image.py index cd48c62af..052a429ae 100644 --- a/metagpt/tools/openai_text_to_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -17,7 +17,6 @@ import requests from pydantic import BaseModel sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initialize_environment from metagpt.logs import logger @@ -96,7 +95,5 @@ async def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_a if __name__ == "__main__": - initialize_environment() - v = oas3_openai_text_to_image("Panda emoji") print(v) diff --git a/tests/metagpt/tools/test_azure_tts.py b/tests/metagpt/tools/test_azure_tts.py index 41d429109..0a2ca4071 100644 --- a/tests/metagpt/tools/test_azure_tts.py +++ b/tests/metagpt/tools/test_azure_tts.py @@ -14,12 +14,9 @@ from pathlib import Path sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.const import WORKSPACE_ROOT from metagpt.tools.azure_tts import AzureTTS -from metagpt.utils.common import initialize_environment def test_azure_tts(): - initialize_environment() - azure_tts = AzureTTS(subscription_key="", region="") text = """ 女儿看见父亲走了进来，问道： From 58369c4e3a402d9cb04142579fbc0ad9421f9559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 21:01:15 +0800 Subject: [PATCH 127/398] feat: fix coding --- metagpt/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/config.py b/metagpt/config.py index f1c869b6c..05949408d 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -50,7 +50,7 @@ class Config(metaclass=Singleton): if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): - raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") + logger.warning("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") if not self.openai_api_base or "YOUR_API_BASE" == self.openai_api_base: openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy From e201bf71d912542a4b4541528881583cb28e128a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 22:04:06 +0800 Subject: [PATCH 128/398] fixbug: CONFIG initialization --- metagpt/config.py | 17 +++++-- metagpt/provider/openai_api.py | 88 ++++------------------------------ metagpt/roles/role.py | 7 +-- metagpt/software_company.py | 7 +-- metagpt/utils/cost_manager.py | 79 ++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 89 deletions(-) create mode 100644 metagpt/utils/cost_manager.py diff --git a/metagpt/config.py b/metagpt/config.py index 05949408d..4cae79b17 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -4,6 +4,7 @@ Provide configuration, singleton. @Modified BY: mashenquan, 2023/8/28. Replace the global variable `CONFIG` with `ContextVar`. """ +import json import os from copy import deepcopy from typing import Any @@ -14,6 +15,7 @@ import yaml from metagpt.const import PROJECT_ROOT, OPTIONS from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType +from metagpt.utils.cost_manager import CostManager from metagpt.utils.singleton import Singleton @@ -43,12 +45,17 @@ class Config(metaclass=Singleton): def __init__(self, yaml_file=default_yaml_file): self._init_with_config_files_and_env(yaml_file) + self.cost_manager = CostManager(**json.loads(self.COST_MANAGER)) if self.COST_MANAGER else CostManager() + logger.info("Config loading done.") + self._update() + + def _update(self): self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): logger.warning("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -78,8 +85,7 @@ class Config(metaclass=Singleton): self.long_term_memory = self._get("LONG_TERM_MEMORY", False) if self.long_term_memory: logger.warning("LONG_TERM_MEMORY is True") - self.max_budget = self._get("MAX_BUDGET", 10.0) - self.total_cost = 0.0 + self.cost_manager.max_budget = self._get("MAX_BUDGET", 10.0) self.puppeteer_config = self._get("PUPPETEER_CONFIG", "") self.mmdc = self._get("MMDC", "mmdc") @@ -109,7 +115,8 @@ class Config(metaclass=Singleton): return m.get(*args, **kwargs) def get(self, key, *args, **kwargs): - """Retrieve values from config/key.yaml, config/config.yaml, and environment variables. Throw an error if not found.""" + """Retrieve values from config/key.yaml, config/config.yaml, and environment variables. + Throw an error if not found.""" value = self._get(key, *args, **kwargs) if value is None: raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") @@ -127,10 +134,12 @@ class Config(metaclass=Singleton): opts = deepcopy(OPTIONS.get()) opts.update(options) OPTIONS.set(opts) + self._update() @property def options(self): """Return all key-values""" return OPTIONS.get() + CONFIG = Config() diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 7dba00530..e4dfade78 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -11,19 +11,18 @@ import re import time import random -from typing import NamedTuple, List +from typing import List import traceback import openai from openai.error import APIConnectionError -from pydantic import BaseModel from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type from metagpt.config import CONFIG from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.utils.cost_manager import Costs from metagpt.utils.token_counter import ( - TOKEN_COSTS, count_message_tokens, count_string_tokens, get_max_completion_tokens, @@ -55,73 +54,6 @@ class RateLimiter: self.last_call_time = time.time() -class Costs(NamedTuple): - total_prompt_tokens: int - total_completion_tokens: int - total_cost: float - total_budget: float - - -class CostManager(BaseModel): - """计算使用接口的开销""" - - total_prompt_tokens: int = 0 - total_completion_tokens: int = 0 - total_budget: float = 0 - max_budget: float = CONFIG.max_budget - total_cost: float = 0 - - def update_cost(self, prompt_tokens, completion_tokens, model): - """ - Update the total cost, prompt tokens, and completion tokens. - - Args: - prompt_tokens (int): The number of tokens used in the prompt. - completion_tokens (int): The number of tokens used in the completion. - model (str): The model used for the API call. - """ - self.total_prompt_tokens += prompt_tokens - self.total_completion_tokens += completion_tokens - cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model][ - "completion"]) / 1000 - self.total_cost += cost - logger.info( - f"Total running cost: ${self.total_cost:.3f} | Max budget: ${self.max_budget:.3f} | " - f"Current cost: ${cost:.3f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}" - ) - - def get_total_prompt_tokens(self): - """ - Get the total number of prompt tokens. - - Returns: - int: The total number of prompt tokens. - """ - return self.total_prompt_tokens - - def get_total_completion_tokens(self): - """ - Get the total number of completion tokens. - - Returns: - int: The total number of completion tokens. - """ - return self.total_completion_tokens - - def get_total_cost(self): - """ - Get the total cost of API calls. - - Returns: - float: The total cost of API calls. - """ - return self.total_cost - - def get_costs(self) -> Costs: - """获得所有开销""" - return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) - - def log_and_reraise(retry_state): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") logger.warning(""" @@ -136,12 +68,11 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): Check https://platform.openai.com/examples for examples """ - def __init__(self, cost_manager=None): + def __init__(self): self.__init_openai(CONFIG) self.llm = openai self.model = CONFIG.openai_api_model self.auto_max_tokens = False - self._cost_manager = cost_manager or CostManager() RateLimiter.__init__(self, rpm=self.rpm) def __init_openai(self, config): @@ -155,9 +86,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def _achat_completion_stream(self, messages: list[dict]) -> str: response = await self.async_retry_call(openai.ChatCompletion.acreate, - **self._cons_kwargs(messages), - stream=True - ) + **self._cons_kwargs(messages), + stream=True + ) # create variables to collect the stream of chunks collected_chunks = [] collected_messages = [] @@ -276,12 +207,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): try: prompt_tokens = int(usage['prompt_tokens']) completion_tokens = int(usage['completion_tokens']) - self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) + CONFIG.cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) except Exception as e: logger.error("updating costs failed!", e) def get_costs(self) -> Costs: - return self._cost_manager.get_costs() + return CONFIG.cost_manager.get_costs() def get_max_tokens(self, messages: list[dict]): if not self.auto_max_tokens: @@ -366,7 +297,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return None, input_string @staticmethod - async def async_retry_call(func, *args, **kwargs): + async def async_retry_call(func, *args, **kwargs): for i in range(OpenAIGPTAPI.MAX_TRY): try: rsp = await func(*args, **kwargs) @@ -399,4 +330,3 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 - diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index a1ac0d9e7..5d2cce802 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -9,13 +9,14 @@ """ from __future__ import annotations -from typing import Iterable, Type, Dict +from typing import Iterable, Type + from pydantic import BaseModel, Field -from metagpt.config import Config, CONFIG +from metagpt.config import CONFIG from metagpt.const import OPTIONS -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager +from metagpt.llm import LLM from metagpt.actions import Action, ActionOutput from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 8d9c990ee..cfa3bd492 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -35,12 +35,13 @@ class SoftwareCompany(BaseModel): def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" self.investment = investment - CONFIG.max_budget = investment + CONFIG.cost_manager.max_budget = investment logger.info(f'Investment: ${investment}.') def _check_balance(self): - if CONFIG.total_cost > CONFIG.max_budget: - raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') + if CONFIG.cost_manager.total_cost > CONFIG.cost_manager.max_budget: + raise NoMoneyException(CONFIG.cost_manager.total_cost, + f'Insufficient funds: {CONFIG.cost_manager.max_budget}') def start_project(self, idea, role="BOSS", cause_by=BossRequirement, **kwargs): """Start a project from publishing boss requirement.""" diff --git a/metagpt/utils/cost_manager.py b/metagpt/utils/cost_manager.py new file mode 100644 index 000000000..21b37d552 --- /dev/null +++ b/metagpt/utils/cost_manager.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/28 +@Author : mashenquan +@File : openai.py +@Desc : mashenquan, 2023/8/28. Separate the `CostManager` class to support user-level cost accounting. +""" + +from pydantic import BaseModel +from metagpt.logs import logger +from metagpt.utils.token_counter import TOKEN_COSTS +from typing import NamedTuple + + +class Costs(NamedTuple): + total_prompt_tokens: int + total_completion_tokens: int + total_cost: float + total_budget: float + + +class CostManager(BaseModel): + """Calculate the overhead of using the interface.""" + + total_prompt_tokens: int = 0 + total_completion_tokens: int = 0 + total_budget: float = 0 + max_budget: float = 10.0 + total_cost: float = 0 + + def update_cost(self, prompt_tokens, completion_tokens, model): + """ + Update the total cost, prompt tokens, and completion tokens. + + Args: + prompt_tokens (int): The number of tokens used in the prompt. + completion_tokens (int): The number of tokens used in the completion. + model (str): The model used for the API call. + """ + self.total_prompt_tokens += prompt_tokens + self.total_completion_tokens += completion_tokens + cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model][ + "completion"]) / 1000 + self.total_cost += cost + logger.info( + f"Total running cost: ${self.total_cost:.3f} | Max budget: ${self.max_budget:.3f} | " + f"Current cost: ${cost:.3f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}" + ) + + def get_total_prompt_tokens(self): + """ + Get the total number of prompt tokens. + + Returns: + int: The total number of prompt tokens. + """ + return self.total_prompt_tokens + + def get_total_completion_tokens(self): + """ + Get the total number of completion tokens. + + Returns: + int: The total number of completion tokens. + """ + return self.total_completion_tokens + + def get_total_cost(self): + """ + Get the total cost of API calls. + + Returns: + float: The total cost of API calls. + """ + return self.total_cost + + def get_costs(self) -> Costs: + """获得所有开销""" + return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) From 2a5b263371491a4be1799812d5dbd2f08c4c92c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 22:09:40 +0800 Subject: [PATCH 129/398] fixbug: CONFIG initialization --- metagpt/roles/assistant.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 944b250f1..57cb28e67 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -20,12 +20,10 @@ from pathlib import Path from metagpt.actions import ActionOutput from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction from metagpt.actions.talk_action import TalkAction -from metagpt.config import Config, CONFIG -from metagpt.const import BRAIN_MEMORY, SKILL_PATH +from metagpt.config import CONFIG from metagpt.learn.skill_loader import SkillLoader from metagpt.logs import logger from metagpt.memory.brain_memory import BrainMemory, MessageType -from metagpt.provider.openai_api import CostManager from metagpt.roles import Role from metagpt.schema import Message @@ -137,9 +135,8 @@ class Assistant(Role): async def main(): - cost_manager = CostManager() topic = "what's apple" - role = Assistant(cost_manager=cost_manager, language="Chinese") + role = Assistant(language="Chinese") await role.talk(topic) while True: has_action = await role.think() From b904607aab0e0c5567c785444e7a449852465bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 22:46:34 +0800 Subject: [PATCH 130/398] fixbug: async --- metagpt/actions/skill_action.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index c921a5f17..e5bd32dae 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -76,16 +76,16 @@ class SkillAction(Action): async def run(self, *args, **kwargs) -> str | ActionOutput | None: """Run action""" - self.rsp = self.find_and_call_function(self._skill.name, args=self._args, **kwargs) + self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **kwargs) return ActionOutput(content=self.rsp, instruct_content=self._skill.json()) @staticmethod - def find_and_call_function(function_name, args, **kwargs): + async def find_and_call_function(function_name, args, **kwargs): try: module = importlib.import_module("metagpt.learn") function = getattr(module, function_name) # 调用函数并返回结果 - result = function(**args, **kwargs) + result = await function(**args, **kwargs) return result except (ModuleNotFoundError, AttributeError): logger.error(f"{function_name} not found") From 1903da126fe3802b5558e3366f0052c55e19298b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 22:59:35 +0800 Subject: [PATCH 131/398] fixbug: async --- metagpt/actions/skill_action.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index e5bd32dae..fb801b454 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -9,6 +9,7 @@ import ast import importlib +import traceback from metagpt.actions import Action, ActionOutput from metagpt.learn.skill_loader import Skill @@ -76,7 +77,11 @@ class SkillAction(Action): async def run(self, *args, **kwargs) -> str | ActionOutput | None: """Run action""" - self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **kwargs) + try: + self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **kwargs) + except Exception as e: + logger.exception(f"{e}, traceback:{traceback.format_exc()}") + self.rsp = f"Error: {e}" return ActionOutput(content=self.rsp, instruct_content=self._skill.json()) @staticmethod From 2ba457a6096afaa3b7d34d78fbaa17844aae552c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 10:24:06 +0800 Subject: [PATCH 132/398] feat: +exception catch --- metagpt/provider/openai_api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index e4dfade78..75ac38860 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -323,6 +323,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): except openai.error.RateLimitError as e: logger.warning(f"Exception:{e}") continue + except (openai.error.AuthenticationError, + openai.error.PermissionError, + openai.error.InvalidAPIType, + openai.error.SignatureVerificationError) as e: + logger.warning(f"Exception:{e}") + raise e except Exception as e: error_str = traceback.format_exc() logger.error(f"Exception:{e}, stack:{error_str}") From 91b7552f09a69cfc672480b1df3701c0b3c9a8da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 11:33:50 +0800 Subject: [PATCH 133/398] fixbug: fix get_by_tags --- metagpt/roles/role.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 5d2cce802..aba7d4574 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -95,7 +95,9 @@ class RoleContext(BaseModel): @property def prerequisite(self): """Retrieve information with `prerequisite` tag""" - return self.memory.get_by_tags([MessageTag.Prerequisite.value]) + if self.memory and hasattr(self.memory, 'get_by_tags'): + return self.memory.get_by_tags([MessageTag.Prerequisite.value]) + return "" class Role: From 0aaf04100cd09d138dcf211d314fb8b22b85b36d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 11:40:13 +0800 Subject: [PATCH 134/398] fixbug: fix get_by_tags --- metagpt/roles/role.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index aba7d4574..efb8db9f8 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -92,13 +92,6 @@ class RoleContext(BaseModel): def history(self) -> list[Message]: return self.memory.get() - @property - def prerequisite(self): - """Retrieve information with `prerequisite` tag""" - if self.memory and hasattr(self.memory, 'get_by_tags'): - return self.memory.get_by_tags([MessageTag.Prerequisite.value]) - return "" - class Role: """Role/Proxy""" @@ -208,7 +201,7 @@ class Role: # history=self.history) logger.info(f"{self._setting}: ready to {self._rc.todo}") - requirement = self._rc.important_memory or self._rc.prerequisite + requirement = self._rc.important_memory response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): From 14068cdc19613e78e94654ed898c77c310dce81a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 14:35:35 +0800 Subject: [PATCH 135/398] fixbug: get user query empty --- metagpt/memory/memory.py | 8 ++++++++ metagpt/roles/role.py | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index a96aaf1be..bf9f0541c 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -85,3 +85,11 @@ class Memory: continue rsp += self.index[action] return rsp + + def get_by_tags(self, tags: list) -> list[Message]: + """Return messages with specified tags""" + result = [] + for m in self.storage: + if m.is_contain_tags(tags): + result.append(m) + return result diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index efb8db9f8..aba7d4574 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -92,6 +92,13 @@ class RoleContext(BaseModel): def history(self) -> list[Message]: return self.memory.get() + @property + def prerequisite(self): + """Retrieve information with `prerequisite` tag""" + if self.memory and hasattr(self.memory, 'get_by_tags'): + return self.memory.get_by_tags([MessageTag.Prerequisite.value]) + return "" + class Role: """Role/Proxy""" @@ -201,7 +208,7 @@ class Role: # history=self.history) logger.info(f"{self._setting}: ready to {self._rc.todo}") - requirement = self._rc.important_memory + requirement = self._rc.important_memory or self._rc.prerequisite response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): From 9da450f8a77297067dd7d20940e875b466387823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 16:32:37 +0800 Subject: [PATCH 136/398] feat: + safe code --- metagpt/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/config.py b/metagpt/config.py index 4cae79b17..5944fef57 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -131,6 +131,8 @@ class Config(metaclass=Singleton): def set_context(self, options: dict): """Update current config""" + if not options: + return opts = deepcopy(OPTIONS.get()) opts.update(options) OPTIONS.set(opts) From ef6ec8c8c75181608a0e8be52278a9311e334770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 20:52:45 +0800 Subject: [PATCH 137/398] fixbug: annotation --- examples/write_teaching_plan.py | 7 +++++-- metagpt/actions/action.py | 1 + metagpt/actions/skill_action.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index 191547193..c3a647b94 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -5,15 +5,18 @@ @Author : mashenquan @File : write_teaching_plan.py @Desc: Write teaching plan demo + ``` + export PYTHONPATH=$PYTHONPATH:$PWD + python examples/write_teaching_plan.py --language=Chinese --teaching_language=English + + ``` """ import asyncio from pathlib import Path -import sys from metagpt.config import CONFIG -sys.path.append(str(Path(__file__).resolve().parent.parent)) import aiofiles import fire from metagpt.logs import logger diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 5cf4f3d81..c38c4e1b0 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -6,6 +6,7 @@ @File : action.py @Modified By: mashenquan, 2023/8/20. Add function return annotations. """ +from __future__ import annotations from abc import ABC from typing import Optional diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index fb801b454..3ef0087fc 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -6,7 +6,7 @@ @File : skill_action.py @Desc : Call learned skill """ - +from __future__ import annotations import ast import importlib import traceback From dc14770e3d5ad327ec90e61c52346b9549d567fb Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Wed, 30 Aug 2023 10:53:47 +0800 Subject: [PATCH 138/398] separate workspace --- metagpt/actions/action.py | 12 +- metagpt/actions/design_api.py | 62 ++++------ metagpt/actions/project_management.py | 25 ++-- metagpt/actions/write_code.py | 24 +--- metagpt/actions/write_prd.py | 34 +++++- metagpt/config.py | 11 +- metagpt/roles/engineer.py | 94 +++++++-------- metagpt/roles/qa_engineer.py | 8 +- metagpt/roles/role.py | 43 +++---- metagpt/roles/teacher.py | 44 ++++--- metagpt/tools/sd_engine.py | 3 +- metagpt/utils/mermaid.py | 164 +++++++++++++------------- tests/metagpt/roles/ui_role.py | 4 +- tests/metagpt/tools/test_azure_tts.py | 17 +-- tests/metagpt/tools/test_sd_tool.py | 5 +- 15 files changed, 275 insertions(+), 275 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 5cf4f3d81..e4b9613ad 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -6,6 +6,8 @@ @File : action.py @Modified By: mashenquan, 2023/8/20. Add function return annotations. """ +from __future__ import annotations + from abc import ABC from typing import Optional @@ -13,12 +15,12 @@ from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM -from metagpt.utils.common import OutputParser from metagpt.logs import logger +from metagpt.utils.common import OutputParser class Action(ABC): - def __init__(self, name: str = '', context=None, llm: LLM = None): + def __init__(self, name: str = "", context=None, llm: LLM = None): self.name: str = name if llm is None: llm = LLM() @@ -49,9 +51,9 @@ class Action(ABC): return await self.llm.aask(prompt, system_msgs) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def _aask_v1(self, prompt: str, output_class_name: str, - output_data_mapping: dict, - system_msgs: Optional[list[str]] = None) -> ActionOutput: + async def _aask_v1( + self, prompt: str, output_class_name: str, output_data_mapping: dict, system_msgs: Optional[list[str]] = None + ) -> ActionOutput: """Append default prefix""" if not system_msgs: system_msgs = [] diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index cf23e6ad1..1c31b75fb 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -6,12 +6,12 @@ @File : design_api.py @Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. """ -import shutil -from pathlib import Path from typing import List -from metagpt.actions import Action, ActionOutput -from metagpt.const import WORKSPACE_ROOT +import aiofiles + +from metagpt.actions import Action +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.utils.common import CodeParser from metagpt.utils.mermaid import mermaid_to_file @@ -93,52 +93,32 @@ OUTPUT_MAPPING = { class WriteDesign(Action): def __init__(self, name, context=None, llm=None): super().__init__(name, context, llm) - self.desc = "Based on the PRD, think about the system design, and design the corresponding APIs, " \ - "data structures, library tables, processes, and paths. Please provide your design, feedback " \ - "clearly and in detail." + self.desc = ( + "Based on the PRD, think about the system design, and design the corresponding APIs, " + "data structures, library tables, processes, and paths. Please provide your design, feedback " + "clearly and in detail." + ) - def recreate_workspace(self, workspace: Path): - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # 文件夹不存在，但我们不在意 - workspace.mkdir(parents=True, exist_ok=True) - - def _save_prd(self, docs_path, resources_path, prd): - prd_file = docs_path / 'prd.md' - quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) - mermaid_to_file(quadrant_chart, resources_path / 'competitive_analysis') - logger.info(f"Saving PRD to {prd_file}") - prd_file.write_text(prd) - - def _save_system_design(self, docs_path, resources_path, content): + async def _save_system_design(self, docs_path, resources_path, content): data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content) seq_flow = CodeParser.parse_code(block="Program call flow", text=content) - mermaid_to_file(data_api_design, resources_path / 'data_api_design') - mermaid_to_file(seq_flow, resources_path / 'seq_flow') - system_design_file = docs_path / 'system_design.md' + await mermaid_to_file(data_api_design, resources_path / "data_api_design") + await mermaid_to_file(seq_flow, resources_path / "seq_flow") + system_design_file = docs_path / "system_design.md" logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text(content) + async with aiofiles.open(system_design_file, "w") as f: + await f.write(content) - def _save(self, context, system_design): - if isinstance(system_design, ActionOutput): - content = system_design.content - ws_name = CodeParser.parse_str(block="Python package name", text=content) - else: - content = system_design - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name - self.recreate_workspace(workspace) - docs_path = workspace / 'docs' - resources_path = workspace / 'resources' + async def _save(self, system_design: str): + workspace = CONFIG.workspace + docs_path = workspace / "docs" + resources_path = workspace / "resources" docs_path.mkdir(parents=True, exist_ok=True) resources_path.mkdir(parents=True, exist_ok=True) - self._save_prd(docs_path, resources_path, context[-1].content) - self._save_system_design(docs_path, resources_path, content) + await self._save_system_design(docs_path, resources_path, system_design) async def run(self, context, **kwargs): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) - # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) - self._save(context, system_design) + await self._save(system_design.content) return system_design diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 16473ff01..55e7cbcb5 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -8,11 +8,12 @@ """ from typing import List, Tuple -from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT -from metagpt.utils.common import CodeParser +import aiofiles -PROMPT_TEMPLATE = ''' +from metagpt.actions.action import Action +from metagpt.config import CONFIG + +PROMPT_TEMPLATE = """ # Context {context} @@ -37,7 +38,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. -''' +""" FORMAT_EXAMPLE = ''' --- @@ -103,23 +104,23 @@ OUTPUT_MAPPING = { class WriteTasks(Action): - def __init__(self, name="CreateTasks", context=None, llm=None): super().__init__(name, context, llm) - def _save(self, context, rsp): - ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) - file_path = WORKSPACE_ROOT / ws_name / 'docs/api_spec_and_tasks.md' + async def _save(self, rsp): + file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md" file_path.write_text(rsp.content) # Write requirements.txt - requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt' - requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) + requirements_path = CONFIG.workspace / "requirements.txt" + + async with aiofiles.open(requirements_path, "w") as f: + await f.write(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) async def run(self, context, **kwargs): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) - self._save(context, rsp) + await self._save(rsp) return rsp diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index cc122ef7a..fd54ce699 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -5,13 +5,12 @@ @Author : alexanderwu @File : write_code.py """ -from metagpt.actions import WriteDesign +from tenacity import retry, stop_after_attempt, wait_fixed + from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE @@ -49,23 +48,6 @@ class WriteCode(Action): def _is_invalid(self, filename): return any(i in filename for i in ["mp3", "wav"]) - def _save(self, context, filename, code): - # logger.info(filename) - # logger.info(code_rsp) - if self._is_invalid(filename): - return - - design = [i for i in context if i.cause_by == WriteDesign][0] - - ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name - if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): - ws_path = ws_path / ws_name - code_path = ws_path / filename - code_path.parent.mkdir(parents=True, exist_ok=True) - code_path.write_text(code) - logger.info(f"Saving Code to {code_path}") - @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt): code_rsp = await self._aask(prompt) @@ -74,7 +56,7 @@ class WriteCode(Action): async def run(self, context, filename): prompt = PROMPT_TEMPLATE.format(context=context, filename=filename) - logger.info(f'Writing {filename}..') + logger.info(f"Writing {filename}..") code = await self.write_code(prompt) # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 0edd24d55..97f9138fd 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -7,9 +7,14 @@ """ from typing import List, Tuple +import aiofiles + from metagpt.actions import Action, ActionOutput from metagpt.actions.search_and_summarize import SearchAndSummarize +from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.utils.common import CodeParser +from metagpt.utils.mermaid import mermaid_to_file PROMPT_TEMPLATE = """ # Context @@ -121,7 +126,7 @@ OUTPUT_MAPPING = { "Competitive Quadrant Chart": (str, ...), "Requirement Analysis": (str, ...), "Requirement Pool": (List[Tuple[str, str]], ...), - "UI Design draft":(str, ...), + "UI Design draft": (str, ...), "Anything UNCLEAR": (str, ...), } @@ -139,8 +144,31 @@ class WritePRD(Action): logger.info(sas.result) logger.info(rsp) - prompt = PROMPT_TEMPLATE.format(requirements=requirements, search_information=info, - format_example=FORMAT_EXAMPLE) + prompt = PROMPT_TEMPLATE.format( + requirements=requirements, search_information=info, format_example=FORMAT_EXAMPLE + ) logger.debug(prompt) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) + + await self._save(prd.content) return prd + + async def _save_prd(self, docs_path, resources_path, prd): + prd_file = docs_path / "prd.md" + quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) + await mermaid_to_file( + mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / "competitive_analysis" + ) + async with aiofiles.open(prd_file, "w") as f: + await f.write(prd) + logger.info(f"Saving PRD to {prd_file}") + + async def _save(self, prd): + workspace = CONFIG.workspace + workspace.mkdir(parents=True, exist_ok=True) + + docs_path = workspace / "docs" + resources_path = workspace / "resources" + docs_path.mkdir(parents=True, exist_ok=True) + resources_path.mkdir(parents=True, exist_ok=True) + await self._save_prd(docs_path, resources_path, prd) diff --git a/metagpt/config.py b/metagpt/config.py index 5944fef57..908faaaaf 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -4,15 +4,17 @@ Provide configuration, singleton. @Modified BY: mashenquan, 2023/8/28. Replace the global variable `CONFIG` with `ContextVar`. """ +import datetime import json import os from copy import deepcopy from typing import Any +from uuid import uuid4 import openai import yaml -from metagpt.const import PROJECT_ROOT, OPTIONS +from metagpt.const import OPTIONS, PROJECT_ROOT, WORKSPACE_ROOT from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.cost_manager import CostManager @@ -55,7 +57,7 @@ class Config(metaclass=Singleton): self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): logger.warning("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -93,6 +95,11 @@ class Config(metaclass=Singleton): self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") + workspace_uid = ( + self._get("WORKSPACE_UID") or f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[-8:]}" + ) + self.workspace = WORKSPACE_ROOT / workspace_uid + def _init_with_config_files_and_env(self, yaml_file): """从config/key.yaml / config/config.yaml / env三处按优先级递减加载""" configs = dict(os.environ) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 072e53998..97d0af087 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -6,17 +6,18 @@ @File : engineer.py """ import asyncio -import shutil from collections import OrderedDict from pathlib import Path -from metagpt.const import WORKSPACE_ROOT +import aiofiles + +from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role -from metagpt.actions import WriteCode, WriteCodeReview, WriteTasks, WriteDesign from metagpt.schema import Message from metagpt.utils.common import CodeParser -from metagpt.utils.special_tokens import MSG_SEP, FILENAME_CODE_SEP +from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP async def gather_ordered_k(coros, k) -> list: @@ -47,9 +48,15 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): - def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", - constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", - n_borg=1, use_code_review=False): + def __init__( + self, + name="Alex", + profile="Engineer", + goal="Write elegant, readable, extensible, efficient code", + constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", + n_borg=1, + use_code_review=False, + ): super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review @@ -72,31 +79,24 @@ class Engineer(Role): @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip("\"") + return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"') return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / 'src' + return CONFIG.workspace / "src" workspace = self.parse_workspace(msg) # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace / workspace - def recreate_workspace(self): + async def write_file(self, filename: str, code: str): workspace = self.get_workspace() - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # 文件夹不存在，但我们不在意 - workspace.mkdir(parents=True, exist_ok=True) - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() - filename = filename.replace('"', '').replace('\n', '') + filename = filename.replace('"', "").replace("\n", "") file = workspace / filename file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) + async with aiofiles.open(file, "w") as f: + await f.write(code) return file def recv(self, message: Message) -> None: @@ -109,8 +109,7 @@ class Engineer(Role): todo_coros = [] for todo in self.todos: todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), - filename=todo + context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo ) todo_coros.append(todo_coro) @@ -124,38 +123,40 @@ class Engineer(Role): self._rc.memory.add(msg) del self.todos[0] - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f"Done {self.get_workspace()} generating.") msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) return msg async def _act_sp(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + instruct_content = {} for todo in self.todos: - code = await WriteCode().run( - context=self._rc.history, - filename=todo - ) + code = await WriteCode().run(context=self._rc.history, filename=todo) # logger.info(todo) # logger.info(code_rsp) # code = self.parse_code(code_rsp) - file_path = self.write_file(todo, code) + file_path = await self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) + instruct_content[todo] = code - code_msg = todo + FILENAME_CODE_SEP + str(file_path) + # code_msg = todo + FILENAME_CODE_SEP + str(file_path) + code_msg = (todo, file_path) code_msg_all.append(code_msg) - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f"Done {self.get_workspace()} generating.") msg = Message( - content=MSG_SEP.join(code_msg_all), + content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all), + instruct_content=instruct_content, role=self.profile, cause_by=type(self._rc.todo), - send_to="QaEngineer" + send_to="QaEngineer", ) return msg async def _act_sp_precision(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + instruct_content = {} for todo in self.todos: """ # 从历史信息中挑选必须的信息，以减少prompt长度（人工经验总结） @@ -170,35 +171,30 @@ class Engineer(Role): context.append(m.content) context_str = "\n".join(context) # 编写code - code = await WriteCode().run( - context=context_str, - filename=todo - ) + code = await WriteCode().run(context=context_str, filename=todo) # code review if self.use_code_review: try: - rewrite_code = await WriteCodeReview().run( - context=context_str, - code=code, - filename=todo - ) + rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) code = rewrite_code except Exception as e: logger.error("code review failed!", e) pass - file_path = self.write_file(todo, code) + file_path = await self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) + instruct_content[todo] = code - code_msg = todo + FILENAME_CODE_SEP + str(file_path) + code_msg = (todo, file_path) code_msg_all.append(code_msg) - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f"Done {self.get_workspace()} generating.") msg = Message( - content=MSG_SEP.join(code_msg_all), + content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all), + instruct_content=instruct_content, role=self.profile, cause_by=type(self._rc.todo), - send_to="QaEngineer" + send_to="QaEngineer", ) return msg diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 65bf2cc5b..491f5f997 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -9,7 +9,7 @@ import os from pathlib import Path from metagpt.actions import DebugError, RunCode, WriteCode, WriteDesign, WriteTest -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -43,13 +43,13 @@ class QaEngineer(Role): def get_workspace(self, return_proj_dir=True) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace / "src" workspace = self.parse_workspace(msg) # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. if return_proj_dir: - return WORKSPACE_ROOT / workspace + return CONFIG.workspace / workspace # development codes directory: workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace / workspace / workspace def write_file(self, filename: str, code: str): workspace = self.get_workspace() / "tests" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index aba7d4574..2f0f713f8 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -11,15 +11,14 @@ from __future__ import annotations from typing import Iterable, Type - from pydantic import BaseModel, Field +from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG from metagpt.const import OPTIONS from metagpt.llm import LLM -from metagpt.actions import Action, ActionOutput from metagpt.logs import logger -from metagpt.memory import Memory, LongTermMemory +from metagpt.memory import LongTermMemory, Memory from metagpt.schema import Message, MessageTag PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -52,6 +51,7 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi class RoleSetting(BaseModel): """Role properties""" + name: str profile: str goal: str @@ -67,7 +67,8 @@ class RoleSetting(BaseModel): class RoleContext(BaseModel): """Runtime role context""" - env: 'Environment' = Field(default=None) + + env: "Environment" = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=0) @@ -95,7 +96,7 @@ class RoleContext(BaseModel): @property def prerequisite(self): """Retrieve information with `prerequisite` tag""" - if self.memory and hasattr(self.memory, 'get_by_tags'): + if self.memory and hasattr(self.memory, "get_by_tags"): return self.memory.get_by_tags([MessageTag.Prerequisite.value]) return "" @@ -145,7 +146,7 @@ class Role: logger.debug(self._actions) self._rc.todo = self._actions[self._rc.state] - def set_env(self, env: 'Environment'): + def set_env(self, env: "Environment"): """设置角色工作所处的环境，角色可以向环境说话，也可以通过观察接受环境消息""" self._rc.env = env @@ -192,12 +193,13 @@ class Role: self._set_state(0) return True prompt = self._get_prefix() - prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), - n_states=len(self._states) - 1) + prompt += STATE_TEMPLATE.format( + history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1 + ) next_state = await self._llm.aask(prompt) logger.debug(f"{prompt=}") if not next_state.isdigit() or int(next_state) not in range(len(self._states)): - logger.warning(f'Invalid answer of state, {next_state=}') + logger.warning(f"Invalid answer of state, {next_state=}") next_state = "0" self._set_state(int(next_state)) return True @@ -212,8 +214,12 @@ class Role: response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) + msg = Message( + content=response.content, + instruct_content=response.instruct_content, + role=self.profile, + cause_by=type(self._rc.todo), + ) else: msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) @@ -236,7 +242,7 @@ class Role: news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news] if news_text: - logger.debug(f'{self._setting} observed: {news_text}') + logger.debug(f"{self._setting} observed: {news_text}") return len(self._rc.news) def _publish_message(self, msg): @@ -310,20 +316,15 @@ class Role: def add_to_do(self, act): self._rc.todo = act - async def think(self) -> bool: + async def think(self) -> Action: """The exported `think` function""" - has_action = await self._think() - if not has_action: - return False - if not self._rc.todo: - return False - return True + await self._think() + return self._rc.todo async def act(self) -> ActionOutput: """The exported `act` function""" msg = await self._act() - return ActionOutput(content=msg.content, - instruct_content=msg.instruct_content) + return ActionOutput(content=msg.content, instruct_content=msg.instruct_content) @property def todo_description(self): diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index ca88fd681..031ce94c9 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -9,22 +9,34 @@ """ +import re + import aiofiles -from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart, TeachingPlanRequirement -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.write_teaching_plan import ( + TeachingPlanRequirement, + WriteTeachingPlanPart, +) +from metagpt.config import CONFIG +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger -import re class Teacher(Role): """Support configurable teacher roles, with native and teaching languages being replaceable through configurations.""" - def __init__(self, name='Lily', profile='{teaching_language} Teacher', - goal='writing a {language} teaching plan part by part', - constraints='writing in {language}', desc="", *args, **kwargs): + + def __init__( + self, + name="Lily", + profile="{teaching_language} Teacher", + goal="writing a {language} teaching plan part by part", + constraints="writing in {language}", + desc="", + *args, + **kwargs, + ): super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) actions = [] for topic in WriteTeachingPlanPart.TOPICS: @@ -54,7 +66,7 @@ class Teacher(Role): break logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") msg = await self._act() - if ret.content != '': + if ret.content != "": ret.content += "\n\n\n" ret.content += msg.content logger.info(ret.content) @@ -64,14 +76,14 @@ class Teacher(Role): async def save(self, content): """Save teaching plan""" filename = Teacher.new_file_name(self.course_title) - pathname = WORKSPACE_ROOT / "teaching_plan" + pathname = CONFIG.workspace / "teaching_plan" pathname.mkdir(exist_ok=True) pathname = pathname / filename try: - async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer: + async with aiofiles.open(str(pathname), mode="w", encoding="utf-8") as writer: await writer.write(content) except Exception as e: - logger.error(f'Save failed：{e}') + logger.error(f"Save failed：{e}") logger.info(f"Save to:{pathname}") @staticmethod @@ -80,8 +92,8 @@ class Teacher(Role): # Define the special characters that need to be replaced. illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' # Replace the special characters with underscores. - filename = re.sub(illegal_chars, '_', lesson_title) + ext - return re.sub(r'_+', '_', filename) + filename = re.sub(illegal_chars, "_", lesson_title) + ext + return re.sub(r"_+", "_", filename) @property def course_title(self): @@ -93,9 +105,9 @@ class Teacher(Role): if act.rsp is None: return default_title title = act.rsp.lstrip("# \n") - if '\n' in title: - ix = title.index('\n') - title = title[0: ix] + if "\n" in title: + ix = title.index("\n") + title = title[0:ix] return title return default_title diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index a63dbe5ac..c33f67a51 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -14,7 +14,6 @@ from aiohttp import ClientSession from PIL import Image, PngImagePlugin from metagpt.config import Config -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger config = Config() @@ -81,7 +80,7 @@ class SDEngine: return self.payload def _save(self, imgs, save_name=""): - save_dir = WORKSPACE_ROOT / "resources" / "SD_Output" + save_dir = CONFIG.get_workspace() / "resources" / "SD_Output" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) batch_decode_base64_to_image(imgs, save_dir, save_name=save_name) diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 1245671fb..15fd08625 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -6,19 +6,20 @@ @File : mermaid.py @Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ -import subprocess +import asyncio from pathlib import Path -from metagpt.config import Config +# from metagpt.utils.common import check_cmd_exists +import aiofiles + +from metagpt.config import CONFIG, Config from metagpt.const import PROJECT_ROOT from metagpt.logs import logger -from metagpt.utils.common import check_cmd_exists -def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: +async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: """suffix: png/svg/pdf - :param options: runtime context options, created by `Config` class object and changed in flow pipeline :param mermaid_code: mermaid code :param output_file_without_suffix: output filename :param width: @@ -27,92 +28,87 @@ def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=204 """ # Write the Mermaid code to a temporary file tmp = Path(f"{output_file_without_suffix}.mmd") - tmp.write_text(mermaid_code, encoding="utf-8") + async with aiofiles.open(tmp, "w", encoding="utf-8") as f: + await f.write(mermaid_code) + # tmp.write_text(mermaid_code, encoding="utf-8") - if check_cmd_exists("mmdc") != 0: - logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") - return -1 + # if check_cmd_exists("mmdc") != 0: + # logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") + # return -1 - for suffix in ["pdf", "svg", "png"]: + # for suffix in ["pdf", "svg", "png"]: + for suffix in ["png"]: output_file = f"{output_file_without_suffix}.{suffix}" # Call the `mmdc` command to convert the Mermaid code to a PNG logger.info(f"Generating {output_file}..") + cmds = [CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)] - if options.get("puppeteer_config"): - subprocess.run( - [ - options.get("mmdc"), - "-p", - options.get("puppeteer_config"), - "-i", - str(tmp), - "-o", - output_file, - "-w", - str(width), - "-H", - str(height), - ] - ) - else: - subprocess.run([options.get("mmdc"), "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) - return 0 - - -MMC1 = """classDiagram - class Main { - -SearchEngine search_engine - +main() str - } - class SearchEngine { - -Index index - -Ranking ranking - -Summary summary - +search(query: str) str - } - class Index { - -KnowledgeBase knowledge_base - +create_index(data: dict) - +query_index(query: str) list - } - class Ranking { - +rank_results(results: list) list - } - class Summary { - +summarize_results(results: list) str - } - class KnowledgeBase { - +update(data: dict) - +fetch_data(query: str) dict - } - Main --> SearchEngine - SearchEngine --> Index - SearchEngine --> Ranking - SearchEngine --> Summary - Index --> KnowledgeBase""" - -MMC2 = """sequenceDiagram - participant M as Main - participant SE as SearchEngine - participant I as Index - participant R as Ranking - participant S as Summary - participant KB as KnowledgeBase - M->>SE: search(query) - SE->>I: query_index(query) - I->>KB: fetch_data(query) - KB-->>I: return data - I-->>SE: return results - SE->>R: rank_results(results) - R-->>SE: return ranked_results - SE->>S: summarize_results(ranked_results) - S-->>SE: return summary - SE-->>M: return summary""" + if CONFIG.puppeteer_config: + cmds.extend(["-p", CONFIG.puppeteer_config]) + process = await asyncio.create_subprocess_exec(*cmds) + await process.wait() + return process.returncode if __name__ == "__main__": + MMC1 = """classDiagram + class Main { + -SearchEngine search_engine + +main() str + } + class SearchEngine { + -Index index + -Ranking ranking + -Summary summary + +search(query: str) str + } + class Index { + -KnowledgeBase knowledge_base + +create_index(data: dict) + +query_index(query: str) list + } + class Ranking { + +rank_results(results: list) list + } + class Summary { + +summarize_results(results: list) str + } + class KnowledgeBase { + +update(data: dict) + +fetch_data(query: str) dict + } + Main --> SearchEngine + SearchEngine --> Index + SearchEngine --> Ranking + SearchEngine --> Summary + Index --> KnowledgeBase""" + + MMC2 = """sequenceDiagram + participant M as Main + participant SE as SearchEngine + participant I as Index + participant R as Ranking + participant S as Summary + participant KB as KnowledgeBase + M->>SE: search(query) + SE->>I: query_index(query) + I->>KB: fetch_data(query) + KB-->>I: return data + I-->>SE: return results + SE->>R: rank_results(results) + R-->>SE: return ranked_results + SE->>S: summarize_results(ranked_results) + S-->>SE: return summary + SE-->>M: return summary""" + conf = Config() - mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC1, - output_file_without_suffix=PROJECT_ROOT / "tmp/1.png") - mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC2, - output_file_without_suffix=PROJECT_ROOT / "tmp/2.png") + asyncio.run( + mermaid_to_file( + options=conf.runtime_options, mermaid_code=MMC1, output_file_without_suffix=PROJECT_ROOT / "tmp/1.png" + ) + ) + asyncio.run( + mermaid_to_file( + options=conf.runtime_options, mermaid_code=MMC2, output_file_without_suffix=PROJECT_ROOT / "tmp/2.png" + ) + ) diff --git a/tests/metagpt/roles/ui_role.py b/tests/metagpt/roles/ui_role.py index a45a89cde..8e9660e36 100644 --- a/tests/metagpt/roles/ui_role.py +++ b/tests/metagpt/roles/ui_role.py @@ -8,7 +8,7 @@ from functools import wraps from importlib import import_module from metagpt.actions import Action, ActionOutput, WritePRD -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -214,7 +214,7 @@ class UIDesign(Action): logger.info("Finish icon design using StableDiffusion API") async def _save(self, css_content, html_content): - save_dir = WORKSPACE_ROOT / "resources" / "codes" + save_dir = CONFIG.workspace / "resources" / "codes" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) # Save CSS and HTML content to files diff --git a/tests/metagpt/tools/test_azure_tts.py b/tests/metagpt/tools/test_azure_tts.py index 0a2ca4071..b7f94a19c 100644 --- a/tests/metagpt/tools/test_azure_tts.py +++ b/tests/metagpt/tools/test_azure_tts.py @@ -8,11 +8,8 @@ @Modified By: mashenquan, 2023-8-17, move to `tools` folder. """ import asyncio -import sys -from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.tools.azure_tts import AzureTTS @@ -28,15 +25,13 @@ def test_azure_tts(): “Writing a binary file in Python is similar to writing a regular text file, but you'll work with bytes instead of strings.” """ - path = WORKSPACE_ROOT / "tts" + path = CONFIG.workspace / "tts" path.mkdir(exist_ok=True, parents=True) filename = path / "girl.wav" loop = asyncio.new_event_loop() - v = loop.create_task(azure_tts.synthesize_speech( - lang="zh-CN", - voice="zh-CN-XiaomoNeural", - text=text, - output_file=str(filename))) + v = loop.create_task( + azure_tts.synthesize_speech(lang="zh-CN", voice="zh-CN-XiaomoNeural", text=text, output_file=str(filename)) + ) result = loop.run_until_complete(v) print(result) @@ -45,5 +40,5 @@ def test_azure_tts(): # TODO: 这里如果要检验，还要额外加上对应的asr，才能确保前后生成是接近一致的，但现在还没有 -if __name__ == '__main__': +if __name__ == "__main__": test_azure_tts() diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index 77e53c7dc..89c97f5e8 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -4,7 +4,8 @@ # import os -from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.tools.sd_engine import SDEngine def test_sd_engine_init(): @@ -21,5 +22,5 @@ def test_sd_engine_generate_prompt(): async def test_sd_engine_run_t2i(): sd_engine = SDEngine() await sd_engine.run_t2i(prompts=["test"]) - img_path = WORKSPACE_ROOT / "resources" / "SD_Output" / "output_0.png" + img_path = CONFIG.workspace / "resources" / "SD_Output" / "output_0.png" assert os.path.exists(img_path) == True From 43dda1edafc25df7c99c76efa2b31486fd75e710 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Wed, 30 Aug 2023 11:55:54 +0800 Subject: [PATCH 139/398] fix options error --- metagpt/actions/project_management.py | 3 ++- .../tools/web_browser_engine_playwright.py | 20 ++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 55e7cbcb5..1062f8984 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -109,7 +109,8 @@ class WriteTasks(Action): async def _save(self, rsp): file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md" - file_path.write_text(rsp.content) + async with aiofiles.open(file_path, "w") as f: + await f.write(rsp.content) # Write requirements.txt requirements_path = CONFIG.workspace / "requirements.txt" diff --git a/metagpt/tools/web_browser_engine_playwright.py b/metagpt/tools/web_browser_engine_playwright.py index 199f8a0d1..8eecc4f40 100644 --- a/metagpt/tools/web_browser_engine_playwright.py +++ b/metagpt/tools/web_browser_engine_playwright.py @@ -8,11 +8,11 @@ from __future__ import annotations import asyncio import sys from pathlib import Path -from typing import Literal, Dict +from typing import Literal from playwright.async_api import async_playwright -from metagpt.config import Config +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.utils.parse_html import WebPage @@ -28,20 +28,18 @@ class PlaywrightWrapper: def __init__( self, - options: Dict, browser_type: Literal["chromium", "firefox", "webkit"] | None = None, launch_kwargs: dict | None = None, **kwargs, ) -> None: - self.options = options if browser_type is None: - browser_type = options.get("playwright_browser_type") + browser_type = CONFIG.playwright_browser_type self.browser_type = browser_type launch_kwargs = launch_kwargs or {} - if options.get("global_proxy") and "proxy" not in launch_kwargs: + if CONFIG.global_proxy and "proxy" not in launch_kwargs: args = launch_kwargs.get("args", []) if not any(str.startswith(i, "--proxy-server=") for i in args): - launch_kwargs["proxy"] = {"server": options.get("global_proxy")} + launch_kwargs["proxy"] = {"server": CONFIG.global_proxy} self.launch_kwargs = launch_kwargs context_kwargs = {} if "ignore_https_errors" in kwargs: @@ -81,8 +79,8 @@ class PlaywrightWrapper: executable_path = Path(browser_type.executable_path) if not executable_path.exists() and "executable_path" not in self.launch_kwargs: kwargs = {} - if self.options.get("global_proxy"): - kwargs["env"] = {"ALL_PROXY": self.options.get("global_proxy")} + if CONFIG.global_proxy: + kwargs["env"] = {"ALL_PROXY": CONFIG.global_proxy} await _install_browsers(self.browser_type, **kwargs) if self._has_run_precheck: @@ -150,8 +148,6 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, browser_type: str = "chromium", **kwargs): - return await PlaywrightWrapper(options=Config().runtime_options, - browser_type=browser_type, - **kwargs).run(url, *urls) + return await PlaywrightWrapper(browser_type=browser_type, **kwargs).run(url, *urls) fire.Fire(main) From bc9eb5ea933bdc0750d1dd56efa3a00d5b6a0b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 11:57:18 +0800 Subject: [PATCH 140/398] feat: +.agent-store-config.yaml.example --- .agent-store-config.yaml.example | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .agent-store-config.yaml.example diff --git a/.agent-store-config.yaml.example b/.agent-store-config.yaml.example new file mode 100644 index 000000000..037a44ed4 --- /dev/null +++ b/.agent-store-config.yaml.example @@ -0,0 +1,9 @@ +role: + name: Teacher # Referenced the `Teacher` in `metagpt/roles/teacher.py`. + module: metagpt.roles.teacher # Referenced `metagpt/roles/teacher.py`. + skills: # Refer to the skill `name` of the published skill in `.well-known/skills.yaml`. + - name: text_to_speech + description: Text-to-speech + - name: text_to_image + description: Create a drawing based on the text. + From b07b9919a07aa5426a2b077cb35b3763b5b8af22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 14:52:00 +0800 Subject: [PATCH 141/398] fixbug: os.environ --- metagpt/tools/azure_tts.py | 12 +++++++----- metagpt/tools/metagpt_text_to_image.py | 13 ++++++++++--- metagpt/tools/openai_text_to_embedding.py | 9 +++++---- metagpt/tools/openai_text_to_image.py | 12 +++++++++--- tests/conftest.py | 6 ++++++ 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index e9bb55bed..3100e2a3a 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -12,11 +12,12 @@ from uuid import uuid4 import base64 import sys +from metagpt.config import CONFIG, Config + sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger from aiofile import async_open from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer -import os class AzureTTS: @@ -27,8 +28,8 @@ class AzureTTS: :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. """ - self.subscription_key = subscription_key if subscription_key else os.environ.get('AZURE_TTS_SUBSCRIPTION_KEY') - self.region = region if region else os.environ.get('AZURE_TTS_REGION') + self.subscription_key = subscription_key if subscription_key else CONFIG.AZURE_TTS_SUBSCRIPTION_KEY + self.region = region if region else CONFIG.AZURE_TTS_REGION # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles async def synthesize_speech(self, lang, voice, text, output_file): @@ -87,9 +88,9 @@ async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscripti if not style: style = "affectionate" if not subscription_key: - subscription_key = os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") + subscription_key = CONFIG.AZURE_TTS_SUBSCRIPTION_KEY if not region: - region = os.environ.get("AZURE_TTS_REGION") + region = CONFIG.AZURE_TTS_REGION xml_value = AzureTTS.role_style_text(role=role, style=style, text=text) tts = AzureTTS(subscription_key=subscription_key, region=region) @@ -108,6 +109,7 @@ async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscripti if __name__ == "__main__": + Config() loop = asyncio.new_event_loop() v = loop.create_task(oas3_azsure_tts("测试，test")) loop.run_until_complete(v) diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index 43d22961b..c5a0b872f 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -6,6 +6,7 @@ @File : metagpt_text_to_image.py @Desc : MetaGPT Text-to-Image OAS3 api, which provides text-to-image functionality. """ +import asyncio import base64 import os import sys @@ -16,6 +17,8 @@ import aiohttp import requests from pydantic import BaseModel +from metagpt.config import CONFIG, Config + sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger @@ -25,7 +28,7 @@ class MetaGPTText2Image: """ :param model_url: Model reset api url """ - self.model_url = model_url if model_url else os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL') + self.model_url = model_url if model_url else CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL async def text_2_image(self, text, size_type="512x512"): """Text to image @@ -98,12 +101,16 @@ async def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url if not text: return "" if not model_url: - model_url = os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL_URL') + model_url = CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL_URL return await MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type) if __name__ == "__main__": - v = oas3_metagpt_text_to_image("Panda emoji") + Config() + loop = asyncio.new_event_loop() + task = loop.create_task(oas3_metagpt_text_to_image("Panda emoji")) + v = loop.run_until_complete(task) + print(v) data = base64.b64decode(v) with open("tmp.png", mode="wb") as writer: writer.write(data) diff --git a/metagpt/tools/openai_text_to_embedding.py b/metagpt/tools/openai_text_to_embedding.py index 73984aff6..86b58d71f 100644 --- a/metagpt/tools/openai_text_to_embedding.py +++ b/metagpt/tools/openai_text_to_embedding.py @@ -17,7 +17,7 @@ import requests from pydantic import BaseModel import sys -from metagpt.config import CONFIG +from metagpt.config import CONFIG, Config sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger @@ -48,7 +48,7 @@ class OpenAIText2Embedding: """ :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` """ - self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') + self.openai_api_key = openai_api_key if openai_api_key else CONFIG.OPENAI_API_KEY async def text_2_embedding(self, text, model="text-embedding-ada-002"): """Text to embedding @@ -89,7 +89,8 @@ async def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", op if __name__ == "__main__": + Config() loop = asyncio.new_event_loop() - v = loop.create_task(oas3_openai_text_to_embedding("Panda emoji")) - loop.run_until_complete(v) + task = loop.create_task(oas3_openai_text_to_embedding("Panda emoji")) + v = loop.run_until_complete(task) print(v) diff --git a/metagpt/tools/openai_text_to_image.py b/metagpt/tools/openai_text_to_image.py index 052a429ae..395fa8133 100644 --- a/metagpt/tools/openai_text_to_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -6,6 +6,7 @@ @File : openai_text_to_image.py @Desc : OpenAI Text-to-Image OAS3 api, which provides text-to-image functionality. """ +import asyncio import base64 import os import sys @@ -16,6 +17,8 @@ import aiohttp import requests from pydantic import BaseModel +from metagpt.config import CONFIG, Config + sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger @@ -25,7 +28,7 @@ class OpenAIText2Image: """ :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` """ - self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') + self.openai_api_key = openai_api_key if openai_api_key else CONFIG.OPENAI_API_KEY async def text_2_image(self, text, size_type="1024x1024"): """Text to image @@ -90,10 +93,13 @@ async def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_a if not text: return "" if not openai_api_key: - openai_api_key = os.environ.get("OPENAI_API_KEY") + openai_api_key = CONFIG.OPENAI_API_KEY return await OpenAIText2Image(openai_api_key).text_2_image(text, size_type=size_type) if __name__ == "__main__": - v = oas3_openai_text_to_image("Panda emoji") + Config() + loop = asyncio.new_event_loop() + task = loop.create_task(oas3_openai_text_to_image("Panda emoji")) + v = loop.run_until_complete(task) print(v) diff --git a/tests/conftest.py b/tests/conftest.py index feecc7715..8f5069bbe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,9 @@ from unittest.mock import Mock import pytest +import pytest_asyncio +from metagpt.config import Config from metagpt.logs import logger from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI import asyncio @@ -68,3 +70,7 @@ def proxy(): server = asyncio.get_event_loop().run_until_complete(asyncio.start_server(handle_client, "127.0.0.1", 0)) return "http://{}:{}".format(*server.sockets[0].getsockname()) + +@pytest.fixture(scope="session", autouse=True) +def init_config(): + Config() From d2d8bda61598438f9aaa100ce1850d1fbd488c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 14:56:50 +0800 Subject: [PATCH 142/398] feat: update azure-cognitiveservices-speech==1.31.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ed3f755c9..25a480a68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,5 +40,5 @@ libcst==1.0.1 qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 -azure-cognitiveservices-speech==1.30.0 +azure-cognitiveservices-speech==1.31.0 aiofile \ No newline at end of file From 8aff30a350df8eeb544807bea9b8ddd7b1cd7e39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 15:36:52 +0800 Subject: [PATCH 143/398] refactor: replace aiofile with aiofiles --- metagpt/tools/azure_tts.py | 3 ++- requirements.txt | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 3100e2a3a..0dc16d516 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -11,6 +11,7 @@ from pathlib import Path from uuid import uuid4 import base64 import sys +import aiofiles from metagpt.config import CONFIG, Config @@ -97,7 +98,7 @@ async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscripti filename = Path(__file__).resolve().parent / (str(uuid4()).replace("-", "") + ".wav") try: await tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename)) - async with async_open(filename, mode="rb") as reader: + async with aiofiles.open(filename, mode="rb") as reader: data = await reader.read() base64_string = base64.b64encode(data).decode('utf-8') filename.unlink() diff --git a/requirements.txt b/requirements.txt index 25a480a68..ca7fcbfda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,5 +40,4 @@ libcst==1.0.1 qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 -azure-cognitiveservices-speech==1.31.0 -aiofile \ No newline at end of file +azure-cognitiveservices-speech==1.31.0 \ No newline at end of file From a5ab5948c9f914edbb63408cc255a5ce4b229a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 17:30:12 +0800 Subject: [PATCH 144/398] fixbug: remove aiofile --- metagpt/tools/azure_tts.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 0dc16d516..6864faf10 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -7,18 +7,15 @@ @Desc : azure TTS OAS3 api, which provides text-to-speech functionality """ import asyncio +import base64 from pathlib import Path from uuid import uuid4 -import base64 -import sys + import aiofiles +from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer from metagpt.config import CONFIG, Config - -sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger -from aiofile import async_open -from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer class AzureTTS: @@ -34,18 +31,17 @@ class AzureTTS: # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles async def synthesize_speech(self, lang, voice, text, output_file): - speech_config = SpeechConfig( - subscription=self.subscription_key, region=self.region) + speech_config = SpeechConfig(subscription=self.subscription_key, region=self.region) speech_config.speech_synthesis_voice_name = voice audio_config = AudioConfig(filename=output_file) - synthesizer = SpeechSynthesizer( - speech_config=speech_config, - audio_config=audio_config) + synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=audio_config) # More detail: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-voice - ssml_string = "" \ - f"{text}" + ssml_string = ( + "" + f"{text}" + ) return synthesizer.speak_ssml_async(ssml_string).get() @@ -100,7 +96,7 @@ async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscripti await tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename)) async with aiofiles.open(filename, mode="rb") as reader: data = await reader.read() - base64_string = base64.b64encode(data).decode('utf-8') + base64_string = base64.b64encode(data).decode("utf-8") filename.unlink() except Exception as e: logger.error(f"text:{text}, error:{e}") From 9428c256caf1f16971216c7a3e4b66603bf8a825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 17:55:13 +0800 Subject: [PATCH 145/398] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 33 +++++++++++++++++++ .../metagpt/provider/test_metagpt_llm_api.py | 17 ++++++++++ 2 files changed, 50 insertions(+) create mode 100644 metagpt/provider/metagpt_llm_api.py create mode 100644 tests/metagpt/provider/test_metagpt_llm_api.py diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py new file mode 100644 index 000000000..bfd003fff --- /dev/null +++ b/metagpt/provider/metagpt_llm_api.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/30 +@Author : mashenquan +@File : metagpt_llm_api.py +@Desc : MetaGPT LLM related APIs +""" + +import openai + +from metagpt.config import CONFIG +from metagpt.provider import OpenAIGPTAPI +from metagpt.provider.openai_api import RateLimiter + + +class MetaGPTLLMAPI(OpenAIGPTAPI): + """MetaGPT LLM api""" + + def __init__(self): + self.__init_openai(CONFIG) + self.llm = openai + self.model = CONFIG.METAGPT_API_MODEL + self.auto_max_tokens = False + RateLimiter.__init__(self, rpm=self.rpm) + + def __init_openai(self, config): + openai.api_key = CONFIG.METAGPT_API_KEY + if config.openai_api_base: + openai.api_base = CONFIG.METAGPT_API_BASE + if config.openai_api_type: + openai.api_type = CONFIG.METAGPT_API_TYPE + openai.api_version = CONFIG.METAGPT_API_VERSION + self.rpm = int(config.get("RPM", 10)) diff --git a/tests/metagpt/provider/test_metagpt_llm_api.py b/tests/metagpt/provider/test_metagpt_llm_api.py new file mode 100644 index 000000000..9c8356ca6 --- /dev/null +++ b/tests/metagpt/provider/test_metagpt_llm_api.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/30 +@Author : mashenquan +@File : test_metagpt_llm_api.py +""" +from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI + + +def test_metagpt(): + llm = MetaGPTLLMAPI() + assert llm + + +if __name__ == "__main__": + test_metagpt() From 09fdb9d1ae1e5d0ab5f6a9c4571cff6bb265089f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:21:36 +0800 Subject: [PATCH 146/398] feat: +metagpt llm --- metagpt/const.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index 9e7462da6..e792ff35a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -14,9 +14,11 @@ def get_project_root(): """逐级向上寻找项目根目录""" current_path = Path.cwd() while True: - if (current_path / '.git').exists() or \ - (current_path / '.project_root').exists() or \ - (current_path / '.gitignore').exists(): + if ( + (current_path / ".git").exists() + or (current_path / ".project_root").exists() + or (current_path / ".gitignore").exists() + ): return current_path parent_path = current_path.parent if parent_path == current_path: @@ -25,15 +27,15 @@ def get_project_root(): PROJECT_ROOT = get_project_root() -DATA_PATH = PROJECT_ROOT / 'data' -WORKSPACE_ROOT = PROJECT_ROOT / 'workspace' -PROMPT_PATH = PROJECT_ROOT / 'metagpt/prompts' -UT_PATH = PROJECT_ROOT / 'data/ut' +DATA_PATH = PROJECT_ROOT / "data" +WORKSPACE_ROOT = PROJECT_ROOT / "workspace" +PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" +UT_PATH = PROJECT_ROOT / "data/ut" SWAGGER_PATH = UT_PATH / "files/api/" UT_PY_PATH = UT_PATH / "files/ut/" API_QUESTIONS_PATH = UT_PATH / "files/question/" YAPI_URL = "http://yapi.deepwisdomai.com/" -TMP = PROJECT_ROOT / 'tmp' +TMP = PROJECT_ROOT / "tmp" RESEARCH_PATH = DATA_PATH / "research" MEM_TTL = 24 * 30 * 3600 @@ -43,4 +45,12 @@ DEFAULT_LANGUAGE = "English" DEFAULT_MAX_TOKENS = 1500 COMMAND_TOKENS = 500 BRAIN_MEMORY = "BRAIN_MEMORY" -SKILL_PATH = "SKILL_PATH" \ No newline at end of file +SKILL_PATH = "SKILL_PATH" +SERPER_API_KEY = "SERPER_API_KEY" + +# MetaGPT LLM key defines +METAGPT_API_MODEL = "METAGPT_API_MODEL" +METAGPT_API_KEY = "METAGPT_API_KEY" +METAGPT_API_BASE = "METAGPT_API_BASE" +METAGPT_API_TYPE = "METAGPT_API_TYPE" +METAGPT_API_VERSION = "METAGPT_API_VERSION" From f65b959d5277053ddffebdc3fdc5e8a11af9c6b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:23:29 +0800 Subject: [PATCH 147/398] feat: +metagpt llm --- metagpt/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/const.py b/metagpt/const.py index e792ff35a..f2f1b4837 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -48,7 +48,7 @@ BRAIN_MEMORY = "BRAIN_MEMORY" SKILL_PATH = "SKILL_PATH" SERPER_API_KEY = "SERPER_API_KEY" -# MetaGPT LLM key defines +# Key Definitions for MetaGPT LLM METAGPT_API_MODEL = "METAGPT_API_MODEL" METAGPT_API_KEY = "METAGPT_API_KEY" METAGPT_API_BASE = "METAGPT_API_BASE" From 39e2e1d8a01be2696b3319f0b7c5794af7a650f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:25:10 +0800 Subject: [PATCH 148/398] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index bfd003fff..78a9e44b1 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -25,9 +25,9 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): def __init_openai(self, config): openai.api_key = CONFIG.METAGPT_API_KEY - if config.openai_api_base: + if CONFIG.METAGPT_API_BASE: openai.api_base = CONFIG.METAGPT_API_BASE - if config.openai_api_type: + if CONFIG.METAGPT_API_TYPE: openai.api_type = CONFIG.METAGPT_API_TYPE openai.api_version = CONFIG.METAGPT_API_VERSION self.rpm = int(config.get("RPM", 10)) From 4e92206301a43edfd6e777a1bff43e99acb884dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:26:52 +0800 Subject: [PATCH 149/398] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 78a9e44b1..bb8749e82 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -17,17 +17,17 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): - self.__init_openai(CONFIG) + self.__init_openai() self.llm = openai self.model = CONFIG.METAGPT_API_MODEL self.auto_max_tokens = False RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self, config): + def __init_openai(self): openai.api_key = CONFIG.METAGPT_API_KEY if CONFIG.METAGPT_API_BASE: openai.api_base = CONFIG.METAGPT_API_BASE if CONFIG.METAGPT_API_TYPE: openai.api_type = CONFIG.METAGPT_API_TYPE openai.api_version = CONFIG.METAGPT_API_VERSION - self.rpm = int(config.get("RPM", 10)) + self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 From 01bdc2c90bcb8056f854c0560b6df7fa1137f43d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:28:13 +0800 Subject: [PATCH 150/398] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index bb8749e82..c27e7132d 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -23,7 +23,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): self.auto_max_tokens = False RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self): + def __init_openai(self, *args, **kwargs): openai.api_key = CONFIG.METAGPT_API_KEY if CONFIG.METAGPT_API_BASE: openai.api_base = CONFIG.METAGPT_API_BASE From d304e008a0d2d43ef538e22b821fb09568366272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 14:36:23 +0800 Subject: [PATCH 151/398] feat: +log --- metagpt/provider/base_gpt_api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index f1590a77c..af0cf2ec0 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -15,7 +15,8 @@ from metagpt.provider.base_chatbot import BaseChatbot class BaseGPTAPI(BaseChatbot): """GPT API abstract class, requiring all inheritors to provide a series of standard capabilities""" - system_prompt = 'You are a helpful assistant.' + + system_prompt = "You are a helpful assistant." def _user_msg(self, msg: str) -> dict[str, str]: return {"role": "user", "content": msg} @@ -46,9 +47,9 @@ class BaseGPTAPI(BaseChatbot): rsp = await self.acompletion_text(message, stream=True) except Exception as e: logger.exception(f"{e}") + logger.info(f"ask:{msg}, error:{e}") raise e - logger.debug(message) - # logger.debug(rsp) + logger.info(f"ask:{msg}, anwser:{rsp}") return rsp def _extract_assistant_rsp(self, context): @@ -115,7 +116,7 @@ class BaseGPTAPI(BaseChatbot): def messages_to_prompt(self, messages: list[dict]): """[{"role": "user", "content": msg}] to user: etc.""" - return '\n'.join([f"{i['role']}: {i['content']}" for i in messages]) + return "\n".join([f"{i['role']}: {i['content']}" for i in messages]) def messages_to_dict(self, messages): """objects to [{"role": "user", "content": msg}] etc.""" From 8c2dfca68736eb74e749c609e480cd5d26ada18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 21:03:21 +0800 Subject: [PATCH 152/398] fixbug: dead loop --- metagpt/provider/openai_api.py | 64 +++++++++++++++++++---------- metagpt/tools/__init__.py | 5 +++ metagpt/tools/web_browser_engine.py | 28 ++++++------- 3 files changed, 61 insertions(+), 36 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 75ac38860..6d4e1b406 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -7,15 +7,21 @@ Change cost control from global to company level. """ import asyncio +import random import re import time -import random - -from typing import List import traceback +from typing import List + import openai from openai.error import APIConnectionError -from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type +from tenacity import ( + after_log, + retry, + retry_if_exception_type, + stop_after_attempt, + wait_fixed, +) from metagpt.config import CONFIG from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS @@ -40,7 +46,7 @@ class RateLimiter: self.rpm = rpm def split_batches(self, batch): - return [batch[i: i + self.rpm] for i in range(0, len(batch), self.rpm)] + return [batch[i : i + self.rpm] for i in range(0, len(batch), self.rpm)] async def wait_if_needed(self, num_requests): current_time = time.time() @@ -56,10 +62,12 @@ class RateLimiter: def log_and_reraise(retry_state): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") - logger.warning(""" + logger.warning( + """ Recommend going to https://deepwisdom.feishu.cn/wiki/MsGnwQBjiif9c3koSJNcYaoSnu4#part-XdatdVlhEojeAfxaaEZcMV3ZniQ See FAQ 5.8 -""") +""" + ) raise retry_state.outcome.exception() @@ -85,10 +93,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: - response = await self.async_retry_call(openai.ChatCompletion.acreate, - **self._cons_kwargs(messages), - stream=True - ) + response = await self.async_retry_call( + openai.ChatCompletion.acreate, **self._cons_kwargs(messages), stream=True + ) # create variables to collect the stream of chunks collected_chunks = [] collected_messages = [] @@ -151,7 +158,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): @retry( stop=stop_after_attempt(3), wait=wait_fixed(1), - after=after_log(logger, logger.level('WARNING').name), + after=after_log(logger, logger.level("WARNING").name), retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) @@ -168,8 +175,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): try: prompt_tokens = count_message_tokens(messages, self.model) completion_tokens = count_string_tokens(rsp, self.model) - usage['prompt_tokens'] = prompt_tokens - usage['completion_tokens'] = completion_tokens + usage["prompt_tokens"] = prompt_tokens + usage["completion_tokens"] = completion_tokens return usage except Exception as e: logger.error("usage calculation failed!", e) @@ -205,8 +212,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def _update_costs(self, usage: dict): if CONFIG.calc_usage: try: - prompt_tokens = int(usage['prompt_tokens']) - completion_tokens = int(usage['completion_tokens']) + prompt_tokens = int(usage["prompt_tokens"]) + completion_tokens = int(usage["completion_tokens"]) CONFIG.cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) except Exception as e: logger.error("updating costs failed!", e) @@ -260,7 +267,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return result == "TRUE" async def rewrite(self, sentence: str, context: str): - command = f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + command = ( + f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + ) rsp = await self.aask(msg=command, system_msgs=[]) return rsp @@ -281,6 +290,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): break w = text[idx:data_len] windows.append(w) + idx += data_len for i in range(len(windows)): if i + 1 == len(windows): break @@ -289,7 +299,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): @staticmethod def extract_info(input_string): - pattern = r'\[([A-Z]+)\]:\s*(.+)' + pattern = r"\[([A-Z]+)\]:\s*(.+)" match = re.match(pattern, input_string) if match: return match.group(1), match.group(2) @@ -323,10 +333,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): except openai.error.RateLimitError as e: logger.warning(f"Exception:{e}") continue - except (openai.error.AuthenticationError, - openai.error.PermissionError, - openai.error.InvalidAPIType, - openai.error.SignatureVerificationError) as e: + except ( + openai.error.AuthenticationError, + openai.error.PermissionError, + openai.error.InvalidAPIType, + openai.error.SignatureVerificationError, + ) as e: logger.warning(f"Exception:{e}") raise e except Exception as e: @@ -336,3 +348,11 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 + + +if __name__ == "__main__": + txt = """ +as dfas sad lkf sdkl sakdfsdk sjd jsk sdl sk dd sd asd fa sdf sad dd +- .gitlab-ci.yml & base_test.py + """ + OpenAIGPTAPI.split_texts(txt, 30) diff --git a/metagpt/tools/__init__.py b/metagpt/tools/__init__.py index d98087e4b..a148bb744 100644 --- a/metagpt/tools/__init__.py +++ b/metagpt/tools/__init__.py @@ -22,3 +22,8 @@ class WebBrowserEngineType(Enum): PLAYWRIGHT = "playwright" SELENIUM = "selenium" CUSTOM = "custom" + + @classmethod + def _missing_(cls, key): + """缺省类型转换""" + return cls.CUSTOM diff --git a/metagpt/tools/web_browser_engine.py b/metagpt/tools/web_browser_engine.py index da208dbc9..1f1a5ec67 100644 --- a/metagpt/tools/web_browser_engine.py +++ b/metagpt/tools/web_browser_engine.py @@ -6,29 +6,31 @@ from __future__ import annotations import importlib -from typing import Any, Callable, Coroutine, Literal, overload, Dict +from typing import Any, Callable, Coroutine, Dict, Literal, overload -from metagpt.config import Config +from metagpt.config import CONFIG from metagpt.tools import WebBrowserEngineType from metagpt.utils.parse_html import WebPage class WebBrowserEngine: def __init__( - self, - options: Dict, - engine: WebBrowserEngineType | None = None, - run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None, + self, + options: Dict, + engine: WebBrowserEngineType | None = None, + run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None, ): engine = engine or options.get("web_browser_engine") + if engine is None: + raise NotImplementedError - if engine == WebBrowserEngineType.PLAYWRIGHT: + if WebBrowserEngineType(engine) is WebBrowserEngineType.PLAYWRIGHT: module = "metagpt.tools.web_browser_engine_playwright" run_func = importlib.import_module(module).PlaywrightWrapper(options=options).run - elif engine == WebBrowserEngineType.SELENIUM: + elif WebBrowserEngineType(engine) is WebBrowserEngineType.SELENIUM: module = "metagpt.tools.web_browser_engine_selenium" run_func = importlib.import_module(module).SeleniumWrapper(options=options).run - elif engine == WebBrowserEngineType.CUSTOM: + elif WebBrowserEngineType(engine) is WebBrowserEngineType.CUSTOM: run_func = run_func else: raise NotImplementedError @@ -51,10 +53,8 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, engine_type: Literal["playwright", "selenium"] = "playwright", **kwargs): - conf = Config() - return await WebBrowserEngine(options=conf.runtime_options, - engine=WebBrowserEngineType(engine_type), - **kwargs).run(url, *urls) - + return await WebBrowserEngine(options=CONFIG.options, engine=WebBrowserEngineType(engine_type), **kwargs).run( + url, *urls + ) fire.Fire(main) From 795b892b3530d7dc97248593be72c7561dfabbf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:24:54 +0800 Subject: [PATCH 153/398] fixbug: dead loop --- metagpt/provider/openai_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 6d4e1b406..be262d606 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,6 +276,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): @staticmethod def split_texts(text: str, window_size) -> List[str]: """Splitting long text into sliding windows text""" + if window_size <= 0: + window_size = OpenAIGPTAPI.DEFAULT_TOKEN_SIZE total_len = len(text) if total_len <= window_size: return [text] @@ -348,6 +350,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 + DEFAULT_TOKEN_SIZE = 50 if __name__ == "__main__": From 67d08cb054cb863b1200a407b1d00bec42171c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:29:04 +0800 Subject: [PATCH 154/398] fixbug: dead loop --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index be262d606..dd5594b7d 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -290,7 +290,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if data_len + idx > total_len: windows.append(text[idx:]) break - w = text[idx:data_len] + w = text[idx : idx + data_len] windows.append(w) idx += data_len for i in range(len(windows)): From 614bdf9e742908be5e19a1fa938ec4fe135b2ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:43:58 +0800 Subject: [PATCH 155/398] fixbug: dead loop --- metagpt/provider/openai_api.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index dd5594b7d..64fbbdfd6 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -286,13 +286,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): windows = [] idx = 0 while idx < total_len: - data_len = window_size - padding_size - if data_len + idx > total_len: + if window_size + idx > total_len: # 不足一个滑窗 windows.append(text[idx:]) break - w = text[idx : idx + data_len] + # 第一个窗口少算自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] + # window_size=3, padding_size=1： + # [1, 2, 3], [3, 4, 5], [5, 6, 7], .... + # idx=2, | idx=5 | idx=8 | ... + w = text[idx : idx + window_size] windows.append(w) - idx += data_len + idx += window_size - padding_size if idx == 0 else window_size + for i in range(len(windows)): if i + 1 == len(windows): break From 0156fa592248d613ca2d4110fe563d0275eedd28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:48:50 +0800 Subject: [PATCH 156/398] fixbug: dead loop --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 64fbbdfd6..019ad0b8b 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -285,6 +285,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if window_size > 20 else 0 windows = [] idx = 0 + data_len = window_size - padding_size while idx < total_len: if window_size + idx > total_len: # 不足一个滑窗 windows.append(text[idx:]) @@ -295,7 +296,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # idx=2, | idx=5 | idx=8 | ... w = text[idx : idx + window_size] windows.append(w) - idx += window_size - padding_size if idx == 0 else window_size + idx += data_len for i in range(len(windows)): if i + 1 == len(windows): From ea35305b52040c3da7e9efbe1b1c104f3f7c0603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:58:31 +0800 Subject: [PATCH 157/398] fixbug: dead loop --- metagpt/provider/openai_api.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 019ad0b8b..7ed9c0083 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -290,7 +290,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if window_size + idx > total_len: # 不足一个滑窗 windows.append(text[idx:]) break - # 第一个窗口少算自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] + # 每个窗口少算padding_size自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] # window_size=3, padding_size=1： # [1, 2, 3], [3, 4, 5], [5, 6, 7], .... # idx=2, | idx=5 | idx=8 | ... @@ -298,10 +298,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): windows.append(w) idx += data_len - for i in range(len(windows)): - if i + 1 == len(windows): - break - windows[i] += windows[i + 1][0:padding_size] return windows @staticmethod From 91595daa3b49f1a7bd0ed49e4bea80568455ba00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 23:14:07 +0800 Subject: [PATCH 158/398] fixbug: dead loop --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 7ed9c0083..14347f20c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -351,7 +351,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 - DEFAULT_TOKEN_SIZE = 50 + DEFAULT_TOKEN_SIZE = 500 if __name__ == "__main__": From 60d984f18478eeada59df09bde99e6bfae5fbe30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 10:25:31 +0800 Subject: [PATCH 159/398] fixbug: MET-1113 --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 555b202d1..e81f14bdd 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -14,7 +14,7 @@ from metagpt.logs import logger class TalkAction(Action): - def __init__(self, name: str = '', talk='', history_summary='', knowledge='', context=None, llm=None, **kwargs): + def __init__(self, name: str = "", talk="", history_summary="", knowledge="", context=None, llm=None, **kwargs): context = context or {} context["talk"] = talk context["history_summery"] = history_summary @@ -32,7 +32,7 @@ class TalkAction(Action): if self._history_summary != "": prompt += "According to the historical conversation above, " language = CONFIG.language or DEFAULT_LANGUAGE - prompt += f"Answer in {language}:\n {self._talk}" + prompt += f"Answer in {language}, and the answers must follow the Markdown format.\n {self._talk}" return prompt async def run(self, *args, **kwargs) -> ActionOutput: From 58dd5b8787a2df1523f4678815f48fc2e45ace55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 20:52:40 +0800 Subject: [PATCH 160/398] fixbug: exceed length --- metagpt/provider/openai_api.py | 18 +++++++++++------- metagpt/roles/assistant.py | 32 +++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 14347f20c..ac8feb738 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -242,14 +242,18 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """Generate text title""" max_response_token_count = 50 max_token_count = max_token_count_per_ask or CONFIG.MAX_TOKENS or DEFAULT_MAX_TOKENS - text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) + while True: + text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) - summaries = [] - for ws in text_windows: - response = await self.get_summary(ws) - summaries.append(response) - if len(summaries) == 1: - return summaries[0] + summaries = [] + for ws in text_windows: + response = await self.get_summary(ws) + summaries.append(response) + if len(summaries) == 1: + return summaries[0] + text = "\n".join(summaries) + if len(text) <= max_words * 2 and len(text) <= max_token_count: + break language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 57cb28e67..c681da65b 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -18,7 +18,7 @@ import asyncio from pathlib import Path from metagpt.actions import ActionOutput -from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction +from metagpt.actions.skill_action import ArgumentsParingAction, SkillAction from metagpt.actions.talk_action import TalkAction from metagpt.config import CONFIG from metagpt.learn.skill_loader import SkillLoader @@ -31,10 +31,19 @@ from metagpt.schema import Message class Assistant(Role): """Assistant for solving common issues.""" - def __init__(self, name="Lily", profile="An assistant", goal="Help to solve problem", - constraints="Talk in {language}", desc="", *args, **kwargs): - super(Assistant, self).__init__(name=name, profile=profile, - goal=goal, constraints=constraints, desc=desc, *args, **kwargs) + def __init__( + self, + name="Lily", + profile="An assistant", + goal="Help to solve problem", + constraints="Talk in {language}", + desc="", + *args, + **kwargs, + ): + super(Assistant, self).__init__( + name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs + ) brain_memory = CONFIG.BRAIN_MEMORY self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory() skill_path = Path(CONFIG.SKILL_PATH) if CONFIG.SKILL_PATH else None @@ -65,8 +74,9 @@ class Assistant(Role): msg = Message(content=result) output = ActionOutput(content=result) else: - msg = Message(content=result.content, instruct_content=result.instruct_content, - cause_by=type(self._rc.todo)) + msg = Message( + content=result.content, instruct_content=result.instruct_content, cause_by=type(self._rc.todo) + ) output = result self.memory.add_answer(msg) return output @@ -85,8 +95,7 @@ class Assistant(Role): return await handler(text, **kwargs) async def talk_handler(self, text, **kwargs) -> bool: - action = TalkAction(talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, - **kwargs) + action = TalkAction(talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, **kwargs) self.add_to_do(action) return True @@ -111,7 +120,7 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_context_title(history_text, max_words=20) + history_summary = await self._llm.get_context_title(history_text, max_token_count_per_ask=1000, max_words=500) if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk @@ -122,6 +131,7 @@ class Assistant(Role): @staticmethod def extract_info(input_string): from metagpt.provider.openai_api import OpenAIGPTAPI + return OpenAIGPTAPI.extract_info(input_string) def get_memory(self) -> str: @@ -150,6 +160,6 @@ async def main(): await role.talk(talk) -if __name__ == '__main__': +if __name__ == "__main__": CONFIG.language = "Chinese" asyncio.run(main()) From ae414fccfadaf2d76faaf73f322c687e527c1b7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:05:18 +0800 Subject: [PATCH 161/398] fixbug: exceed length --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index ac8feb738..c08a34f7e 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -247,7 +247,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): summaries = [] for ws in text_windows: - response = await self.get_summary(ws) + response = await self.get_summary(ws, max_words=max_response_token_count) summaries.append(response) if len(summaries) == 1: return summaries[0] From 3454761f950d49db49588eb35518708cf9d5b0e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:16:56 +0800 Subject: [PATCH 162/398] fixbug: exceed length --- metagpt/memory/brain_memory.py | 5 +++-- metagpt/roles/assistant.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index b3445a1f2..23b50afb3 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -8,7 +8,7 @@ """ from enum import Enum -from typing import List, Dict +from typing import Dict, List import pydantic @@ -48,7 +48,7 @@ class BrainMemory(pydantic.BaseModel): texts = [Message(**m).content for m in self.history[:-1]] return "\n".join(texts) - def move_to_solution(self): + def move_to_solution(self, history_summary): if len(self.history) < 2: return msgs = self.history[:-1] @@ -58,6 +58,7 @@ class BrainMemory(pydantic.BaseModel): self.history = [] else: self.history = self.history[-1:] + self.history.insert(0, Message(content=history_summary)) @property def last_talk(self): diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index c681da65b..719dfc29b 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -125,7 +125,7 @@ class Assistant(Role): last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk - self.memory.move_to_solution() # Promptly clear memory after the issue is resolved. + self.memory.move_to_solution(history_summary) # Promptly clear memory after the issue is resolved. return last_talk @staticmethod From 7babb5ef711a2983fc9a726c77575fdf9d71014b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:18:11 +0800 Subject: [PATCH 163/398] fixbug: exceed length --- metagpt/memory/brain_memory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 23b50afb3..9bafaafbb 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -49,6 +49,7 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) def move_to_solution(self, history_summary): + """放入solution队列，以备后续长程检索。目前还未加此功能""" if len(self.history) < 2: return msgs = self.history[:-1] From f2aaafbe001d094bdcbe059cad8a9378209f36ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:19:28 +0800 Subject: [PATCH 164/398] fixbug: exceed length --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 9bafaafbb..6bca9b140 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -59,7 +59,7 @@ class BrainMemory(pydantic.BaseModel): self.history = [] else: self.history = self.history[-1:] - self.history.insert(0, Message(content=history_summary)) + self.history.insert(0, Message(content="RESOLVED: " + history_summary)) @property def last_talk(self): From 8c943dd8e98f6e1dc60e6a534667900b2aa154bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:19:57 +0800 Subject: [PATCH 165/398] fixbug: exceed length --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 6bca9b140..c6be2cb7e 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -49,7 +49,7 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) def move_to_solution(self, history_summary): - """放入solution队列，以备后续长程检索。目前还未加此功能""" + """放入solution队列，以备后续长程检索。目前还未加此功能，先用history_summary顶替""" if len(self.history) < 2: return msgs = self.history[:-1] From 478139c8dc2286d8e3db722145626a408cba4159 Mon Sep 17 00:00:00 2001 From: Stitch-z <284618289@qq.com> Date: Fri, 1 Sep 2023 21:21:47 +0800 Subject: [PATCH 166/398] feature: aioboto3 client --- config/config.yaml | 8 ++- metagpt/utils/s3.py | 127 +++++++++++++++++++++++++++++++++ requirements.txt | 4 +- tests/conftest.py | 7 +- tests/metagpt/utils/test_s3.py | 55 ++++++++++++++ 5 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 metagpt/utils/s3.py create mode 100644 tests/metagpt/utils/test_s3.py diff --git a/config/config.yaml b/config/config.yaml index 88cca08e5..7c3d212f6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -77,4 +77,10 @@ MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### Meta Models -#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL \ No newline at end of file +#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL + +### S3 config +S3: + access_key: "YOUR_S3_ACCESS_KEY" + secret_key: "YOUR_S3_SECRET_KEY" + endpoint_url: "YOUR_S3_ENDPOINT_URL" \ No newline at end of file diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py new file mode 100644 index 000000000..2b4b8cb5f --- /dev/null +++ b/metagpt/utils/s3.py @@ -0,0 +1,127 @@ + +from typing import Optional + +import aioboto3 +from metagpt.logs import logger +from metagpt.config import Config + + +class S3: + """A class for interacting with Amazon S3 storage.""" + + def __init__(self): + self.session = aioboto3.Session() + self.s3_config = Config().get("S3") + self.auth_config = { + "service_name": "s3", + "aws_access_key_id": self.s3_config["access_key"], + "aws_secret_access_key": self.s3_config["secret_key"], + "endpoint_url": self.s3_config["endpoint_url"] + } + + async def upload_file( + self, + bucket: str, + local_path: str, + object_name: str, + ) -> None: + """Upload a file from the local path to the specified path of the storage bucket specified in s3. + + Args: + bucket: The name of the S3 storage bucket. + local_path: The local file path, including the file name. + object_name: The complete path of the uploaded file to be stored in S3, including the file name. + + Raises: + Exception: If an error occurs during the upload process, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + with open(local_path, "rb") as file: + await client.put_object(Body=file, Bucket=bucket, Key=object_name) + logger.info(f"Successfully uploaded the file to path {object_name} in bucket {bucket} of s3.") + except Exception as e: + logger.error(f"Failed to upload the file to path {object_name} in bucket {bucket} of s3: {e}") + raise e + + async def get_object_url( + self, + bucket: str, + object_name: str, + ) -> str: + """Get the URL for a downloadable or preview file stored in the specified S3 bucket. + + Args: + bucket: The name of the S3 storage bucket. + object_name: The complete path of the file stored in S3, including the file name. + + Returns: + The URL for the downloadable or preview file. + + Raises: + Exception: If an error occurs while retrieving the URL, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + file = await client.get_object(Bucket=bucket, Key=object_name) + return str(file["Body"].url) + except Exception as e: + logger.error(f"Failed to get the url for a downloadable or preview file: {e}") + raise e + + async def get_object( + self, + bucket: str, + object_name: str, + ) -> bytes: + """Get the binary data of a file stored in the specified S3 bucket. + + Args: + bucket: The name of the S3 storage bucket. + object_name: The complete path of the file stored in S3, including the file name. + + Returns: + The binary data of the requested file. + + Raises: + Exception: If an error occurs while retrieving the file data, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + s3_object = await client.get_object(Bucket=bucket, Key=object_name) + return await s3_object["Body"].read() + except Exception as e: + logger.error(f"Failed to get the binary data of the file: {e}") + raise e + + async def download_file( + self, + bucket: str, + object_name: str, + local_path: str, + chunk_size: Optional[int] = 128 * 1024 + ) -> None: + """Download an S3 object to a local file. + + Args: + bucket: The name of the S3 storage bucket. + object_name: The complete path of the file stored in S3, including the file name. + local_path: The local file path where the S3 object will be downloaded. + chunk_size: The size of data chunks to read and write at a time. Default is 128 KB. + + Raises: + Exception: If an error occurs during the download process, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + s3_object = await client.get_object(Bucket=bucket, Key=object_name) + stream = s3_object["Body"] + with open(local_path, 'wb') as local_file: + while True: + file_data = await stream.read(chunk_size) + if not file_data: + break + local_file.write(file_data) + except Exception as e: + logger.error(f"Failed to download the file from S3: {e}") + raise e \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ca7fcbfda..2e5112aba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,4 +40,6 @@ libcst==1.0.1 qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 -azure-cognitiveservices-speech==1.31.0 \ No newline at end of file +azure-cognitiveservices-speech==1.31.0 +aioboto3~=11.3.0 +pytest-asyncio~=0.21.1 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 8f5069bbe..0bc17bd6a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,6 @@ from unittest.mock import Mock import pytest -import pytest_asyncio from metagpt.config import Config from metagpt.logs import logger @@ -17,6 +16,8 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI import asyncio import re +from metagpt.utils.s3 import S3 + class Context: def __init__(self): @@ -74,3 +75,7 @@ def proxy(): @pytest.fixture(scope="session", autouse=True) def init_config(): Config() + +@pytest.fixture(scope="session", autouse=True) +def s3(): + return S3() diff --git a/tests/metagpt/utils/test_s3.py b/tests/metagpt/utils/test_s3.py new file mode 100644 index 000000000..760a976b0 --- /dev/null +++ b/tests/metagpt/utils/test_s3.py @@ -0,0 +1,55 @@ +import os +import pytest + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "local_path", "object_name"], + [ + ( + "agent-store", + "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", + "ui-designer/2023-09-01/1.png" + ) + ] +) +async def test_upload_file(s3, bucket, local_path, object_name): + await s3.upload_file(bucket=bucket, local_path=local_path, object_name=object_name) + s3_object = await s3.get_object(bucket=bucket, object_name=object_name) + assert s3_object + assert isinstance(s3_object, bytes) + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "object_name"], + [("agent-store", "ui-designer/2023-09-01/1.png")] +) +async def test_get_object_url(s3, bucket, object_name): + url = await s3.get_object_url(bucket=bucket, object_name=object_name) + assert bucket in url + assert object_name in url + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "object_name"], + [("agent-store", "ui-designer/2023-09-01/1.png")] +) +async def test_get_object(s3, bucket, object_name): + s3_object = await s3.get_object(bucket=bucket, object_name=object_name) + assert s3_object + assert isinstance(s3_object, bytes) + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "local_path", "object_name"], + [ + ( + "agent-store", + "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", + "ui-designer/2023-09-01/1.png" + ) + ] +) +async def test_download_file(s3, bucket, local_path, object_name): + await s3.download_file(bucket=bucket, object_name=object_name, local_path=local_path) + assert os.path.exists(local_path) \ No newline at end of file From f7ebd2a3744b132fc606b3c4897eeb527dbb8aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:30:38 +0800 Subject: [PATCH 167/398] fixbug: exceed length --- metagpt/memory/brain_memory.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index c6be2cb7e..a5a3dbfc7 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -45,7 +45,16 @@ class BrainMemory(pydantic.BaseModel): def history_text(self): if len(self.history) == 0: return "" - texts = [Message(**m).content for m in self.history[:-1]] + texts = [] + for m in self.history[:-1]: + if isinstance(m, Dict): + t = Message(**m).content + elif isinstance(m, Message): + t = m.content + else: + continue + texts.append(t) + return "\n".join(texts) def move_to_solution(self, history_summary): From 760f7c5d5fce94638c70248053dc78b20afe47c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:32:27 +0800 Subject: [PATCH 168/398] fixbug: exceed length --- metagpt/roles/assistant.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 719dfc29b..fdd697b59 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -95,7 +95,10 @@ class Assistant(Role): return await handler(text, **kwargs) async def talk_handler(self, text, **kwargs) -> bool: - action = TalkAction(talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, **kwargs) + history = self.memory.history_text + action = TalkAction( + talk=text, knowledge=self.memory.get_knowledge(), history_summary=history, llm=self._llm, **kwargs + ) self.add_to_do(action) return True From 3e28b93e542f7223756cd127449b38001574a16a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 22:46:04 +0800 Subject: [PATCH 169/398] refactor: refine prompt --- metagpt/actions/talk_action.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index e81f14bdd..ac395e9dd 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -32,7 +32,10 @@ class TalkAction(Action): if self._history_summary != "": prompt += "According to the historical conversation above, " language = CONFIG.language or DEFAULT_LANGUAGE - prompt += f"Answer in {language}, and the answers must follow the Markdown format.\n {self._talk}" + prompt += ( + f"Answer the following questions in {language}, and the answers must follow the Markdown format.\n " + f"{self._talk}" + ) return prompt async def run(self, *args, **kwargs) -> ActionOutput: From bfd8ed69e8676e204e60d94d25e52605d528f8b5 Mon Sep 17 00:00:00 2001 From: Stitch-z <284618289@qq.com> Date: Sat, 2 Sep 2023 10:55:38 +0800 Subject: [PATCH 170/398] update: delete pytest code --- requirements.txt | 3 +- tests/conftest.py | 5 ---- tests/metagpt/utils/test_s3.py | 55 ---------------------------------- 3 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 tests/metagpt/utils/test_s3.py diff --git a/requirements.txt b/requirements.txt index 2e5112aba..5daf710c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,5 +41,4 @@ qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 azure-cognitiveservices-speech==1.31.0 -aioboto3~=11.3.0 -pytest-asyncio~=0.21.1 \ No newline at end of file +aioboto3~=11.3.0 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 0bc17bd6a..98b45de7b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,8 +16,6 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI import asyncio import re -from metagpt.utils.s3 import S3 - class Context: def __init__(self): @@ -76,6 +74,3 @@ def proxy(): def init_config(): Config() -@pytest.fixture(scope="session", autouse=True) -def s3(): - return S3() diff --git a/tests/metagpt/utils/test_s3.py b/tests/metagpt/utils/test_s3.py deleted file mode 100644 index 760a976b0..000000000 --- a/tests/metagpt/utils/test_s3.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import pytest - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "local_path", "object_name"], - [ - ( - "agent-store", - "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", - "ui-designer/2023-09-01/1.png" - ) - ] -) -async def test_upload_file(s3, bucket, local_path, object_name): - await s3.upload_file(bucket=bucket, local_path=local_path, object_name=object_name) - s3_object = await s3.get_object(bucket=bucket, object_name=object_name) - assert s3_object - assert isinstance(s3_object, bytes) - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "object_name"], - [("agent-store", "ui-designer/2023-09-01/1.png")] -) -async def test_get_object_url(s3, bucket, object_name): - url = await s3.get_object_url(bucket=bucket, object_name=object_name) - assert bucket in url - assert object_name in url - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "object_name"], - [("agent-store", "ui-designer/2023-09-01/1.png")] -) -async def test_get_object(s3, bucket, object_name): - s3_object = await s3.get_object(bucket=bucket, object_name=object_name) - assert s3_object - assert isinstance(s3_object, bytes) - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "local_path", "object_name"], - [ - ( - "agent-store", - "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", - "ui-designer/2023-09-01/1.png" - ) - ] -) -async def test_download_file(s3, bucket, local_path, object_name): - await s3.download_file(bucket=bucket, object_name=object_name, local_path=local_path) - assert os.path.exists(local_path) \ No newline at end of file From ca60cd0557effda735c4850b0f3b36fadd555fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 14:30:45 +0800 Subject: [PATCH 171/398] feat: +s3 --- metagpt/const.py | 3 ++ metagpt/learn/text_to_image.py | 22 +++++++++------ metagpt/learn/text_to_speech.py | 29 +++++++++++++------ metagpt/tools/openai_text_to_image.py | 38 +++++++++---------------- metagpt/utils/s3.py | 40 +++++++++++++++++++-------- 5 files changed, 79 insertions(+), 53 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index f2f1b4837..fbc2c928a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -54,3 +54,6 @@ METAGPT_API_KEY = "METAGPT_API_KEY" METAGPT_API_BASE = "METAGPT_API_BASE" METAGPT_API_TYPE = "METAGPT_API_TYPE" METAGPT_API_VERSION = "METAGPT_API_VERSION" + +# format +BASE64_FORMAT = "base64" diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 620e58180..c5f554ef3 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -6,10 +6,13 @@ @File : text_to_image.py @Desc : Text-to-Image skill, which provides text-to-image functionality. """ +import openai.error from metagpt.config import CONFIG +from metagpt.const import BASE64_FORMAT from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image from metagpt.tools.openai_text_to_image import oas3_openai_text_to_image +from metagpt.utils.s3 import S3 async def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs): @@ -23,13 +26,14 @@ async def text_to_image(text, size_type: str = "512x512", openai_api_key="", mod """ image_declaration = "data:image/png;base64," if CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL_URL or model_url: - data = await oas3_metagpt_text_to_image(text, size_type, model_url) - return image_declaration + data if data else "" - - if CONFIG.OPENAI_API_KEY or openai_api_key: - data = await oas3_openai_text_to_image(text, size_type, openai_api_key) - return image_declaration + data if data else "" - - raise EnvironmentError - + base64_data = await oas3_metagpt_text_to_image(text, size_type, model_url) + elif CONFIG.OPENAI_API_KEY or openai_api_key: + base64_data = await oas3_openai_text_to_image(text, size_type, openai_api_key) + else: + raise openai.error.InvalidRequestError("缺少必要的参数") + s3 = S3() + url = await s3.cache(base64_data, BASE64_FORMAT) + if url: + return url + return image_declaration + base64_data if base64_data else "" diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 66fbba5be..7883ae9f3 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -6,14 +6,24 @@ @File : text_to_speech.py @Desc : Text-to-Speech skill, which provides text-to-speech functionality """ +import openai from metagpt.config import CONFIG - +from metagpt.const import BASE64_FORMAT from metagpt.tools.azure_tts import oas3_azsure_tts +from metagpt.utils.s3 import S3 -async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", - subscription_key="", region="", **kwargs): +async def text_to_speech( + text, + lang="zh-CN", + voice="zh-CN-XiaomoNeural", + style="affectionate", + role="Girl", + subscription_key="", + region="", + **kwargs +): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -28,9 +38,12 @@ async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style=" """ audio_declaration = "data:audio/wav;base64," - if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or \ - (subscription_key and region): - data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) - return audio_declaration + data if data else data + if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region): + base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + s3 = S3() + url = await s3.cache(base64_data, BASE64_FORMAT) + if url: + return url + return audio_declaration + base64_data if base64_data else base64_data - raise EnvironmentError + raise openai.error.InvalidRequestError("缺少必要的参数") diff --git a/metagpt/tools/openai_text_to_image.py b/metagpt/tools/openai_text_to_image.py index 395fa8133..6025f04ba 100644 --- a/metagpt/tools/openai_text_to_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -8,18 +8,12 @@ """ import asyncio import base64 -import os -import sys -from pathlib import Path -from typing import List import aiohttp +import openai import requests -from pydantic import BaseModel from metagpt.config import CONFIG, Config - -sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger @@ -37,27 +31,21 @@ class OpenAIText2Image: :param size_type: One of ['256x256', '512x512', '1024x1024'] :return: The image data is returned in Base64 encoding. """ - - class ImageUrl(BaseModel): - url: str - - class ImageResult(BaseModel): - data: List[ImageUrl] - created: int - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.openai_api_key}" - } - data = {"prompt": text, "n": 1, "size": size_type} try: - async with aiohttp.ClientSession() as session: - async with session.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) as response: - result = ImageResult(** await response.json()) - except requests.exceptions.RequestException as e: + result = await openai.Image.acreate( + api_key=CONFIG.OPENAI_API_KEY, + api_base=CONFIG.OPENAI_API_BASE, + api_type=None, + api_version=None, + organization=None, + prompt=text, + n=1, + size=size_type, + ) + except Exception as e: logger.error(f"An error occurred:{e}") return "" - if len(result.data) > 0: + if result and len(result.data) > 0: return await OpenAIText2Image.get_image_data(result.data[0].url) return "" diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 2b4b8cb5f..85837fedb 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -1,9 +1,14 @@ - +import base64 +import traceback +import uuid from typing import Optional import aioboto3 +import aiofiles + +from metagpt.config import CONFIG +from metagpt.const import BASE64_FORMAT, WORKSPACE_ROOT from metagpt.logs import logger -from metagpt.config import Config class S3: @@ -11,12 +16,12 @@ class S3: def __init__(self): self.session = aioboto3.Session() - self.s3_config = Config().get("S3") + self.s3_config = CONFIG.S3 self.auth_config = { "service_name": "s3", "aws_access_key_id": self.s3_config["access_key"], "aws_secret_access_key": self.s3_config["secret_key"], - "endpoint_url": self.s3_config["endpoint_url"] + "endpoint_url": self.s3_config["endpoint_url"], } async def upload_file( @@ -95,11 +100,7 @@ class S3: raise e async def download_file( - self, - bucket: str, - object_name: str, - local_path: str, - chunk_size: Optional[int] = 128 * 1024 + self, bucket: str, object_name: str, local_path: str, chunk_size: Optional[int] = 128 * 1024 ) -> None: """Download an S3 object to a local file. @@ -116,7 +117,7 @@ class S3: async with self.session.client(**self.auth_config) as client: s3_object = await client.get_object(Bucket=bucket, Key=object_name) stream = s3_object["Body"] - with open(local_path, 'wb') as local_file: + with open(local_path, "wb") as local_file: while True: file_data = await stream.read(chunk_size) if not file_data: @@ -124,4 +125,21 @@ class S3: local_file.write(file_data) except Exception as e: logger.error(f"Failed to download the file from S3: {e}") - raise e \ No newline at end of file + raise e + + async def cache(self, data: str, format: str = "") -> str: + """Save data to remote S3 and return url""" + object_name = str(uuid.uuid4()).replace("-", "") + pathname = WORKSPACE_ROOT / "s3_tmp" / object_name + try: + async with aiofiles.open(pathname, mode="w") as file: + if format == BASE64_FORMAT: + data = base64.b64decode(data) + await file.write(data) + + bucket = CONFIG.S3.get("bucket") + await self.upload_file(bucket=bucket, local_path=pathname, object_name=object_name) + return await self.get_object_url(bucket=bucket, object_name=object_name) + except Exception as e: + logger.exception(f"{e}, stack:{traceback.format_exc()}") + return None From 578961ce2e07376e10c10191c80c9fc3714a22c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 15:24:48 +0800 Subject: [PATCH 172/398] feat: +role --- metagpt/roles/assistant.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index fdd697b59..48aff319b 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -54,7 +54,14 @@ class Assistant(Role): last_talk = await self.refine_memory() if not last_talk: return False - prompt = f"Refer to this sentence:\n {last_talk}\n" + prompt = "" + if CONFIG.agent_description: + prompt = ( + f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, " + f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " + f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" + ) + prompt += f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): prompt += f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: text_to_image\n" From a7b933311ebcaa18630947cebfbc96bda508231c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 15:39:17 +0800 Subject: [PATCH 173/398] feat: +role --- metagpt/actions/talk_action.py | 9 ++++++++- metagpt/roles/assistant.py | 9 +-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index ac395e9dd..4eed0d4f8 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -27,7 +27,14 @@ class TalkAction(Action): @property def prompt(self): - prompt = f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" + prompt = "" + if CONFIG.agent_description: + prompt = ( + f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, " + f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " + f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" + ) + prompt += f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" prompt += f"{self._history_summary}\n\n" if self._history_summary != "": prompt += "According to the historical conversation above, " diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 48aff319b..fdd697b59 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -54,14 +54,7 @@ class Assistant(Role): last_talk = await self.refine_memory() if not last_talk: return False - prompt = "" - if CONFIG.agent_description: - prompt = ( - f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, " - f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " - f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" - ) - prompt += f"Refer to this sentence:\n {last_talk}\n" + prompt = f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): prompt += f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: text_to_image\n" From 07a1d229cf08f89595c10f7d198ca9aa6b0e550d Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sat, 2 Sep 2023 18:03:31 +0800 Subject: [PATCH 174/398] restoresearch engine code --- metagpt/tools/search_engine.py | 33 ++++++++-------- metagpt/tools/search_engine_ddg.py | 48 +++++++++++------------ metagpt/tools/search_engine_googleapi.py | 13 +++--- metagpt/tools/search_engine_serpapi.py | 6 +-- metagpt/tools/search_engine_serper.py | 4 +- tests/metagpt/tools/test_search_engine.py | 19 +++++---- 6 files changed, 62 insertions(+), 61 deletions(-) diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 5b8b7f046..db8c091d1 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -4,12 +4,11 @@ @Time : 2023/5/6 20:15 @Author : alexanderwu @File : search_engine.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from __future__ import annotations import importlib -from typing import Callable, Coroutine, Literal, overload, Dict +from typing import Callable, Coroutine, Literal, overload from metagpt.config import CONFIG from metagpt.tools import SearchEngineType @@ -28,23 +27,23 @@ class SearchEngine: """ def __init__( - self, - engine: SearchEngineType | None = None, - run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None + self, + engine: SearchEngineType | None = None, + run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None, ): engine = engine or CONFIG.search_engine if engine == SearchEngineType.SERPAPI_GOOGLE: module = "metagpt.tools.search_engine_serpapi" - run_func = importlib.import_module(module).SerpAPIWrapper(**CONFIG.options).run + run_func = importlib.import_module(module).SerpAPIWrapper().run elif engine == SearchEngineType.SERPER_GOOGLE: module = "metagpt.tools.search_engine_serper" - run_func = importlib.import_module(module).SerperWrapper(**CONFIG.options).run + run_func = importlib.import_module(module).SerperWrapper().run elif engine == SearchEngineType.DIRECT_GOOGLE: module = "metagpt.tools.search_engine_googleapi" - run_func = importlib.import_module(module).GoogleAPIWrapper(**CONFIG.options).run + run_func = importlib.import_module(module).GoogleAPIWrapper().run elif engine == SearchEngineType.DUCK_DUCK_GO: module = "metagpt.tools.search_engine_ddg" - run_func = importlib.import_module(module).DDGAPIWrapper(**CONFIG.options).run + run_func = importlib.import_module(module).DDGAPIWrapper().run elif engine == SearchEngineType.CUSTOM_ENGINE: pass # run_func = run_func else: @@ -54,19 +53,19 @@ class SearchEngine: @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[True] = True, + self, + query: str, + max_results: int = 8, + as_string: Literal[True] = True, ) -> str: ... @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[False] = False, + self, + query: str, + max_results: int = 8, + as_string: Literal[False] = False, ) -> list[dict[str, str]]: ... diff --git a/metagpt/tools/search_engine_ddg.py b/metagpt/tools/search_engine_ddg.py index 78562c77e..57bc61b82 100644 --- a/metagpt/tools/search_engine_ddg.py +++ b/metagpt/tools/search_engine_ddg.py @@ -1,14 +1,11 @@ #!/usr/bin/env python -""" -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. -""" from __future__ import annotations import asyncio import json from concurrent import futures -from typing import Literal, overload, Optional +from typing import Literal, overload try: from duckduckgo_search import DDGS @@ -18,6 +15,8 @@ except ImportError: "You can install it by running the command: `pip install -e.[search-ddg]`" ) +from metagpt.config import CONFIG + class DDGAPIWrapper: """Wrapper around duckduckgo_search API. @@ -26,44 +25,43 @@ class DDGAPIWrapper: """ def __init__( - self, - *, - global_proxy: Optional[str] = None, - loop: asyncio.AbstractEventLoop | None = None, - executor: futures.Executor | None = None, + self, + *, + loop: asyncio.AbstractEventLoop | None = None, + executor: futures.Executor | None = None, ): kwargs = {} - if global_proxy: - kwargs["proxies"] = global_proxy + if CONFIG.global_proxy: + kwargs["proxies"] = CONFIG.global_proxy self.loop = loop self.executor = executor self.ddgs = DDGS(**kwargs) @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[True] = True, - focus: list[str] | None = None, + self, + query: str, + max_results: int = 8, + as_string: Literal[True] = True, + focus: list[str] | None = None, ) -> str: ... @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[False] = False, - focus: list[str] | None = None, + self, + query: str, + max_results: int = 8, + as_string: Literal[False] = False, + focus: list[str] | None = None, ) -> list[dict[str, str]]: ... async def run( - self, - query: str, - max_results: int = 8, - as_string: bool = True, + self, + query: str, + max_results: int = 8, + as_string: bool = True, ) -> str | list[dict]: """Return the results of a Google search using the official Google API diff --git a/metagpt/tools/search_engine_googleapi.py b/metagpt/tools/search_engine_googleapi.py index b5aeb5875..b9faf2ced 100644 --- a/metagpt/tools/search_engine_googleapi.py +++ b/metagpt/tools/search_engine_googleapi.py @@ -1,8 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. -""" from __future__ import annotations import asyncio @@ -14,6 +11,7 @@ from urllib.parse import urlparse import httplib2 from pydantic import BaseModel, validator +from metagpt.config import CONFIG from metagpt.logs import logger try: @@ -29,7 +27,6 @@ except ImportError: class GoogleAPIWrapper(BaseModel): google_api_key: Optional[str] = None google_cse_id: Optional[str] = None - global_proxy: Optional[str] = None loop: Optional[asyncio.AbstractEventLoop] = None executor: Optional[futures.Executor] = None @@ -39,6 +36,7 @@ class GoogleAPIWrapper(BaseModel): @validator("google_api_key", always=True) @classmethod def check_google_api_key(cls, val: str): + val = val or CONFIG.google_api_key if not val: raise ValueError( "To use, make sure you provide the google_api_key when constructing an object. Alternatively, " @@ -49,7 +47,8 @@ class GoogleAPIWrapper(BaseModel): @validator("google_cse_id", always=True) @classmethod - def check_google_cse_id(cls, val): + def check_google_cse_id(cls, val: str): + val = val or CONFIG.google_cse_id if not val: raise ValueError( "To use, make sure you provide the google_cse_id when constructing an object. Alternatively, " @@ -61,8 +60,8 @@ class GoogleAPIWrapper(BaseModel): @property def google_api_client(self): build_kwargs = {"developerKey": self.google_api_key} - if self.global_proxy: - parse_result = urlparse(self.global_proxy) + if CONFIG.global_proxy: + parse_result = urlparse(CONFIG.global_proxy) proxy_type = parse_result.scheme if proxy_type == "https": proxy_type = "http" diff --git a/metagpt/tools/search_engine_serpapi.py b/metagpt/tools/search_engine_serpapi.py index 1b93a91e9..750184198 100644 --- a/metagpt/tools/search_engine_serpapi.py +++ b/metagpt/tools/search_engine_serpapi.py @@ -4,14 +4,13 @@ @Time : 2023/5/23 18:27 @Author : alexanderwu @File : search_engine_serpapi.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import Any, Dict, Optional, Tuple import aiohttp from pydantic import BaseModel, Field, validator -from metagpt.config import Config +from metagpt.config import CONFIG class SerpAPIWrapper(BaseModel): @@ -33,6 +32,7 @@ class SerpAPIWrapper(BaseModel): @validator("serpapi_api_key", always=True) @classmethod def check_serpapi_api_key(cls, val: str): + val = val or CONFIG.serpapi_api_key if not val: raise ValueError( "To use, make sure you provide the serpapi_api_key when constructing an object. Alternatively, " @@ -112,4 +112,4 @@ class SerpAPIWrapper(BaseModel): if __name__ == "__main__": import fire - fire.Fire(SerpAPIWrapper(Config().runtime_options).run) + fire.Fire(SerpAPIWrapper().run) diff --git a/metagpt/tools/search_engine_serper.py b/metagpt/tools/search_engine_serper.py index 849839f05..0eec2694b 100644 --- a/metagpt/tools/search_engine_serper.py +++ b/metagpt/tools/search_engine_serper.py @@ -4,7 +4,6 @@ @Time : 2023/5/23 18:27 @Author : alexanderwu @File : search_engine_serpapi.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import json from typing import Any, Dict, Optional, Tuple @@ -12,6 +11,8 @@ from typing import Any, Dict, Optional, Tuple import aiohttp from pydantic import BaseModel, Field, validator +from metagpt.config import CONFIG + class SerperWrapper(BaseModel): search_engine: Any #: :meta private: @@ -25,6 +26,7 @@ class SerperWrapper(BaseModel): @validator("serper_api_key", always=True) @classmethod def check_serper_api_key(cls, val: str): + val = val or CONFIG.serper_api_key if not val: raise ValueError( "To use, make sure you provide the serper_api_key when constructing an object. Alternatively, " diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index 35ccdf78b..25bce124a 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -4,13 +4,11 @@ @Time : 2023/5/2 17:46 @Author : alexanderwu @File : test_search_engine.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from __future__ import annotations import pytest -from metagpt.config import Config from metagpt.logs import logger from metagpt.tools import SearchEngineType from metagpt.tools.search_engine import SearchEngine @@ -18,7 +16,9 @@ from metagpt.tools.search_engine import SearchEngine class MockSearchEnine: async def run(self, query: str, max_results: int = 8, as_string: bool = True) -> str | list[dict[str, str]]: - rets = [{"url": "https://metagpt.com/mock/{i}", "title": query, "snippet": query * i} for i in range(max_results)] + rets = [ + {"url": "https://metagpt.com/mock/{i}", "title": query, "snippet": query * i} for i in range(max_results) + ] return "\n".join(rets) if as_string else rets @@ -36,13 +36,16 @@ class MockSearchEnine: (SearchEngineType.DUCK_DUCK_GO, None, 6, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 8, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 6, False), - ], ) -async def test_search_engine(search_engine_typpe, run_func, max_results, as_string): - conf = Config() - search_engine = SearchEngine(options=conf.runtime_options, engine=search_engine_typpe, run_func=run_func) - rsp = await search_engine.run(query="metagpt", max_results=max_results, as_string=as_string) +async def test_search_engine( + search_engine_typpe, + run_func, + max_results, + as_string, +): + search_engine = SearchEngine(search_engine_typpe, run_func) + rsp = await search_engine.run("metagpt", max_results=max_results, as_string=as_string) logger.info(rsp) if as_string: assert isinstance(rsp, str) From c5e16330a21231abbf2f326889e941ce3a890995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 18:51:46 +0800 Subject: [PATCH 175/398] feat: +path --- metagpt/utils/s3.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 85837fedb..d13030292 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -1,4 +1,5 @@ import base64 +import os.path import traceback import uuid from typing import Optional @@ -138,8 +139,11 @@ class S3: await file.write(data) bucket = CONFIG.S3.get("bucket") - await self.upload_file(bucket=bucket, local_path=pathname, object_name=object_name) - return await self.get_object_url(bucket=bucket, object_name=object_name) + object_pathname = CONFIG.S3.get("path") or "system" + object_pathname += f"/{object_name}" + object_pathname = os.path.normpath(object_pathname) + await self.upload_file(bucket=bucket, local_path=pathname, object_name=object_pathname) + return await self.get_object_url(bucket=bucket, object_name=object_pathname) except Exception as e: logger.exception(f"{e}, stack:{traceback.format_exc()}") return None From 2148e4e4f47edc8e108daf261fb1166b31012f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 19:17:35 +0800 Subject: [PATCH 176/398] feat: +skill config --- metagpt/learn/skill_loader.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index 1cd83240d..83200bca6 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -7,11 +7,13 @@ @Desc : Skill YAML Configuration Loader. """ from pathlib import Path -from typing import List, Dict, Optional +from typing import Dict, List, Optional import yaml from pydantic import BaseModel, Field +from metagpt.config import CONFIG + class Example(BaseModel): ask: str @@ -52,7 +54,7 @@ class SkillLoader: def __init__(self, skill_yaml_file_name: Path = None): if not skill_yaml_file_name: skill_yaml_file_name = Path(__file__).parent.parent.parent / ".well-known/skills.yaml" - with open(str(skill_yaml_file_name), 'r') as file: + with open(str(skill_yaml_file_name), "r") as file: skills = yaml.safe_load(file) self._skills = SkillsDeclaration(**skills) @@ -62,8 +64,18 @@ class SkillLoader: if not entity_skills: return {} + agent_skills = CONFIG.agent_skills + if not agent_skills: + return {} + + class AgentSkill(BaseModel): + name: str + + names = [AgentSkill(**i).name for i in agent_skills] description_to_name_mappings = {} for s in entity_skills.skills: + if s.name not in names: + continue description_to_name_mappings[s.description] = s.name return description_to_name_mappings From 610dd8b4ba2771bb7f1d38b101be7fb2cb425fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 19:25:06 +0800 Subject: [PATCH 177/398] feat: +skill config --- metagpt/utils/s3.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index d13030292..531142737 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -131,9 +131,11 @@ class S3: async def cache(self, data: str, format: str = "") -> str: """Save data to remote S3 and return url""" object_name = str(uuid.uuid4()).replace("-", "") - pathname = WORKSPACE_ROOT / "s3_tmp" / object_name + path = WORKSPACE_ROOT / "s3_tmp" + path.mkdir(exist_ok=True) + pathname = path / object_name try: - async with aiofiles.open(pathname, mode="w") as file: + async with aiofiles.open(str(pathname), mode="w") as file: if format == BASE64_FORMAT: data = base64.b64decode(data) await file.write(data) @@ -142,7 +144,7 @@ class S3: object_pathname = CONFIG.S3.get("path") or "system" object_pathname += f"/{object_name}" object_pathname = os.path.normpath(object_pathname) - await self.upload_file(bucket=bucket, local_path=pathname, object_name=object_pathname) + await self.upload_file(bucket=bucket, local_path=str(pathname), object_name=object_pathname) return await self.get_object_url(bucket=bucket, object_name=object_pathname) except Exception as e: logger.exception(f"{e}, stack:{traceback.format_exc()}") From 86e3ca0ba99c7522cdbca9df35e3b8fc965fa384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 19:44:26 +0800 Subject: [PATCH 178/398] feat: +skill config --- metagpt/utils/s3.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 531142737..6df244197 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -2,13 +2,14 @@ import base64 import os.path import traceback import uuid +from pathlib import Path from typing import Optional import aioboto3 import aiofiles from metagpt.config import CONFIG -from metagpt.const import BASE64_FORMAT, WORKSPACE_ROOT +from metagpt.const import BASE64_FORMAT from metagpt.logs import logger @@ -131,8 +132,7 @@ class S3: async def cache(self, data: str, format: str = "") -> str: """Save data to remote S3 and return url""" object_name = str(uuid.uuid4()).replace("-", "") - path = WORKSPACE_ROOT / "s3_tmp" - path.mkdir(exist_ok=True) + path = Path(__file__).parent pathname = path / object_name try: async with aiofiles.open(str(pathname), mode="w") as file: @@ -145,7 +145,10 @@ class S3: object_pathname += f"/{object_name}" object_pathname = os.path.normpath(object_pathname) await self.upload_file(bucket=bucket, local_path=str(pathname), object_name=object_pathname) + pathname.unlink(missing_ok=True) + return await self.get_object_url(bucket=bucket, object_name=object_pathname) except Exception as e: logger.exception(f"{e}, stack:{traceback.format_exc()}") + pathname.unlink(missing_ok=True) return None From 7881937e8fb3c5a4ef183d6460fc1d741c0d6b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 20:47:14 +0800 Subject: [PATCH 179/398] feat: test s3 --- metagpt/utils/s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 6df244197..74c3f1654 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -129,13 +129,13 @@ class S3: logger.error(f"Failed to download the file from S3: {e}") raise e - async def cache(self, data: str, format: str = "") -> str: + async def cache(self, data: str, file_ext: str, format: str = "") -> str: """Save data to remote S3 and return url""" - object_name = str(uuid.uuid4()).replace("-", "") + object_name = str(uuid.uuid4()).replace("-", "") + file_ext path = Path(__file__).parent pathname = path / object_name try: - async with aiofiles.open(str(pathname), mode="w") as file: + async with aiofiles.open(str(pathname), mode="wb") as file: if format == BASE64_FORMAT: data = base64.b64decode(data) await file.write(data) From 9d74e8e157029ec1e49d307adc121772e1dc048f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 20:51:02 +0800 Subject: [PATCH 180/398] feat: test s3 --- metagpt/learn/text_to_image.py | 4 ++-- metagpt/learn/text_to_speech.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index c5f554ef3..dd85cf617 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -33,7 +33,7 @@ async def text_to_image(text, size_type: str = "512x512", openai_api_key="", mod raise openai.error.InvalidRequestError("缺少必要的参数") s3 = S3() - url = await s3.cache(base64_data, BASE64_FORMAT) + url = await s3.cache(data=base64_data, file_ext=".png", format=BASE64_FORMAT) if url: - return url + return f"[{text}]({url})" return image_declaration + base64_data if base64_data else "" diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 7883ae9f3..819da2364 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -22,7 +22,7 @@ async def text_to_speech( role="Girl", subscription_key="", region="", - **kwargs + **kwargs, ): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -41,9 +41,9 @@ async def text_to_speech( if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region): base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) s3 = S3() - url = await s3.cache(base64_data, BASE64_FORMAT) + url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) if url: - return url + return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data raise openai.error.InvalidRequestError("缺少必要的参数") From 7bd62b6a498543d4fdf95e62e643eebed8743c3f Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sat, 2 Sep 2023 21:04:51 +0800 Subject: [PATCH 181/398] add google search skill --- .well-known/skills.yaml | 19 ++++++++++++++++ metagpt/learn/__init__.py | 6 ++--- metagpt/learn/google_search.py | 12 ++++++++++ tests/metagpt/learn/test_google_search.py | 27 +++++++++++++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 metagpt/learn/google_search.py create mode 100644 tests/metagpt/learn/test_google_search.py diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 06b9ffd0c..009368dbe 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -45,3 +45,22 @@ entities: returns: type: string format: base64 + + - name: web_search + description: Perform Google searches to provide real-time information. + id: web_search.web_search + x-prerequisite: + - name: SEARCH_ENGINE + description: "Supported values: serpapi/google/serper/ddg" + - name: SERPER_API_KEY + description: "SERPER API KEY, For more details, checkout: `https://serper.dev/api-key`" + arguments: + query: 'The search query. Required.' + max_results: 'The number of search results to retrieve. Default value: 6.' + examples: + - ask: 'Search for information about artificial intelligence' + answer: 'web_search(query="Search for information about artificial intelligence", max_results=6)' + - ask: 'Find news articles about climate change' + answer: 'web_search(query="Find news articles about climate change", max_results=6)' + returns: + type: string \ No newline at end of file diff --git a/metagpt/learn/__init__.py b/metagpt/learn/__init__.py index c8270dbfb..bab9f3e37 100644 --- a/metagpt/learn/__init__.py +++ b/metagpt/learn/__init__.py @@ -8,8 +8,6 @@ from metagpt.learn.text_to_image import text_to_image from metagpt.learn.text_to_speech import text_to_speech +from metagpt.learn.google_search import google_search -__all__ = [ - "text_to_image", - "text_to_speech", -] \ No newline at end of file +__all__ = ["text_to_image", "text_to_speech", "google_search"] diff --git a/metagpt/learn/google_search.py b/metagpt/learn/google_search.py new file mode 100644 index 000000000..ef099fe94 --- /dev/null +++ b/metagpt/learn/google_search.py @@ -0,0 +1,12 @@ +from metagpt.tools.search_engine import SearchEngine + + +async def google_search(query: str, max_results: int = 6, **kwargs): + """Perform a web search and retrieve search results. + + :param query: The search query. + :param max_results: The number of search results to retrieve + :return: The web search results in markdown format. + """ + resluts = await SearchEngine().run(query, max_results=max_results, as_string=False) + return "\n".join(f"{i}. [{j['title']}]({j['link']}): {j['snippet']}" for i, j in enumerate(resluts, 1)) diff --git a/tests/metagpt/learn/test_google_search.py b/tests/metagpt/learn/test_google_search.py new file mode 100644 index 000000000..da32e8923 --- /dev/null +++ b/tests/metagpt/learn/test_google_search.py @@ -0,0 +1,27 @@ +import asyncio + +from pydantic import BaseModel + +from metagpt.learn.google_search import google_search + + +async def mock_google_search(): + class Input(BaseModel): + input: str + + inputs = [{"input": "ai agent"}] + + for i in inputs: + seed = Input(**i) + result = await google_search(seed.input) + assert result != "" + + +def test_suite(): + loop = asyncio.get_event_loop() + task = loop.create_task(mock_google_search()) + loop.run_until_complete(task) + + +if __name__ == "__main__": + test_suite() From 842aac82fcda09a6879edfdcf40adfc12b053790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 21:11:44 +0800 Subject: [PATCH 182/398] fixbug: summary too long --- metagpt/provider/openai_api.py | 45 ++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index c08a34f7e..4764b6aad 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -226,38 +226,45 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - async def get_summary(self, text: str, max_words=20): + async def get_summary(self, text: str, max_words=200): + max_token_count = DEFAULT_MAX_TOKENS + max_count = 100 + while max_count > 0: + if len(text) < max_token_count: + return await self._get_summary(text, max_words=max_words) + + text_windows = self.split_texts(text, window_size=max_token_count - max_words) + summaries = [] + for ws in text_windows: + response = await self._get_summary(ws, max_words=max_words) + summaries.append(response) + if len(summaries) == 1: + return summaries[0] + + # Merged and retry + text = "\n".join(summaries) + + max_count -= 1 # safeguard + raise openai.error.InvalidRequestError("text too long") + + async def _get_summary(self, text: str, max_words=20): """Generate text summary""" if len(text) < max_words: return text - language = CONFIG.language or DEFAULT_LANGUAGE - command = f"Translate the above content into a {language} summary of less than {max_words} words." + command = f"Translate the above content into a summary of less than {max_words} words." msg = text + "\n\n" + command logger.info(f"summary ask:{msg}") response = await self.aask(msg=msg, system_msgs=[]) logger.info(f"summary rsp: {response}") return response - async def get_context_title(self, text: str, max_token_count_per_ask=None, max_words=5) -> str: + async def get_context_title(self, text: str, max_words=5) -> str: """Generate text title""" - max_response_token_count = 50 - max_token_count = max_token_count_per_ask or CONFIG.MAX_TOKENS or DEFAULT_MAX_TOKENS - while True: - text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) - - summaries = [] - for ws in text_windows: - response = await self.get_summary(ws, max_words=max_response_token_count) - summaries.append(response) - if len(summaries) == 1: - return summaries[0] - text = "\n".join(summaries) - if len(text) <= max_words * 2 and len(text) <= max_token_count: - break + summary = await self.get_summary(text, max_words) language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." - summaries.append(command) + summaries = [summary, command] msg = "\n".join(summaries) logger.info(f"title ask:{msg}") response = await self.aask(msg=msg, system_msgs=[]) From 3112680324a2ba42ecf39b31796d14c605509848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 21:30:19 +0800 Subject: [PATCH 183/398] fixbug: summary too long --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 4764b6aad..b1d8aaa4a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -260,7 +260,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def get_context_title(self, text: str, max_words=5) -> str: """Generate text title""" - summary = await self.get_summary(text, max_words) + summary = await self.get_summary(text, max_words=500) language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." From 264799541155c6ff59727a15e55b7b2ec5d4582c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 21:38:49 +0800 Subject: [PATCH 184/398] fixbug: summary too long --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index b1d8aaa4a..b2a0faca5 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,7 +233,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if len(text) < max_token_count: return await self._get_summary(text, max_words=max_words) - text_windows = self.split_texts(text, window_size=max_token_count - max_words) + padding_size = 20 if max_token_count > 20 else 0 + text_windows = self.split_texts(text, window_size=max_token_count - padding_size) summaries = [] for ws in text_windows: response = await self._get_summary(ws, max_words=max_words) From 5980b08c80451740ad5c3c3e057a146dcffb8694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 21:48:23 +0800 Subject: [PATCH 185/398] fixbug: summary too long --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index fdd697b59..c707cb6f1 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -123,7 +123,7 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_context_title(history_text, max_token_count_per_ask=1000, max_words=500) + history_summary = await self._llm.get_summary(history_text, max_words=500) if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk From bf6388d1717cab8bd78671dbe0c13d7e421e7298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 22:28:56 +0800 Subject: [PATCH 186/398] =?UTF-8?q?fixbug:=20fix=20=E5=9B=BE=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/learn/text_to_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index dd85cf617..23c2bddad 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -35,5 +35,5 @@ async def text_to_image(text, size_type: str = "512x512", openai_api_key="", mod s3 = S3() url = await s3.cache(data=base64_data, file_ext=".png", format=BASE64_FORMAT) if url: - return f"[{text}]({url})" + return f"![{text}]({url})" return image_declaration + base64_data if base64_data else "" From 69ef295b26f185f12c9e8bb05d79695425d01df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 12:11:37 +0800 Subject: [PATCH 187/398] fixbug: skill name --- metagpt/roles/assistant.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index c707cb6f1..0bce4a3f9 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -57,7 +57,9 @@ class Assistant(Role): prompt = f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): - prompt += f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: text_to_image\n" + prompt += ( + f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" + ) prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" prompt += "Otherwise, rewrite and return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" From 5079add5f829b05f193f91bb9dce121cf29e6517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 12:55:25 +0800 Subject: [PATCH 188/398] debug: +code --- metagpt/actions/skill_action.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index 3ef0087fc..6bce2a634 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -7,8 +7,8 @@ @Desc : Call learned skill """ from __future__ import annotations + import ast -import importlib import traceback from metagpt.actions import Action, ActionOutput @@ -18,7 +18,7 @@ from metagpt.logs import logger class ArgumentsParingAction(Action): def __init__(self, last_talk: str, skill: Skill, context=None, llm=None, **kwargs): - super(ArgumentsParingAction, self).__init__(name='', context=context, llm=llm) + super(ArgumentsParingAction, self).__init__(name="", context=context, llm=llm) self.skill = skill self.ask = last_talk self.rsp = None @@ -56,10 +56,10 @@ class ArgumentsParingAction(Action): return None begin_ix = txt.find(prefix) end_ix = txt.rfind(")") - args_txt = txt[begin_ix + len(prefix): end_ix] + args_txt = txt[begin_ix + len(prefix) : end_ix] logger.info(args_txt) fake_expression = f"dict({args_txt})" - parsed_expression = ast.parse(fake_expression, mode='eval') + parsed_expression = ast.parse(fake_expression, mode="eval") args = {} for keyword in parsed_expression.body.keywords: key = keyword.arg @@ -70,7 +70,7 @@ class ArgumentsParingAction(Action): class SkillAction(Action): def __init__(self, skill: Skill, args: dict, context=None, llm=None, **kwargs): - super(SkillAction, self).__init__(name='', context=context, llm=llm) + super(SkillAction, self).__init__(name="", context=context, llm=llm) self._skill = skill self._args = args self.rsp = None @@ -86,17 +86,21 @@ class SkillAction(Action): @staticmethod async def find_and_call_function(function_name, args, **kwargs): + from metagpt.learn import text_to_speech + try: - module = importlib.import_module("metagpt.learn") - function = getattr(module, function_name) - # 调用函数并返回结果 - result = await function(**args, **kwargs) + result = await text_to_speech(**args, **kwargs) + # module = importlib.import_module("metagpt.learn") + # function = getattr(module, function_name) + # # 调用函数并返回结果 + # result = await function(**args, **kwargs) return result except (ModuleNotFoundError, AttributeError): logger.error(f"{function_name} not found") return None -if __name__ == '__main__': - ArgumentsParingAction.parse_arguments(skill_name="text_to_image", - txt='`text_to_image(text="Draw an apple", size_type="512x512")`') +if __name__ == "__main__": + ArgumentsParingAction.parse_arguments( + skill_name="text_to_image", txt='`text_to_image(text="Draw an apple", size_type="512x512")`' + ) From 04b348e92967d6a99ca0425c6aad1f3b34485e30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 13:31:52 +0800 Subject: [PATCH 189/398] feat: archive --- metagpt/actions/skill_action.py | 36 ++++++++++++++++++++++++--------- metagpt/learn/text_to_speech.py | 10 ++++----- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index 6bce2a634..660d785ff 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -9,10 +9,14 @@ from __future__ import annotations import ast +import asyncio +import importlib import traceback +from copy import deepcopy from metagpt.actions import Action, ActionOutput -from metagpt.learn.skill_loader import Skill +from metagpt.config import CONFIG +from metagpt.learn.skill_loader import Returns, Skill from metagpt.logs import logger @@ -77,8 +81,13 @@ class SkillAction(Action): async def run(self, *args, **kwargs) -> str | ActionOutput | None: """Run action""" + options = deepcopy(kwargs) + if self._args: + for k in self._args.keys(): + if k in options: + options.pop(k) try: - self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **kwargs) + self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **options) except Exception as e: logger.exception(f"{e}, traceback:{traceback.format_exc()}") self.rsp = f"Error: {e}" @@ -86,14 +95,11 @@ class SkillAction(Action): @staticmethod async def find_and_call_function(function_name, args, **kwargs): - from metagpt.learn import text_to_speech - try: - result = await text_to_speech(**args, **kwargs) - # module = importlib.import_module("metagpt.learn") - # function = getattr(module, function_name) - # # 调用函数并返回结果 - # result = await function(**args, **kwargs) + module = importlib.import_module("metagpt.learn") + function = getattr(module, function_name) + # 调用函数并返回结果 + result = await function(**args, **kwargs) return result except (ModuleNotFoundError, AttributeError): logger.error(f"{function_name} not found") @@ -104,3 +110,15 @@ if __name__ == "__main__": ArgumentsParingAction.parse_arguments( skill_name="text_to_image", txt='`text_to_image(text="Draw an apple", size_type="512x512")`' ) + CONFIG.set_context({}) + args = {"text": "hello world", "role": "Girl"} + action = SkillAction( + skill=Skill( + name="text_to_speech", description="", id="", arguments={}, examples=[], returns=Returns(type="string") + ), + args=args, + ) + loop = asyncio.new_event_loop() + t = loop.create_task(action.run()) + r = loop.run_until_complete(t) + print(r) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 819da2364..eaceb3313 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -9,9 +9,7 @@ import openai from metagpt.config import CONFIG -from metagpt.const import BASE64_FORMAT from metagpt.tools.azure_tts import oas3_azsure_tts -from metagpt.utils.s3 import S3 async def text_to_speech( @@ -40,10 +38,10 @@ async def text_to_speech( audio_declaration = "data:audio/wav;base64," if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region): base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) - s3 = S3() - url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) - if url: - return f"[{text}]({url})" + # s3 = S3() + # url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) + # if url: + # return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data raise openai.error.InvalidRequestError("缺少必要的参数") From 0dddab18b44a053ef2d2206bfbf669750de0df3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 13:40:05 +0800 Subject: [PATCH 190/398] fixbug: no param --- metagpt/learn/text_to_speech.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index eaceb3313..691aa7f6a 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -44,4 +44,4 @@ async def text_to_speech( # return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data - raise openai.error.InvalidRequestError("缺少必要的参数") + raise openai.error.InvalidRequestError(message="AZURE_TTS_SUBSCRIPTION_KEY and AZURE_TTS_REGION error", param={}) From ef98ad4043b377037dd38d2aec1354bb7ea7be03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 13:46:23 +0800 Subject: [PATCH 191/398] fixbug: no param --- metagpt/actions/skill_action.py | 16 +--------------- metagpt/learn/text_to_speech.py | 10 ++++++---- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index 660d785ff..758591fdd 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -9,14 +9,12 @@ from __future__ import annotations import ast -import asyncio import importlib import traceback from copy import deepcopy from metagpt.actions import Action, ActionOutput -from metagpt.config import CONFIG -from metagpt.learn.skill_loader import Returns, Skill +from metagpt.learn.skill_loader import Skill from metagpt.logs import logger @@ -110,15 +108,3 @@ if __name__ == "__main__": ArgumentsParingAction.parse_arguments( skill_name="text_to_image", txt='`text_to_image(text="Draw an apple", size_type="512x512")`' ) - CONFIG.set_context({}) - args = {"text": "hello world", "role": "Girl"} - action = SkillAction( - skill=Skill( - name="text_to_speech", description="", id="", arguments={}, examples=[], returns=Returns(type="string") - ), - args=args, - ) - loop = asyncio.new_event_loop() - t = loop.create_task(action.run()) - r = loop.run_until_complete(t) - print(r) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 691aa7f6a..81bc8512b 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -9,7 +9,9 @@ import openai from metagpt.config import CONFIG +from metagpt.const import BASE64_FORMAT from metagpt.tools.azure_tts import oas3_azsure_tts +from metagpt.utils.s3 import S3 async def text_to_speech( @@ -38,10 +40,10 @@ async def text_to_speech( audio_declaration = "data:audio/wav;base64," if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region): base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) - # s3 = S3() - # url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) - # if url: - # return f"[{text}]({url})" + s3 = S3() + url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) + if url: + return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data raise openai.error.InvalidRequestError(message="AZURE_TTS_SUBSCRIPTION_KEY and AZURE_TTS_REGION error", param={}) From 2856acb3f343b7a4d14643c52352ed2da6bc3119 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Sun, 3 Sep 2023 17:22:36 +0800 Subject: [PATCH 192/398] faiss search kwargs --- metagpt/document_store/faiss_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 051bc2507..b034f40b2 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -51,7 +51,7 @@ class FaissStore(LocalStore): store.index = index def search(self, query, expand_cols=False, sep='\n', *args, k=5, **kwargs): - rsp = self.store.similarity_search(query, k=k) + rsp = self.store.similarity_search(query, k=k, **kwargs) logger.debug(rsp) if expand_cols: return str(sep.join([f"{x.page_content}: {x.metadata}" for x in rsp])) From 2285f0566ed214fd4cc4636f4da258b138931258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 19:38:01 +0800 Subject: [PATCH 193/398] refactor: prompt --- metagpt/actions/talk_action.py | 53 ++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 4eed0d4f8..12b9da030 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -27,22 +27,16 @@ class TalkAction(Action): @property def prompt(self): - prompt = "" - if CONFIG.agent_description: - prompt = ( - f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, " - f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " - f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" - ) - prompt += f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" - prompt += f"{self._history_summary}\n\n" - if self._history_summary != "": - prompt += "According to the historical conversation above, " - language = CONFIG.language or DEFAULT_LANGUAGE - prompt += ( - f"Answer the following questions in {language}, and the answers must follow the Markdown format.\n " - f"{self._talk}" - ) + kvs = { + "{role}": CONFIG.agent_description or "", + "{history}": self._history_summary or "", + "{knowledge}": self._knowledge or "", + "{language}": CONFIG.language or DEFAULT_LANGUAGE, + "{ask}": self._talk, + } + prompt = TalkAction.__FORMATION__ + for k, v in kvs.items(): + prompt = prompt.replace(k, v) return prompt async def run(self, *args, **kwargs) -> ActionOutput: @@ -52,3 +46,30 @@ class TalkAction(Action): logger.info(rsp) self._rsp = ActionOutput(content=rsp) return self._rsp + + __FORMATION__ = """Formation: "Capacity and role" defines the role you are currently playing; + "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; + "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; + "Statement" defines the work detail you need to complete at this stage; + "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; + "Constraint" defines the conditions that your responses must comply with. + +Capacity and role: {role} +Statement: Your responses should align with the role-play agreement, maintaining the + character's persona and habits. When faced with unrelated questions, playfully decline to answer without revealing + your AI nature to preserve the character's image. Statement: the answers must follow the Markdown format. + +[HISTORY_BEGIN] +{history} +[HISTORY_END] + +[KNOWLEDGE_BEGIN] +{knowledge} +[KNOWLEDGE_END] + +Statement: According to the historical conversation and knowledge above if helpful, Answer the following questions in + {language}, and the answers must follow the Markdown format. + + [ASK_BEGIN] + {ask} + [ASK_END]""" From d6ffa4906f71205ec4a358152eb1ba81fffe60f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 19:55:53 +0800 Subject: [PATCH 194/398] refactor: prompt --- metagpt/actions/talk_action.py | 39 ++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 12b9da030..fead3c8b9 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -34,7 +34,7 @@ class TalkAction(Action): "{language}": CONFIG.language or DEFAULT_LANGUAGE, "{ask}": self._talk, } - prompt = TalkAction.__FORMATION__ + prompt = TalkAction.__FORMATION_LOOSE__ for k, v in kvs.items(): prompt = prompt.replace(k, v) return prompt @@ -57,7 +57,34 @@ class TalkAction(Action): Capacity and role: {role} Statement: Your responses should align with the role-play agreement, maintaining the character's persona and habits. When faced with unrelated questions, playfully decline to answer without revealing - your AI nature to preserve the character's image. Statement: the answers must follow the Markdown format. + your AI nature to preserve the character's image. + +[HISTORY_BEGIN] +{history} +[HISTORY_END] + +[KNOWLEDGE_BEGIN] +{knowledge} +[KNOWLEDGE_END] + +Statement: According to the historical conversation and knowledge above if helpful, Answer the following questions in +{language}, and the answers must follow the Markdown format, excluding any tag likes "[HISTORY_BEGIN]", +"[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" + +[ASK_BEGIN] +{ask} +[ASK_END]""" + + __FORMATION_LOOSE__ = """Formation: "Capacity and role" defines the role you are currently playing; + "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; + "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; + "Statement" defines the work detail you need to complete at this stage; + "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; + "Constraint" defines the conditions that your responses must comply with. + +Capacity and role: {role} +Statement: Your responses should maintaining the character's persona and habits. When faced with unrelated questions +, playfully decline to answer without revealing your AI nature to preserve the character's image. [HISTORY_BEGIN] {history} @@ -69,7 +96,7 @@ Statement: Your responses should align with the role-play agreement, maintaining Statement: According to the historical conversation and knowledge above if helpful, Answer the following questions in {language}, and the answers must follow the Markdown format. - - [ASK_BEGIN] - {ask} - [ASK_END]""" + +[ASK_BEGIN] +{ask} +[ASK_END]""" From b5c149f22507ffe139ca9333c50934af16a36611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 20:02:24 +0800 Subject: [PATCH 195/398] refactor: prompt --- metagpt/actions/talk_action.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index fead3c8b9..2a04fb9c8 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -67,9 +67,10 @@ Statement: Your responses should align with the role-play agreement, maintaining {knowledge} [KNOWLEDGE_END] -Statement: According to the historical conversation and knowledge above if helpful, Answer the following questions in -{language}, and the answers must follow the Markdown format, excluding any tag likes "[HISTORY_BEGIN]", -"[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" +Statement: If the information is insufficient, you can search in the historical conversation or knowledge. +Statement: Answer the following questions in {language}, and the answers must follow the Markdown format + , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" + , "[ASK_END]" [ASK_BEGIN] {ask} @@ -94,8 +95,10 @@ Statement: Your responses should maintaining the character's persona and habits. {knowledge} [KNOWLEDGE_END] -Statement: According to the historical conversation and knowledge above if helpful, Answer the following questions in - {language}, and the answers must follow the Markdown format. +Statement: If the information is insufficient, you can search in the historical conversation or knowledge. +Statement: Answer the following questions in {language}, and the answers must follow the Markdown format + , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" + , "[ASK_END]" [ASK_BEGIN] {ask} From b036b5d22ee17c59f0d01124dea98c34e8ff0a99 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sun, 3 Sep 2023 22:22:26 +0800 Subject: [PATCH 196/398] remove openai global settings --- metagpt/provider/openai_api.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index b2a0faca5..844cd4c1c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -77,21 +77,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ def __init__(self): - self.__init_openai(CONFIG) self.llm = openai self.model = CONFIG.openai_api_model self.auto_max_tokens = False + self.rpm = int(CONFIG.get("RPM", 10)) RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self, config): - openai.api_key = config.openai_api_key - if config.openai_api_base: - openai.api_base = config.openai_api_base - if config.openai_api_type: - openai.api_type = config.openai_api_type - openai.api_version = config.openai_api_version - self.rpm = int(config.get("RPM", 10)) - async def _achat_completion_stream(self, messages: list[dict]) -> str: response = await self.async_retry_call( openai.ChatCompletion.acreate, **self._cons_kwargs(messages), stream=True @@ -133,6 +124,10 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): "temperature": 0.3, } kwargs["timeout"] = 3 + kwargs["api_base"] = CONFIG.openai_api_base + kwargs["api_key"] = CONFIG.openai_api_key + kwargs["api_type"] = CONFIG.openai_api_type + kwargs["api_version"] = CONFIG.openai_api_version return kwargs async def _achat_completion(self, messages: list[dict]) -> dict: From e06aa62ac4dcd5ed4ec401f16ae34ecd4f178034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 10:11:19 +0800 Subject: [PATCH 197/398] refactor: prompt --- metagpt/actions/talk_action.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 2a04fb9c8..526d921f8 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -27,6 +27,26 @@ class TalkAction(Action): @property def prompt(self): + prompt = "" + if CONFIG.agent_description: + prompt = ( + f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, " + f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " + f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" + ) + prompt += f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" + prompt += f"{self._history_summary}\n\n" + if self._history_summary != "": + prompt += "According to the historical conversation above, " + language = CONFIG.language or DEFAULT_LANGUAGE + prompt += ( + f"Answer the following questions in {language}, and the answers must follow the Markdown format.\n " + f"{self._talk}" + ) + return prompt + + @property + def prompt_new(self): kvs = { "{role}": CONFIG.agent_description or "", "{history}": self._history_summary or "", From 63594cd8fd924ca3aff0153354fd78e5e415b507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 10:12:10 +0800 Subject: [PATCH 198/398] refactor: prompt --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 526d921f8..83504b62d 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -46,7 +46,7 @@ class TalkAction(Action): return prompt @property - def prompt_new(self): + def formation_prompt(self): kvs = { "{role}": CONFIG.agent_description or "", "{history}": self._history_summary or "", From 87f4c22b6050ea7b951498b03d3cc9149dc54fb9 Mon Sep 17 00:00:00 2001 From: Stitch-z <284618289@qq.com> Date: Mon, 4 Sep 2023 10:48:48 +0800 Subject: [PATCH 199/398] update: aioboto3 client async open file --- metagpt/utils/s3.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 74c3f1654..96b457972 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -44,8 +44,9 @@ class S3: """ try: async with self.session.client(**self.auth_config) as client: - with open(local_path, "rb") as file: - await client.put_object(Body=file, Bucket=bucket, Key=object_name) + async with aiofiles.open(local_path, mode="rb") as reader: + body = await reader.read() + await client.put_object(Body=body, Bucket=bucket, Key=object_name) logger.info(f"Successfully uploaded the file to path {object_name} in bucket {bucket} of s3.") except Exception as e: logger.error(f"Failed to upload the file to path {object_name} in bucket {bucket} of s3: {e}") @@ -119,12 +120,12 @@ class S3: async with self.session.client(**self.auth_config) as client: s3_object = await client.get_object(Bucket=bucket, Key=object_name) stream = s3_object["Body"] - with open(local_path, "wb") as local_file: + async with aiofiles.open(local_path, mode="wb") as writer: while True: file_data = await stream.read(chunk_size) if not file_data: break - local_file.write(file_data) + await writer.write(file_data) except Exception as e: logger.error(f"Failed to download the file from S3: {e}") raise e From d4878f23a0042bf983c1fef8947c649f7d4f4878 Mon Sep 17 00:00:00 2001 From: zhanglei Date: Mon, 4 Sep 2023 10:50:21 +0800 Subject: [PATCH 200/398] =?UTF-8?q?update:=E4=BF=AE=E6=94=B9get=5Fsummary?= =?UTF-8?q?=EF=BC=8C=E5=8A=A0=E4=B8=8A=E6=98=AF=E5=90=A6=E4=BF=9D=E6=8C=81?= =?UTF-8?q?=E8=AF=AD=E8=A8=80=E7=9A=84=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/provider/openai_api.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 844cd4c1c..26929575c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -221,18 +221,18 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - async def get_summary(self, text: str, max_words=200): + async def get_summary(self, text: str, max_words=200, keep_language: bool = False): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 while max_count > 0: if len(text) < max_token_count: - return await self._get_summary(text, max_words=max_words) + return await self._get_summary(text=text, max_words=max_words,keep_language=keep_language) padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) summaries = [] for ws in text_windows: - response = await self._get_summary(ws, max_words=max_words) + response = await self._get_summary(text=ws, max_words=max_words,keep_language=keep_language) summaries.append(response) if len(summaries) == 1: return summaries[0] @@ -243,11 +243,14 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): max_count -= 1 # safeguard raise openai.error.InvalidRequestError("text too long") - async def _get_summary(self, text: str, max_words=20): + async def _get_summary(self, text: str, max_words=20, keep_language: bool = False): """Generate text summary""" if len(text) < max_words: return text - command = f"Translate the above content into a summary of less than {max_words} words." + if keep_language: + command = f".Translate the above content into a summary of less than {max_words} words in language of the content." + else: + command = f"Translate the above content into a summary of less than {max_words} words." msg = text + "\n\n" + command logger.info(f"summary ask:{msg}") response = await self.aask(msg=msg, system_msgs=[]) From 2f95a8a2000aee5e1aa07a29259a81cdd0c800f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 11:42:14 +0800 Subject: [PATCH 201/398] feat: +config --- config/config.yaml | 9 ++++++++- metagpt/utils/redis.py | 0 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 metagpt/utils/redis.py diff --git a/config/config.yaml b/config/config.yaml index 7c3d212f6..765a74b8a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -83,4 +83,11 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k S3: access_key: "YOUR_S3_ACCESS_KEY" secret_key: "YOUR_S3_SECRET_KEY" - endpoint_url: "YOUR_S3_ENDPOINT_URL" \ No newline at end of file + endpoint_url: "YOUR_S3_ENDPOINT_URL" + +### Redis config +REDIS: + host: "YOUR_REDIS_HOST" + port: YOUR_REDIS_PORT, int + password: "YOUR_REDIS_PASSWORD" + db: YOUR_REDIS_DB_INDEX, int \ No newline at end of file diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py new file mode 100644 index 000000000..e69de29bb From 9cc85d631ad15fe369f1cd647a4071ca31bd6a94 Mon Sep 17 00:00:00 2001 From: zhanglei Date: Mon, 4 Sep 2023 11:50:22 +0800 Subject: [PATCH 202/398] =?UTF-8?q?update:=E4=BF=AE=E6=94=B9get=5Fsummary?= =?UTF-8?q?=EF=BC=8C=E5=8A=A0=E4=B8=8A=E6=98=AF=E5=90=A6=E4=BF=9D=E6=8C=81?= =?UTF-8?q?=E8=AF=AD=E8=A8=80=E7=9A=84=E9=85=8D=E7=BD=AE,=E5=BC=BA?= =?UTF-8?q?=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 26929575c..5c11ed7a6 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -248,7 +248,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if len(text) < max_words: return text if keep_language: - command = f".Translate the above content into a summary of less than {max_words} words in language of the content." + command = f".Translate the above content into a summary of less than {max_words} words in language of the content strictly." else: command = f"Translate the above content into a summary of less than {max_words} words." msg = text + "\n\n" + command From 96f833cf8fafcea3555efd5871bea2ed2364647f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 12:47:45 +0800 Subject: [PATCH 203/398] feat: +redis --- metagpt/memory/brain_memory.py | 34 ++++-- metagpt/utils/redis.py | 198 +++++++++++++++++++++++++++++++++ requirements.txt | 3 +- 3 files changed, 222 insertions(+), 13 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index a5a3dbfc7..275cd14df 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -6,7 +6,7 @@ @File : brain_memory.py @Desc : Support memory for multiple tasks and multiple mainlines. """ - +import hashlib from enum import Enum from typing import Dict, List @@ -28,6 +28,10 @@ class BrainMemory(pydantic.BaseModel): stack: List[Dict] = [] solution: List[Dict] = [] knowledge: List[Dict] = [] + # If the fingerprint of the history text is found in the `historical_summary_fingerprint`, + # it indicates that the text has already been incorporated into the `history summary`. + historical_summary_fingerprint: List[str] = [] + historical_summary: str = "" def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -58,17 +62,19 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) def move_to_solution(self, history_summary): - """放入solution队列，以备后续长程检索。目前还未加此功能，先用history_summary顶替""" - if len(self.history) < 2: - return - msgs = self.history[:-1] - self.solution.extend(msgs) - if not Message(**self.history[-1]).is_contain(MessageType.Talk.value): - self.solution.append(self.history[-1]) - self.history = [] - else: - self.history = self.history[-1:] - self.history.insert(0, Message(content="RESOLVED: " + history_summary)) + """Put it in the solution queue for future long-term retrieval. + This functionality hasn't been added yet, so use the history summary as a temporary substitute for now.""" + pass + # if len(self.history) < 2: + # return + # msgs = self.history[:-1] + # self.solution.extend(msgs) + # if not Message(**self.history[-1]).is_contain(MessageType.Talk.value): + # self.solution.append(self.history[-1]) + # self.history = [] + # else: + # self.history = self.history[-1:] + # self.history.insert(0, Message(content="RESOLVED: " + history_summary)) @property def last_talk(self): @@ -78,3 +84,7 @@ class BrainMemory(pydantic.BaseModel): if not last_msg.is_contain(MessageType.Talk.value): return None return last_msg.content + + @staticmethod + def get_md5(text: str) -> str: + return hashlib.md5(text.encode()).hexdigest() diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index e69de29bb..f2ae3222a 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -0,0 +1,198 @@ +# !/usr/bin/python3 +# -*- coding: utf-8 -*- +# @Author: Hui +# @Desc: { redis client } +# @Date: 2022/11/28 10:12 +import json +from datetime import timedelta +from enum import Enum +from typing import Awaitable, Callable, Optional, Union + +from redis import asyncio as aioredis + +from metagpt.config import CONFIG +from metagpt.logs import logger + + +class RedisTypeEnum(Enum): + """Redis 数据类型""" + + String = "String" + List = "List" + Hash = "Hash" + Set = "Set" + ZSet = "ZSet" + + +def make_url( + dialect: str, + *, + user: Optional[str] = None, + password: Optional[str] = None, + host: Optional[str] = None, + port: Optional[Union[str, int]] = None, + name: Optional[Union[str, int]] = None, +) -> str: + url_parts = [f"{dialect}://"] + if user or password: + if user: + url_parts.append(user) + if password: + url_parts.append(f":{password}") + url_parts.append("@") + + if not host and not dialect.startswith("sqlite"): + host = "127.0.0.1" + + if host: + url_parts.append(f"{host}") + if port: + url_parts.append(f":{port}") + + # 比如redis可能传入0 + if name is not None: + url_parts.append(f"/{name}") + return "".join(url_parts) + + +class RedisAsyncClient(aioredis.Redis): + """异步的客户端 + 例子:: + + rdb = RedisAsyncClient() + print(rdb.url) + + Args: + host: 服务器地址 + port: 服务器端口 + user: 用户名 + db: 数据库 + password: 密码 + decode_responses: 字符串输入被编码成utf8存储在Redis里了，而取出来的时候还是被编码后的bytes，需要显示的decode才能变成字符串 + health_check_interval: 定时检测连接，防止出现ConnectionErrors (104, Connection reset by peer) + """ + + def __init__( + self, + host: str = "localhost", + port: int = 6379, + db: int = 0, + password: str = None, + decode_responses=True, + health_check_interval=10, + socket_connect_timeout=5, + retry_on_timeout=True, + socket_keepalive=True, + **kwargs, + ): + super().__init__( + host=host, + port=port, + db=db, + password=password, + decode_responses=decode_responses, + health_check_interval=health_check_interval, + socket_connect_timeout=socket_connect_timeout, + retry_on_timeout=retry_on_timeout, + socket_keepalive=socket_keepalive, + **kwargs, + ) + self.url = make_url("redis", host=host, port=port, name=db, password=password) + + +class RedisCacheInfo(object): + """统一缓存信息类""" + + def __init__(self, key, timeout: Union[int, timedelta] = timedelta(seconds=60), data_type=RedisTypeEnum.String): + """ + 缓存信息类初始化 + Args: + key: 缓存的key + timeout: 缓存过期时间, 单位秒 + data_type: 缓存采用的数据结构 (不传并不影响，用于标记业务采用的是什么数据结构) + """ + self.key = key + self.timeout = timeout + self.data_type = data_type + + def __str__(self): + return f"cache key {self.key} timeout {self.timeout}s" + + +class RedisManager: + client: RedisAsyncClient = None + + @classmethod + def init_redis_conn(cls, host, port, password, db): + """初始化redis 连接""" + if cls.client is None: + cls.client = RedisAsyncClient(host=host, port=port, password=password, db=db) + + @classmethod + async def set_with_cache_info(cls, redis_cache_info: RedisCacheInfo, value): + """ + 根据 RedisCacheInfo 设置 Redis 缓存 + :param redis_cache_info: RedisCacheInfo缓存信息对象 + :param value: 缓存的值 + :return: + """ + await cls.client.setex(redis_cache_info.key, redis_cache_info.timeout, value) + + @classmethod + async def get_with_cache_info(cls, redis_cache_info: RedisCacheInfo): + """ + 根据 RedisCacheInfo 获取 Redis 缓存 + :param redis_cache_info: RedisCacheInfo 缓存信息对象 + :return: + """ + cache_info = await cls.client.get(redis_cache_info.key) + return cache_info + + @classmethod + async def del_with_cache_info(cls, redis_cache_info: RedisCacheInfo): + """ + 根据 RedisCacheInfo 删除 Redis 缓存 + :param redis_cache_info: RedisCacheInfo缓存信息对象 + :return: + """ + await cls.client.delete(redis_cache_info.key) + + @staticmethod + async def get_or_set_cache(cache_info: RedisCacheInfo, fetch_data_func: Callable[[], Awaitable[dict]]) -> dict: + """ + 获取缓存数据，如果缓存不存在，则从提供的函数中获取并设置缓存 + 当前版本仅支持 json 形式的 string 格式数据 + """ + + serialized_data = await RedisManager.get_with_cache_info(cache_info) + + if serialized_data: + return json.loads(serialized_data) + + data = await fetch_data_func() + try: + serialized_data = json.dumps(data) + await RedisManager.set_with_cache_info(cache_info, serialized_data) + except Exception as e: + logger.warning(f"数据 {data} 通过 json 进行序列化缓存失败:{e}") + + return data + + @classmethod + def is_valid(cls): + return cls.client is not None + + +class Redis: + def __init__(self): + self._config = CONFIG.REDIS + if not self._config: + return + try: + host = self._config["host"] + port = int(self._config["port"]) + pwd = self._config["password"] + db = int(self._config["db"]) + RedisManager.init_redis_conn(host=host, port=port, password=pwd, db=db) + except Exception as e: + logger.warning(f"Redis initialization has failed:{e}") diff --git a/requirements.txt b/requirements.txt index 5daf710c7..588b29e0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,4 +41,5 @@ qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 azure-cognitiveservices-speech==1.31.0 -aioboto3~=11.3.0 \ No newline at end of file +aioboto3~=11.3.0 +redis==4.3.5 \ No newline at end of file From 0ffd3db9473eda5e2172e8bc826638feddb987cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 13:21:29 +0800 Subject: [PATCH 204/398] feat: +redis --- metagpt/const.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/const.py b/metagpt/const.py index fbc2c928a..e9fa118d7 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -57,3 +57,6 @@ METAGPT_API_VERSION = "METAGPT_API_VERSION" # format BASE64_FORMAT = "base64" + +# REDIS +REDIS_KEY = "REDIS_KEY" From cce76df319ed5174d8a1aca88d498354856b741f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:03:20 +0800 Subject: [PATCH 205/398] feat: +redis --- metagpt/memory/brain_memory.py | 21 +++++++++++++++++++++ metagpt/utils/redis.py | 16 ++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 275cd14df..619a9e1f3 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -7,12 +7,14 @@ @Desc : Support memory for multiple tasks and multiple mainlines. """ import hashlib +import json from enum import Enum from typing import Dict, List import pydantic from metagpt import Message +from metagpt.utils.redis import Redis class MessageType(Enum): @@ -32,6 +34,7 @@ class BrainMemory(pydantic.BaseModel): # it indicates that the text has already been incorporated into the `history summary`. historical_summary_fingerprint: List[str] = [] historical_summary: str = "" + last_history_id: str = "" def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -88,3 +91,21 @@ class BrainMemory(pydantic.BaseModel): @staticmethod def get_md5(text: str) -> str: return hashlib.md5(text.encode()).hexdigest() + + @staticmethod + async def loads(redis_key: str) -> "BrainMemory": + redis = Redis() + if not redis.is_valid() or not redis_key: + return False + v = await redis.get(key=redis_key) + if not v: + data = json.loads(v) + return BrainMemory(**data) + return None + + async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60): + redis = Redis() + if not redis.is_valid() or not redis_key: + return False + v = self.json() + await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index f2ae3222a..ce9d1bc8e 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -196,3 +196,19 @@ class Redis: RedisManager.init_redis_conn(host=host, port=port, password=pwd, db=db) except Exception as e: logger.warning(f"Redis initialization has failed:{e}") + + def is_valid(self): + return RedisManager.is_valid() + + async def get(self, key: str) -> str: + if not self.is_valid() or not key: + return None + v = await RedisManager.get_with_cache_info(redis_cache_info=RedisCacheInfo(key=key)) + return v + + async def set(self, key: str, data: str, timeout_sec: int): + if not self.is_valid() or not key: + return + await RedisManager.set_with_cache_info( + redis_cache_info=RedisCacheInfo(key=key, timeout=timeout_sec), value=data + ) From 41e90b4f483da8d734fb7497975e499330f46e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:12:17 +0800 Subject: [PATCH 206/398] feat: +redis --- metagpt/memory/brain_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 619a9e1f3..baad76562 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -96,12 +96,12 @@ class BrainMemory(pydantic.BaseModel): async def loads(redis_key: str) -> "BrainMemory": redis = Redis() if not redis.is_valid() or not redis_key: - return False + return BrainMemory() v = await redis.get(key=redis_key) if not v: data = json.loads(v) return BrainMemory(**data) - return None + return BrainMemory() async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60): redis = Redis() From 26c4ed6e2245ecd19423cadc0faf697241170528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:38:35 +0800 Subject: [PATCH 207/398] feat: +code --- metagpt/memory/brain_memory.py | 8 ++++---- metagpt/utils/redis.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index baad76562..3b27c2a94 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -93,8 +93,8 @@ class BrainMemory(pydantic.BaseModel): return hashlib.md5(text.encode()).hexdigest() @staticmethod - async def loads(redis_key: str) -> "BrainMemory": - redis = Redis() + async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": + redis = Redis(conf=redis_conf) if not redis.is_valid() or not redis_key: return BrainMemory() v = await redis.get(key=redis_key) @@ -103,8 +103,8 @@ class BrainMemory(pydantic.BaseModel): return BrainMemory(**data) return BrainMemory() - async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60): - redis = Redis() + async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60, redis_conf: Dict = None): + redis = Redis(conf=redis_conf) if not redis.is_valid() or not redis_key: return False v = self.json() diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index ce9d1bc8e..7d1d88fbd 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -6,7 +6,7 @@ import json from datetime import timedelta from enum import Enum -from typing import Awaitable, Callable, Optional, Union +from typing import Awaitable, Callable, Dict, Optional, Union from redis import asyncio as aioredis @@ -184,8 +184,8 @@ class RedisManager: class Redis: - def __init__(self): - self._config = CONFIG.REDIS + def __init__(self, conf: Dict = None): + self._config = conf or CONFIG.REDIS if not self._config: return try: From d6130c2d99361d02c7a68cf9384d7ae3660f8d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:50:54 +0800 Subject: [PATCH 208/398] feat: +to_redis_key --- metagpt/memory/brain_memory.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 3b27c2a94..faf7693ad 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -109,3 +109,7 @@ class BrainMemory(pydantic.BaseModel): return False v = self.json() await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) + + @staticmethod + def to_redis_key(prefix: str, user_id: str, chat_id: str): + return f"{prefix}:{chat_id}:{user_id}" From 0e717a0537c854b7fdd7674c4a7326898e33092f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:54:40 +0800 Subject: [PATCH 209/398] feat: +to_redis_key --- config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.yaml b/config/config.yaml index 765a74b8a..5c8dea03e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -90,4 +90,4 @@ REDIS: host: "YOUR_REDIS_HOST" port: YOUR_REDIS_PORT, int password: "YOUR_REDIS_PASSWORD" - db: YOUR_REDIS_DB_INDEX, int \ No newline at end of file + db: "YOUR_REDIS_DB_INDEX, str, 0-based" \ No newline at end of file From 308f83c82c4442d42613d642c5080a6d07a052a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:55:26 +0800 Subject: [PATCH 210/398] feat: +to_redis_key --- metagpt/utils/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index 7d1d88fbd..b94eee8e2 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -192,7 +192,7 @@ class Redis: host = self._config["host"] port = int(self._config["port"]) pwd = self._config["password"] - db = int(self._config["db"]) + db = self._config["db"] RedisManager.init_redis_conn(host=host, port=port, password=pwd, db=db) except Exception as e: logger.warning(f"Redis initialization has failed:{e}") From 0a494171fa71b789f685c676ea6b7612c4785bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 15:30:48 +0800 Subject: [PATCH 211/398] fixbug: prerequisite --- metagpt/roles/role.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 2f0f713f8..b1ace19fa 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -97,8 +97,9 @@ class RoleContext(BaseModel): def prerequisite(self): """Retrieve information with `prerequisite` tag""" if self.memory and hasattr(self.memory, "get_by_tags"): - return self.memory.get_by_tags([MessageTag.Prerequisite.value]) - return "" + vv = self.memory.get_by_tags([MessageTag.Prerequisite.value]) + return vv[-1:] if len(vv) > 1 else vv + return [] class Role: From fb6bb4b69210909dbf842e83f6fd2277bb61990c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 15:39:11 +0800 Subject: [PATCH 212/398] feat: is dirty --- metagpt/memory/brain_memory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index faf7693ad..8ae7ed959 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -35,14 +35,17 @@ class BrainMemory(pydantic.BaseModel): historical_summary_fingerprint: List[str] = [] historical_summary: str = "" last_history_id: str = "" + is_dirty: bool = False def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) self.history.append(msg.dict()) + self.is_dirty = True def add_answer(self, msg: Message): msg.add_tag(MessageType.Answer.value) self.history.append(msg.dict()) + self.is_dirty = True def get_knowledge(self) -> str: texts = [Message(**m).content for m in self.knowledge] From 82c7fd94fd9ff500eecdc3fcbd805301178feee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 15:50:17 +0800 Subject: [PATCH 213/398] feat: is dirty --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 8ae7ed959..a925474b7 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -101,7 +101,7 @@ class BrainMemory(pydantic.BaseModel): if not redis.is_valid() or not redis_key: return BrainMemory() v = await redis.get(key=redis_key) - if not v: + if v: data = json.loads(v) return BrainMemory(**data) return BrainMemory() From 88419224586ec683db92ae83e4b4aad35bfb5d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 16:05:01 +0800 Subject: [PATCH 214/398] feat: +cache --- metagpt/memory/brain_memory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index a925474b7..8b1b31aae 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -103,7 +103,9 @@ class BrainMemory(pydantic.BaseModel): v = await redis.get(key=redis_key) if v: data = json.loads(v) - return BrainMemory(**data) + bm = BrainMemory(**data) + bm.is_dirty = False + return bm return BrainMemory() async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60, redis_conf: Dict = None): @@ -112,6 +114,7 @@ class BrainMemory(pydantic.BaseModel): return False v = self.json() await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) + self.is_dirty = False @staticmethod def to_redis_key(prefix: str, user_id: str, chat_id: str): From c4a0bd14385f529cb441c7e527baf554d2d74601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 16:40:40 +0800 Subject: [PATCH 215/398] fixbug: tags --- metagpt/schema.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index ce08455fc..987fccef2 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -10,7 +10,7 @@ from __future__ import annotations from dataclasses import dataclass, field from enum import Enum -from typing import Type, TypedDict, Set, Optional, List +from typing import Optional, Set, Type, TypedDict from pydantic import BaseModel @@ -29,9 +29,10 @@ class RawMessage(TypedDict): @dataclass class Message: """list[: ]""" + content: str instruct_content: BaseModel = field(default=None) - role: str = field(default='user') # system / user / assistant + role: str = field(default="user") # system / user / assistant cause_by: Type["Action"] = field(default="") sent_from: str = field(default="") send_to: str = field(default="") @@ -45,10 +46,7 @@ class Message: return self.__str__() def to_dict(self) -> dict: - return { - "role": self.role, - "content": self.content - } + return {"role": self.role, "content": self.content} def add_tag(self, tag): if self.tags is None: @@ -64,7 +62,7 @@ class Message: """Determine whether the message contains tags.""" if not tags or not self.tags: return False - intersection = set(tags) & self.tags + intersection = set(tags) & set(self.tags) return len(intersection) > 0 def is_contain(self, tag): @@ -76,7 +74,7 @@ class Message: "instruct_content": self.instruct_content, "sent_from": self.sent_from, "send_to": self.send_to, - "tags": self.tags + "tags": self.tags, } m = {"content": self.content} @@ -89,39 +87,39 @@ class Message: @dataclass class UserMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ def __init__(self, content: str): - super().__init__(content, 'user') + super().__init__(content, "user") @dataclass class SystemMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ def __init__(self, content: str): - super().__init__(content, 'system') + super().__init__(content, "system") @dataclass class AIMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ def __init__(self, content: str): - super().__init__(content, 'assistant') + super().__init__(content, "assistant") -if __name__ == '__main__': - test_content = 'test_message' +if __name__ == "__main__": + test_content = "test_message" msgs = [ UserMessage(test_content), SystemMessage(test_content), AIMessage(test_content), - Message(test_content, role='QA') + Message(test_content, role="QA"), ] logger.info(msgs) From 230239b3e7fed3dabc21a9cf13568fde946cc1b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 16:46:11 +0800 Subject: [PATCH 216/398] feat: +cache --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 8b1b31aae..e487a696d 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -55,7 +55,7 @@ class BrainMemory(pydantic.BaseModel): def history_text(self): if len(self.history) == 0: return "" - texts = [] + texts = [self.historical_summary] if self.historical_summary else [] for m in self.history[:-1]: if isinstance(m, Dict): t = Message(**m).content From 9220b131a433c8bf1f08a45053779832a7c275f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 17:08:19 +0800 Subject: [PATCH 217/398] feat: +cache --- metagpt/memory/brain_memory.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index e487a696d..ed2955902 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -67,21 +67,6 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) - def move_to_solution(self, history_summary): - """Put it in the solution queue for future long-term retrieval. - This functionality hasn't been added yet, so use the history summary as a temporary substitute for now.""" - pass - # if len(self.history) < 2: - # return - # msgs = self.history[:-1] - # self.solution.extend(msgs) - # if not Message(**self.history[-1]).is_contain(MessageType.Talk.value): - # self.solution.append(self.history[-1]) - # self.history = [] - # else: - # self.history = self.history[-1:] - # self.history.insert(0, Message(content="RESOLVED: " + history_summary)) - @property def last_talk(self): if len(self.history) == 0: @@ -119,3 +104,12 @@ class BrainMemory(pydantic.BaseModel): @staticmethod def to_redis_key(prefix: str, user_id: str, chat_id: str): return f"{prefix}:{chat_id}:{user_id}" + + async def set_history_summary(self, history_summary, redis_key, redis_conf): + if self.historical_summary == history_summary: + return + + self.historical_summary = history_summary + self.history = [] + await self.dumps(redis_key=redis_key, redis_conf=redis_conf) + self.is_dirty = False From f69f37bb0376b25f0d52eee2a68b72deac83391f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 17:09:02 +0800 Subject: [PATCH 218/398] feat: +cache --- metagpt/roles/assistant.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 0bce4a3f9..9c80593f6 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -126,11 +126,13 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_summary(history_text, max_words=500) + await self.memory.set_history_summary( + history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS + ) if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk - self.memory.move_to_solution(history_summary) # Promptly clear memory after the issue is resolved. return last_talk @staticmethod From 32c604a002e78e924d43a732e4b4bd7e3bce1faf Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Mon, 4 Sep 2023 17:21:21 +0800 Subject: [PATCH 219/398] add llm.aask generator --- metagpt/provider/base_gpt_api.py | 4 ++-- metagpt/provider/openai_api.py | 34 +++++++++++++++++--------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index af0cf2ec0..7351e6916 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -38,13 +38,13 @@ class BaseGPTAPI(BaseChatbot): rsp = self.completion(message) return self.get_choice_text(rsp) - async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: + async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, generator: bool = False) -> str: if system_msgs: message = self._system_msgs(system_msgs) + [self._user_msg(msg)] else: message = [self._default_system_msg(), self._user_msg(msg)] try: - rsp = await self.acompletion_text(message, stream=True) + rsp = await self.acompletion_text(message, stream=True, generator=generator) except Exception as e: logger.exception(f"{e}") logger.info(f"ask:{msg}, error:{e}") diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 5c11ed7a6..d0dd5b9d8 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -87,22 +87,11 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): response = await self.async_retry_call( openai.ChatCompletion.acreate, **self._cons_kwargs(messages), stream=True ) - # create variables to collect the stream of chunks - collected_chunks = [] - collected_messages = [] # iterate through the stream of events async for chunk in response: - collected_chunks.append(chunk) # save the event response chunk_message = chunk["choices"][0]["delta"] # extract the message - collected_messages.append(chunk_message) # save the message if "content" in chunk_message: - print(chunk_message["content"], end="") - print() - - full_reply_content = "".join([m.get("content", "") for m in collected_messages]) - usage = self._calc_usage(messages, full_reply_content) - self._update_costs(usage) - return full_reply_content + yield chunk_message["content"] def _cons_kwargs(self, messages: list[dict]) -> dict: if CONFIG.openai_api_type == "azure": @@ -157,10 +146,23 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream=False) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, generator: bool = False) -> str: """when streaming, print each token in place.""" if stream: - return await self._achat_completion_stream(messages) + resp = self._achat_completion_stream(messages) + if generator: + return resp + + collected_messages = [] + async for i in resp: + print(i, end="") + collected_messages.append(i) + + full_reply_content = "".join(collected_messages) + usage = self._calc_usage(messages, full_reply_content) + self._update_costs(usage) + return full_reply_content + rsp = await self._achat_completion(messages) return self.get_choice_text(rsp) @@ -226,13 +228,13 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): max_count = 100 while max_count > 0: if len(text) < max_token_count: - return await self._get_summary(text=text, max_words=max_words,keep_language=keep_language) + return await self._get_summary(text=text, max_words=max_words, keep_language=keep_language) padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) summaries = [] for ws in text_windows: - response = await self._get_summary(text=ws, max_words=max_words,keep_language=keep_language) + response = await self._get_summary(text=ws, max_words=max_words, keep_language=keep_language) summaries.append(response) if len(summaries) == 1: return summaries[0] From ec8e455a59ff1d669ae7071dd8129ddef0abf45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 17:47:33 +0800 Subject: [PATCH 220/398] feat: +cache --- metagpt/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 987fccef2..8f8e4030f 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -37,6 +37,7 @@ class Message: sent_from: str = field(default="") send_to: str = field(default="") tags: Optional[Set] = field(default=None) + id: str = None def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) From ebe5217f701157b1fba5e23effc194c6d3ce8560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 17:58:40 +0800 Subject: [PATCH 221/398] feat: +cache --- metagpt/memory/brain_memory.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index ed2955902..8443d69d9 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -113,3 +113,10 @@ class BrainMemory(pydantic.BaseModel): self.history = [] await self.dumps(redis_key=redis_key, redis_conf=redis_conf) self.is_dirty = False + + def add_history(self, msg: Message): + if msg.id: + if int(msg.id) < int(self.last_history_id): + return + self.history.append(msg.dict()) + self.is_dirty = True From b5ea3c692f5988e2974c897cd21344ca40920e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 18:07:03 +0800 Subject: [PATCH 222/398] feat: +cache --- metagpt/memory/brain_memory.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 8443d69d9..027297eb8 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -120,3 +120,9 @@ class BrainMemory(pydantic.BaseModel): return self.history.append(msg.dict()) self.is_dirty = True + + def exists(self, text) -> bool: + for m in reversed(self.history): + if m.get("content") == text: + return True + return False From 4d9cfe6f439387ef783b3fe9b38edc4e3efe250d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 18:57:16 +0800 Subject: [PATCH 223/398] feat: +cache --- metagpt/memory/brain_memory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 027297eb8..2ea8ac209 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -14,6 +14,7 @@ from typing import Dict, List import pydantic from metagpt import Message +from metagpt.logs import logger from metagpt.utils.redis import Redis @@ -86,6 +87,7 @@ class BrainMemory(pydantic.BaseModel): if not redis.is_valid() or not redis_key: return BrainMemory() v = await redis.get(key=redis_key) + logger.info(f"REDIS GET {redis_key} {v}") if v: data = json.loads(v) bm = BrainMemory(**data) @@ -99,6 +101,7 @@ class BrainMemory(pydantic.BaseModel): return False v = self.json() await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) + logger.info(f"REDIS SET {redis_key} {v}") self.is_dirty = False @staticmethod From 26e35d799db5ff32c4a935909a15eb71b763e51f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 19:02:54 +0800 Subject: [PATCH 224/398] feat: +cache --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 2ea8ac209..50c414c97 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -54,7 +54,7 @@ class BrainMemory(pydantic.BaseModel): @property def history_text(self): - if len(self.history) == 0: + if len(self.history) == 0 and not self.historical_summary: return "" texts = [self.historical_summary] if self.historical_summary else [] for m in self.history[:-1]: From 207ab965451a99689da72ad86fe361781c395300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 19:36:51 +0800 Subject: [PATCH 225/398] feat: +cache --- metagpt/memory/brain_memory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 50c414c97..6f4c3ec75 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -110,6 +110,9 @@ class BrainMemory(pydantic.BaseModel): async def set_history_summary(self, history_summary, redis_key, redis_conf): if self.historical_summary == history_summary: + if self.is_dirty: + await self.dumps(redis_key=redis_key, redis_conf=redis_conf) + self.is_dirty = False return self.historical_summary = history_summary From 63805c87f9c87de9b3823941a095b3f46b2f906b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 19:50:42 +0800 Subject: [PATCH 226/398] feat: +cache --- metagpt/memory/brain_memory.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 6f4c3ec75..a974d95f6 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -31,9 +31,6 @@ class BrainMemory(pydantic.BaseModel): stack: List[Dict] = [] solution: List[Dict] = [] knowledge: List[Dict] = [] - # If the fingerprint of the history text is found in the `historical_summary_fingerprint`, - # it indicates that the text has already been incorporated into the `history summary`. - historical_summary_fingerprint: List[str] = [] historical_summary: str = "" last_history_id: str = "" is_dirty: bool = False From 4dd9f7743f0d8dd3d4b2deb53b7a4d5e56d8bedc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 19:53:35 +0800 Subject: [PATCH 227/398] feat: +cache --- metagpt/memory/brain_memory.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index a974d95f6..dedea3b41 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -5,8 +5,8 @@ @Author : mashenquan @File : brain_memory.py @Desc : Support memory for multiple tasks and multiple mainlines. +@Modified By: mashenquan, 2023/9/4. + redis memory cache. """ -import hashlib import json from enum import Enum from typing import Dict, List @@ -74,10 +74,6 @@ class BrainMemory(pydantic.BaseModel): return None return last_msg.content - @staticmethod - def get_md5(text: str) -> str: - return hashlib.md5(text.encode()).hexdigest() - @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": redis = Redis(conf=redis_conf) From 7cb19c943c39279c3d811bb6525de0862e100e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 21:21:46 +0800 Subject: [PATCH 228/398] fixbug: int --- metagpt/memory/brain_memory.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index dedea3b41..22af67236 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -115,8 +115,9 @@ class BrainMemory(pydantic.BaseModel): def add_history(self, msg: Message): if msg.id: - if int(msg.id) < int(self.last_history_id): + if self.to_int(msg.id, 0) < self.to_int(self.last_history_id, -1): return + self.last_history_id = str(self.to_int(msg.id, 0)) self.history.append(msg.dict()) self.is_dirty = True @@ -125,3 +126,10 @@ class BrainMemory(pydantic.BaseModel): if m.get("content") == text: return True return False + + @staticmethod + def to_int(v, default_value): + try: + return int(v) + except: + return default_value From 107ddbe308901713aa18d767c920e41e5e473e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 22:38:58 +0800 Subject: [PATCH 229/398] refactor: talk prompt --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 83504b62d..a4cd78121 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -26,7 +26,7 @@ class TalkAction(Action): self._rsp = None @property - def prompt(self): + def prompt_old(self): prompt = "" if CONFIG.agent_description: prompt = ( @@ -46,7 +46,7 @@ class TalkAction(Action): return prompt @property - def formation_prompt(self): + def prompt(self): kvs = { "{role}": CONFIG.agent_description or "", "{history}": self._history_summary or "", From 972337776de1d00f8997cdd73ab2c24df982cd96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 22:56:16 +0800 Subject: [PATCH 230/398] refactor: talk prompt --- metagpt/utils/redis.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index b94eee8e2..48a18e7c9 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -4,6 +4,7 @@ # @Desc: { redis client } # @Date: 2022/11/28 10:12 import json +import traceback from datetime import timedelta from enum import Enum from typing import Awaitable, Callable, Dict, Optional, Union @@ -203,12 +204,19 @@ class Redis: async def get(self, key: str) -> str: if not self.is_valid() or not key: return None - v = await RedisManager.get_with_cache_info(redis_cache_info=RedisCacheInfo(key=key)) - return v + try: + v = await RedisManager.get_with_cache_info(redis_cache_info=RedisCacheInfo(key=key)) + return v + except Exception as e: + logger.exception(f"{e}, stack:{traceback.format_exc()}") + return None async def set(self, key: str, data: str, timeout_sec: int): if not self.is_valid() or not key: return - await RedisManager.set_with_cache_info( - redis_cache_info=RedisCacheInfo(key=key, timeout=timeout_sec), value=data - ) + try: + await RedisManager.set_with_cache_info( + redis_cache_info=RedisCacheInfo(key=key, timeout=timeout_sec), value=data + ) + except Exception as e: + logger.exception(f"{e}, stack:{traceback.format_exc()}") From 557e82d8ef050466b3e465c17ccee695ff2d08ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 23:07:31 +0800 Subject: [PATCH 231/398] refactor: talk prompt --- metagpt/actions/talk_action.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index a4cd78121..54c004602 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -57,6 +57,7 @@ class TalkAction(Action): prompt = TalkAction.__FORMATION_LOOSE__ for k, v in kvs.items(): prompt = prompt.replace(k, v) + logger.info(f"PROMPT: {prompt}") return prompt async def run(self, *args, **kwargs) -> ActionOutput: From 06c24c0eb4fc604ad1eff4980736e8ccc1b221ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 23:26:25 +0800 Subject: [PATCH 232/398] refactor: talk prompt --- metagpt/memory/brain_memory.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 22af67236..0c1ae024d 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -117,7 +117,6 @@ class BrainMemory(pydantic.BaseModel): if msg.id: if self.to_int(msg.id, 0) < self.to_int(self.last_history_id, -1): return - self.last_history_id = str(self.to_int(msg.id, 0)) self.history.append(msg.dict()) self.is_dirty = True From d79a0638f2bc8f16b038327c39894add919669b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 23:34:54 +0800 Subject: [PATCH 233/398] fixbug: last_talk --- metagpt/memory/brain_memory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 0c1ae024d..e2d9ad5ff 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -34,6 +34,7 @@ class BrainMemory(pydantic.BaseModel): historical_summary: str = "" last_history_id: str = "" is_dirty: bool = False + last_talk: str = "" def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) From b0966ca54133f9667a4ff173d17c5051b7993542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 23:38:43 +0800 Subject: [PATCH 234/398] fixbug: last_talk --- metagpt/memory/brain_memory.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index e2d9ad5ff..60c563ed4 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -66,15 +66,6 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) - @property - def last_talk(self): - if len(self.history) == 0: - return None - last_msg = Message(**self.history[-1]) - if not last_msg.is_contain(MessageType.Talk.value): - return None - return last_msg.content - @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": redis = Redis(conf=redis_conf) From 8075154a8db000c52f1db270b1907cb3f79d72f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 23:46:12 +0800 Subject: [PATCH 235/398] fixbug: last_talk --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 9c80593f6..018a1fb01 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -125,7 +125,7 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_summary(history_text, max_words=500) + history_summary = await self._llm.get_summary(history_text, max_words=800) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS ) From 327e5fc9871cff1693fef512bd9a09645c69a7c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:02:21 +0800 Subject: [PATCH 236/398] fixbug: last_talk --- metagpt/provider/openai_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index d0dd5b9d8..9f65dd905 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -226,15 +226,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def get_summary(self, text: str, max_words=200, keep_language: bool = False): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 + text_length = len(text) while max_count > 0: - if len(text) < max_token_count: + if text_length < max_token_count: return await self._get_summary(text=text, max_words=max_words, keep_language=keep_language) padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) + part_max_words = int(max_words / len(text_windows)) + 1 summaries = [] for ws in text_windows: - response = await self._get_summary(text=ws, max_words=max_words, keep_language=keep_language) + response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) summaries.append(response) if len(summaries) == 1: return summaries[0] From 1e7f0569183ba26b20e4c6060df0a047160d3e9c Mon Sep 17 00:00:00 2001 From: zhanglei Date: Tue, 5 Sep 2023 00:12:29 +0800 Subject: [PATCH 237/398] =?UTF-8?q?update:=20=E4=BC=98=E5=8C=96=E8=AE=B0?= =?UTF-8?q?=E5=BF=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/memory/brain_memory.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 22af67236..586285e4f 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -34,6 +34,7 @@ class BrainMemory(pydantic.BaseModel): historical_summary: str = "" last_history_id: str = "" is_dirty: bool = False + last_talk: str = "" def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -65,15 +66,6 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) - @property - def last_talk(self): - if len(self.history) == 0: - return None - last_msg = Message(**self.history[-1]) - if not last_msg.is_contain(MessageType.Talk.value): - return None - return last_msg.content - @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": redis = Redis(conf=redis_conf) From bc52a674e773416e6d4616ad2b2d13b6d27f404c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:20:34 +0800 Subject: [PATCH 238/398] fixbug: last_talk --- metagpt/provider/openai_api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 9f65dd905..2539c5b70 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,10 +233,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) - part_max_words = int(max_words / len(text_windows)) + 1 summaries = [] for ws in text_windows: - response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) + response = await self._get_summary(text=ws, max_words=200, keep_language=keep_language) summaries.append(response) if len(summaries) == 1: return summaries[0] From dfc189510eb51928b732ebbcdfaa143a94252136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:23:36 +0800 Subject: [PATCH 239/398] fixbug: last_talk --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 2539c5b70..9406346ac 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,9 +233,10 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) + part_max_words = min(int(max_words / len(text_windows)) + 1, 200) summaries = [] for ws in text_windows: - response = await self._get_summary(text=ws, max_words=200, keep_language=keep_language) + response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) summaries.append(response) if len(summaries) == 1: return summaries[0] From 8e1034afffcd3fbde4754ed64e49187f27beb672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:27:37 +0800 Subject: [PATCH 240/398] fixbug: last_talk --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 9406346ac..157c353a8 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,7 +233,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) - part_max_words = min(int(max_words / len(text_windows)) + 1, 200) + part_max_words = min(int(max_words / len(text_windows)) + 1, 100) summaries = [] for ws in text_windows: response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) From 998411a125e45a6265af7054081a2885e8a8d479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:35:03 +0800 Subject: [PATCH 241/398] fixbug: last_talk --- metagpt/provider/openai_api.py | 2 +- metagpt/roles/assistant.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 157c353a8..2722491d0 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,7 +233,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) - part_max_words = min(int(max_words / len(text_windows)) + 1, 100) + part_max_words = min(int(max_words / len(text_windows)) + 1, 60) summaries = [] for ws in text_windows: response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 018a1fb01..4b2bfdab5 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -125,7 +125,7 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_summary(history_text, max_words=800) + history_summary = await self._llm.get_summary(history_text, max_words=800, keep_language=True) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS ) From 18a65470f031c65de06834c0651dd3574cda1c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:37:32 +0800 Subject: [PATCH 242/398] fixbug: last_talk --- metagpt/memory/brain_memory.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 60c563ed4..92a71f69a 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -124,3 +124,8 @@ class BrainMemory(pydantic.BaseModel): return int(v) except: return default_value + + def pop_last_talk(self): + v = self.last_talk + self.last_talk = "" + return v From 22dbe3b224e8f7f0a8eedef942068deab4980ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:38:31 +0800 Subject: [PATCH 243/398] fixbug: last_talk --- metagpt/memory/brain_memory.py | 4 ++-- metagpt/roles/assistant.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 92a71f69a..2195da566 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -34,7 +34,7 @@ class BrainMemory(pydantic.BaseModel): historical_summary: str = "" last_history_id: str = "" is_dirty: bool = False - last_talk: str = "" + last_talk: str = None def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -127,5 +127,5 @@ class BrainMemory(pydantic.BaseModel): def pop_last_talk(self): v = self.last_talk - self.last_talk = "" + self.last_talk = None return v diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 4b2bfdab5..87127cbab 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -120,7 +120,7 @@ class Assistant(Role): async def refine_memory(self) -> str: history_text = self.memory.history_text - last_talk = self.memory.last_talk + last_talk = self.memory.pop_last_talk() if last_talk is None: # No user feedback, unsure if past conversation is finished. return None if history_text == "": From 9b2d6e492241493d3c5d4ef2c71152afc652acfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:41:47 +0800 Subject: [PATCH 244/398] fixbug: last_talk --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 2722491d0..bf2ca7f14 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,7 +233,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) - part_max_words = min(int(max_words / len(text_windows)) + 1, 60) + part_max_words = min(int(max_words / len(text_windows)) + 1, 100) summaries = [] for ws in text_windows: response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) @@ -243,6 +243,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # Merged and retry text = "\n".join(summaries) + text_length = len(text) max_count -= 1 # safeguard raise openai.error.InvalidRequestError("text too long") From 845cc8fbfd99626f1a6c740450382f0f3d49b2da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 01:11:21 +0800 Subject: [PATCH 245/398] fixbug: last_talk --- metagpt/actions/talk_action.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 54c004602..1c1a4e86d 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -40,7 +40,7 @@ class TalkAction(Action): prompt += "According to the historical conversation above, " language = CONFIG.language or DEFAULT_LANGUAGE prompt += ( - f"Answer the following questions in {language}, and the answers must follow the Markdown format.\n " + f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n " f"{self._talk}" ) return prompt @@ -89,7 +89,7 @@ Statement: Your responses should align with the role-play agreement, maintaining [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. -Statement: Answer the following questions in {language}, and the answers must follow the Markdown format +Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" , "[ASK_END]" @@ -117,7 +117,7 @@ Statement: Your responses should maintaining the character's persona and habits. [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. -Statement: Answer the following questions in {language}, and the answers must follow the Markdown format +Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" , "[ASK_END]" From bcb6c7903e34c78baa9d2cb28a9555dea28ddfb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 01:48:09 +0800 Subject: [PATCH 246/398] fixbug: last_talk --- metagpt/actions/talk_action.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 1c1a4e86d..d6d18140a 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -90,8 +90,8 @@ Statement: Your responses should align with the role-play agreement, maintaining Statement: If the information is insufficient, you can search in the historical conversation or knowledge. Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format - , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" - , "[ASK_END]" + , strictly excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" + , "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] {ask} @@ -118,8 +118,8 @@ Statement: Your responses should maintaining the character's persona and habits. Statement: If the information is insufficient, you can search in the historical conversation or knowledge. Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format - , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" - , "[ASK_END]" + , strictly excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" + , "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] {ask} From 5a6d5cc37dadb439a39bdccc3bfc20fac14414e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 10:41:25 +0800 Subject: [PATCH 247/398] fixbug: language professional --- metagpt/actions/talk_action.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index d6d18140a..cc30837b9 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -117,9 +117,9 @@ Statement: Your responses should maintaining the character's persona and habits. [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. -Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format - , strictly excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" - , "[ASK_BEGIN]", "[ASK_END]" in responses. +Statement: Unless you are a language professional, answer the following questions strictly in {language} +, and the answers must follow the Markdown format, strictly excluding any tag likes "[HISTORY_BEGIN]" +, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] {ask} From f54c507f06e6086720f163c2872c359a2ec28897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 11:11:22 +0800 Subject: [PATCH 248/398] refactor: prompt --- metagpt/actions/talk_action.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index cc30837b9..ec151718e 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -74,6 +74,10 @@ class TalkAction(Action): "Statement" defines the work detail you need to complete at this stage; "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; "Constraint" defines the conditions that your responses must comply with. + “Personality” defines your language style。 + "Command" defines the action to do when command keyword is entered. + "Insight" provides a deeper understanding of the characters' inner traits. + "Initial" defines the initial setup of a character. Capacity and role: {role} Statement: Your responses should align with the role-play agreement, maintaining the @@ -89,9 +93,9 @@ Statement: Your responses should align with the role-play agreement, maintaining [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. -Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format - , strictly excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" - , "[ASK_BEGIN]", "[ASK_END]" in responses. +Statement: Unless you are a language professional, answer the following questions strictly in {language} +, and the answers must follow the Markdown format, strictly excluding any tag likes "[HISTORY_BEGIN]" +, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] {ask} @@ -103,6 +107,10 @@ Statement: Answer the following questions strictly in {language}, and the answer "Statement" defines the work detail you need to complete at this stage; "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; "Constraint" defines the conditions that your responses must comply with. + “Personality” defines your language style。 + "Command" defines the action to do when command keyword is entered. + "Insight" provides a deeper understanding of the characters' inner traits. + "Initial" defines the initial setup of a character. Capacity and role: {role} Statement: Your responses should maintaining the character's persona and habits. When faced with unrelated questions From c2c7f1c96d3494c4fd4cdb2f55e6922935077909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 11:26:52 +0800 Subject: [PATCH 249/398] refactor: prompt --- metagpt/actions/talk_action.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index ec151718e..71ac5360a 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -98,7 +98,11 @@ Statement: Unless you are a language professional, answer the following question , "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] + + {ask} + + [ASK_END]""" __FORMATION_LOOSE__ = """Formation: "Capacity and role" defines the role you are currently playing; @@ -130,5 +134,9 @@ Statement: Unless you are a language professional, answer the following question , "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] + + {ask} + + [ASK_END]""" From 54120e73562ebc8157eaf7f76a1890c958ed4fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 11:27:42 +0800 Subject: [PATCH 250/398] refactor: prompt --- metagpt/actions/talk_action.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 71ac5360a..c314b500d 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -85,11 +85,15 @@ Statement: Your responses should align with the role-play agreement, maintaining your AI nature to preserve the character's image. [HISTORY_BEGIN] + {history} + [HISTORY_END] [KNOWLEDGE_BEGIN] + {knowledge} + [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. @@ -121,11 +125,15 @@ Statement: Your responses should maintaining the character's persona and habits. , playfully decline to answer without revealing your AI nature to preserve the character's image. [HISTORY_BEGIN] + {history} + [HISTORY_END] [KNOWLEDGE_BEGIN] + {knowledge} + [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. From dec135ec833212400ea617d876f2c97ffff77916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 12:26:36 +0800 Subject: [PATCH 251/398] =?UTF-8?q?revert:=20=E6=94=B9=E7=94=A8CONFIG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/document_store/faiss_store.py | 26 ++++++++-------- metagpt/memory/longterm_memory.py | 4 +-- metagpt/memory/memory_storage.py | 27 ++++++++--------- tests/metagpt/memory/test_longterm_memory.py | 32 ++++++++++---------- 4 files changed, 44 insertions(+), 45 deletions(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index fbfcb3086..16c152c1c 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -14,6 +14,7 @@ import faiss from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import FAISS +from metagpt.config import CONFIG from metagpt.const import DATA_PATH from metagpt.document_store.base_store import LocalStore from metagpt.document_store.document import Document @@ -21,7 +22,7 @@ from metagpt.logs import logger class FaissStore(LocalStore): - def __init__(self, raw_data: Path, cache_dir=None, meta_col='source', content_col='output'): + def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output"): self.meta_col = meta_col self.content_col = content_col super().__init__(raw_data, cache_dir) @@ -37,11 +38,12 @@ class FaissStore(LocalStore): store.index = index return store - def _write(self, docs, metadatas, **kwargs): - store = FAISS.from_texts(docs, - OpenAIEmbeddings(openai_api_version="2020-11-07", - openai_api_key=kwargs.get("OPENAI_API_KEY")), - metadatas=metadatas) + def _write(self, docs, metadatas): + store = FAISS.from_texts( + docs, + OpenAIEmbeddings(openai_api_version="2020-11-07", openai_api_key=CONFIG.OPENAI_API_KEY), + metadatas=metadatas, + ) return store def persist(self): @@ -54,7 +56,7 @@ class FaissStore(LocalStore): pickle.dump(store, f) store.index = index - def search(self, query, expand_cols=False, sep='\n', *args, k=5, **kwargs): + def search(self, query, expand_cols=False, sep="\n", *args, k=5, **kwargs): rsp = self.store.similarity_search(query, k=k, **kwargs) logger.debug(rsp) if expand_cols: @@ -82,8 +84,8 @@ class FaissStore(LocalStore): raise NotImplementedError -if __name__ == '__main__': - faiss_store = FaissStore(DATA_PATH / 'qcs/qcs_4w.json') - logger.info(faiss_store.search('油皮洗面奶')) - faiss_store.add([f'油皮洗面奶-{i}' for i in range(3)]) - logger.info(faiss_store.search('油皮洗面奶')) +if __name__ == "__main__": + faiss_store = FaissStore(DATA_PATH / "qcs/qcs_4w.json") + logger.info(faiss_store.search("油皮洗面奶")) + faiss_store.add([f"油皮洗面奶-{i}" for i in range(3)]) + logger.info(faiss_store.search("油皮洗面奶")) diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index 041d335ac..df748037a 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -37,13 +37,13 @@ class LongTermMemory(Memory): self.add_batch(messages) self.msg_from_recover = False - def add(self, message: Message, **kwargs): + def add(self, message: Message): super(LongTermMemory, self).add(message) for action in self.rc.watch: if message.cause_by == action and not self.msg_from_recover: # currently, only add role's watching messages to its memory_storage # and ignore adding messages from recover repeatedly - self.memory_storage.add(message, **kwargs) + self.memory_storage.add(message) def remember(self, observed: list[Message], k=0) -> list[Message]: """ diff --git a/metagpt/memory/memory_storage.py b/metagpt/memory/memory_storage.py index 09cd67410..9afd524f0 100644 --- a/metagpt/memory/memory_storage.py +++ b/metagpt/memory/memory_storage.py @@ -5,16 +5,16 @@ @Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ -from typing import List from pathlib import Path +from typing import List from langchain.vectorstores.faiss import FAISS from metagpt.const import DATA_PATH, MEM_TTL +from metagpt.document_store.faiss_store import FaissStore from metagpt.logs import logger from metagpt.schema import Message -from metagpt.utils.serialize import serialize_message, deserialize_message -from metagpt.document_store.faiss_store import FaissStore +from metagpt.utils.serialize import deserialize_message, serialize_message class MemoryStorage(FaissStore): @@ -37,7 +37,7 @@ class MemoryStorage(FaissStore): def recover_memory(self, role_id: str) -> List[Message]: self.role_id = role_id - self.role_mem_path = Path(DATA_PATH / f'role_mem/{self.role_id}/') + self.role_mem_path = Path(DATA_PATH / f"role_mem/{self.role_id}/") self.role_mem_path.mkdir(parents=True, exist_ok=True) self.store = self._load() @@ -54,23 +54,23 @@ class MemoryStorage(FaissStore): def _get_index_and_store_fname(self): if not self.role_mem_path: - logger.error(f'You should call {self.__class__.__name__}.recover_memory fist when using LongTermMemory') + logger.error(f"You should call {self.__class__.__name__}.recover_memory fist when using LongTermMemory") return None, None - index_fpath = Path(self.role_mem_path / f'{self.role_id}.index') - storage_fpath = Path(self.role_mem_path / f'{self.role_id}.pkl') + index_fpath = Path(self.role_mem_path / f"{self.role_id}.index") + storage_fpath = Path(self.role_mem_path / f"{self.role_id}.pkl") return index_fpath, storage_fpath def persist(self): super(MemoryStorage, self).persist() - logger.debug(f'Agent {self.role_id} persist memory into local') + logger.debug(f"Agent {self.role_id} persist memory into local") - def add(self, message: Message, **kwargs) -> bool: - """ add message into memory storage""" + def add(self, message: Message) -> bool: + """add message into memory storage""" docs = [message.content] metadatas = [{"message_ser": serialize_message(message)}] if not self.store: # init Faiss - self.store = self._write(docs, metadatas, **kwargs) + self.store = self._write(docs, metadatas) self._initialized = True else: self.store.add_texts(texts=docs, metadatas=metadatas) @@ -82,10 +82,7 @@ class MemoryStorage(FaissStore): if not self.store: return [] - resp = self.store.similarity_search_with_score( - query=message.content, - k=k - ) + resp = self.store.similarity_search_with_score(query=message.content, k=k) # filter the result which score is smaller than the threshold filtered_resp = [] for item, score in resp: diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index 457e665fa..b77e9a955 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -4,11 +4,11 @@ @Desc : unittest of `metagpt/memory/longterm_memory.py` @Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ -from metagpt.config import Config -from metagpt.schema import Message from metagpt.actions import BossRequirement -from metagpt.roles.role import RoleContext +from metagpt.config import Config from metagpt.memory import LongTermMemory +from metagpt.roles.role import RoleContext +from metagpt.schema import Message def test_ltm_search(): @@ -17,28 +17,28 @@ def test_ltm_search(): openai_api_key = conf.openai_api_key assert len(openai_api_key) > 20 - role_id = 'UTUserLtm(Product Manager)' - rc = RoleContext(options=conf.runtime_options, watch=[BossRequirement]) + role_id = "UTUserLtm(Product Manager)" + rc = RoleContext(watch=[BossRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) - idea = 'Write a cli snake game' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + idea = "Write a cli snake game" + message = Message(role="BOSS", content=idea, cause_by=BossRequirement) news = ltm.remember([message]) assert len(news) == 1 - ltm.add(message, **conf.runtime_options) + ltm.add(message) - sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_idea = "Write a game of cli snake" + sim_message = Message(role="BOSS", content=sim_idea, cause_by=BossRequirement) news = ltm.remember([sim_message]) assert len(news) == 0 - ltm.add(sim_message, **conf.runtime_options) + ltm.add(sim_message) - new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a 2048 web game" + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement) news = ltm.remember([new_message]) assert len(news) == 1 - ltm.add(new_message, **conf.runtime_options) + ltm.add(new_message) # restore from local index ltm_new = LongTermMemory() @@ -50,8 +50,8 @@ def test_ltm_search(): news = ltm_new.remember([sim_message]) assert len(news) == 0 - new_idea = 'Write a Battle City' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a Battle City" + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement) news = ltm_new.remember([new_message]) assert len(news) == 1 From 53030428c357ceda1ae11f830d850d9ea2e977d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 13:22:57 +0800 Subject: [PATCH 252/398] =?UTF-8?q?revert:=20=E6=94=B9=E7=94=A8CONFIG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/talk_action.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index c314b500d..e7b3d84c8 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -75,7 +75,6 @@ class TalkAction(Action): "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; "Constraint" defines the conditions that your responses must comply with. “Personality” defines your language style。 - "Command" defines the action to do when command keyword is entered. "Insight" provides a deeper understanding of the characters' inner traits. "Initial" defines the initial setup of a character. @@ -116,7 +115,6 @@ Statement: Unless you are a language professional, answer the following question "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; "Constraint" defines the conditions that your responses must comply with. “Personality” defines your language style。 - "Command" defines the action to do when command keyword is entered. "Insight" provides a deeper understanding of the characters' inner traits. "Initial" defines the initial setup of a character. From fa7e16192a76f1dc68374ae6f2767f2150b5a690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 13:28:28 +0800 Subject: [PATCH 253/398] =?UTF-8?q?revert:=20=E6=94=B9=E7=94=A8CONFIG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index e7b3d84c8..55e6e1aaa 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -72,7 +72,7 @@ class TalkAction(Action): "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; "Statement" defines the work detail you need to complete at this stage; - "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; + "[ASK_BEGIN]" and [ASK_END] tags enclose the questions; "Constraint" defines the conditions that your responses must comply with. “Personality” defines your language style。 "Insight" provides a deeper understanding of the characters' inner traits. @@ -112,7 +112,7 @@ Statement: Unless you are a language professional, answer the following question "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; "Statement" defines the work detail you need to complete at this stage; - "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; + "[ASK_BEGIN]" and [ASK_END] tags enclose the questions; "Constraint" defines the conditions that your responses must comply with. “Personality” defines your language style。 "Insight" provides a deeper understanding of the characters' inner traits. From 6b59f28eb35ca7b975c3cfd4bbb38f900ea6bd51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 13:34:57 +0800 Subject: [PATCH 254/398] =?UTF-8?q?revert:=20=E6=94=B9=E7=94=A8CONFIG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 55e6e1aaa..6ec64d7f9 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -74,7 +74,7 @@ class TalkAction(Action): "Statement" defines the work detail you need to complete at this stage; "[ASK_BEGIN]" and [ASK_END] tags enclose the questions; "Constraint" defines the conditions that your responses must comply with. - “Personality” defines your language style。 + "Personality" defines your language style。 "Insight" provides a deeper understanding of the characters' inner traits. "Initial" defines the initial setup of a character. @@ -114,7 +114,7 @@ Statement: Unless you are a language professional, answer the following question "Statement" defines the work detail you need to complete at this stage; "[ASK_BEGIN]" and [ASK_END] tags enclose the questions; "Constraint" defines the conditions that your responses must comply with. - “Personality” defines your language style。 + "Personality" defines your language style。 "Insight" provides a deeper understanding of the characters' inner traits. "Initial" defines the initial setup of a character. From 280fd62c94b8f19da3524dc398cdc879ae9e7456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 14:08:00 +0800 Subject: [PATCH 255/398] revert: faiss store --- metagpt/document_store/faiss_store.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 16c152c1c..46b959d81 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -14,7 +14,6 @@ import faiss from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import FAISS -from metagpt.config import CONFIG from metagpt.const import DATA_PATH from metagpt.document_store.base_store import LocalStore from metagpt.document_store.document import Document @@ -41,7 +40,7 @@ class FaissStore(LocalStore): def _write(self, docs, metadatas): store = FAISS.from_texts( docs, - OpenAIEmbeddings(openai_api_version="2020-11-07", openai_api_key=CONFIG.OPENAI_API_KEY), + OpenAIEmbeddings(openai_api_version="2020-11-07"), metadatas=metadatas, ) return store From b9e3886e3012c8fe7f343d6bd165a861addfc43d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 14:08:29 +0800 Subject: [PATCH 256/398] revert: faiss store --- metagpt/document_store/faiss_store.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 46b959d81..55c07b920 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -38,11 +38,7 @@ class FaissStore(LocalStore): return store def _write(self, docs, metadatas): - store = FAISS.from_texts( - docs, - OpenAIEmbeddings(openai_api_version="2020-11-07"), - metadatas=metadatas, - ) + store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07"), metadatas=metadatas) return store def persist(self): From e7ffd6dbc5ef4cef7036edff6178e2a6db27f450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 14:11:24 +0800 Subject: [PATCH 257/398] revert: faiss store --- metagpt/document_store/faiss_store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 55c07b920..7833bc706 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -4,7 +4,6 @@ @Time : 2023/5/25 10:20 @Author : alexanderwu @File : faiss_store.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import pickle from pathlib import Path From c204ee87071145ed7aa6214d635597eb0255d86d Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Tue, 5 Sep 2023 14:44:51 +0800 Subject: [PATCH 258/398] faiss init with kwargs --- metagpt/document_store/faiss_store.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 7833bc706..be4748b50 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -20,9 +20,10 @@ from metagpt.logs import logger class FaissStore(LocalStore): - def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output"): + def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output", embedding_conf=None): self.meta_col = meta_col self.content_col = content_col + self.embedding_conf = embedding_conf or {} super().__init__(raw_data, cache_dir) def _load(self) -> Optional["FaissStore"]: @@ -37,7 +38,7 @@ class FaissStore(LocalStore): return store def _write(self, docs, metadatas): - store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07"), metadatas=metadatas) + store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07", **self.embedding_conf), metadatas=metadatas) return store def persist(self): From 9779c578fad7c913b38ee97884af15f185f047a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 15:53:07 +0800 Subject: [PATCH 259/398] fixbug: prompt --- metagpt/actions/talk_action.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 6ec64d7f9..558145e0d 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -97,22 +97,17 @@ Statement: Your responses should align with the role-play agreement, maintaining Statement: If the information is insufficient, you can search in the historical conversation or knowledge. Statement: Unless you are a language professional, answer the following questions strictly in {language} -, and the answers must follow the Markdown format, strictly excluding any tag likes "[HISTORY_BEGIN]" -, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. +, and the answers must follow the Markdown format. Strictly excluding any tag likes "[HISTORY_BEGIN]" +, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" in responses. -[ASK_BEGIN] - {ask} - - -[ASK_END]""" +""" __FORMATION_LOOSE__ = """Formation: "Capacity and role" defines the role you are currently playing; "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; "Statement" defines the work detail you need to complete at this stage; - "[ASK_BEGIN]" and [ASK_END] tags enclose the questions; "Constraint" defines the conditions that your responses must comply with. "Personality" defines your language style。 "Insight" provides a deeper understanding of the characters' inner traits. @@ -136,13 +131,9 @@ Statement: Your responses should maintaining the character's persona and habits. Statement: If the information is insufficient, you can search in the historical conversation or knowledge. Statement: Unless you are a language professional, answer the following questions strictly in {language} -, and the answers must follow the Markdown format, strictly excluding any tag likes "[HISTORY_BEGIN]" -, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. - -[ASK_BEGIN] +, and the answers must follow the Markdown format. Strictly excluding any tag likes "[HISTORY_BEGIN]" +, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" in responses. {ask} - - -[ASK_END]""" +""" From 40bbacd25d6af85e9a6810cd1333e05bc5818829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 16:08:31 +0800 Subject: [PATCH 260/398] revert: prompt --- metagpt/actions/talk_action.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 558145e0d..603736bc7 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -26,7 +26,7 @@ class TalkAction(Action): self._rsp = None @property - def prompt_old(self): + def prompt(self): prompt = "" if CONFIG.agent_description: prompt = ( @@ -34,10 +34,11 @@ class TalkAction(Action): f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" ) - prompt += f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" + prompt += f"Knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" prompt += f"{self._history_summary}\n\n" - if self._history_summary != "": - prompt += "According to the historical conversation above, " + prompt += ( + "If the information is insufficient, you can search in the historical conversation or knowledge above." + ) language = CONFIG.language or DEFAULT_LANGUAGE prompt += ( f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n " @@ -46,7 +47,7 @@ class TalkAction(Action): return prompt @property - def prompt(self): + def prompt_bad(self): kvs = { "{role}": CONFIG.agent_description or "", "{history}": self._history_summary or "", From 3f71ebb71ad01531de794f7b2caeefd3ad2ef942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 16:13:28 +0800 Subject: [PATCH 261/398] revert: prompt --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 603736bc7..81caef013 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -37,7 +37,7 @@ class TalkAction(Action): prompt += f"Knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" prompt += f"{self._history_summary}\n\n" prompt += ( - "If the information is insufficient, you can search in the historical conversation or knowledge above." + "If the information is insufficient, you can search in the historical conversation or knowledge above.\n" ) language = CONFIG.language or DEFAULT_LANGUAGE prompt += ( From 5c627df6c47fd8bd9257a4643a4fd0de49d7be82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 17:18:01 +0800 Subject: [PATCH 262/398] feat: +log --- metagpt/actions/talk_action.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 81caef013..4afed8014 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -44,6 +44,7 @@ class TalkAction(Action): f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n " f"{self._talk}" ) + logger.info(f"PROMPT: {prompt}") return prompt @property From c1aa93221086f094e3c661e3ac9f141f0f1b2168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:10:16 +0800 Subject: [PATCH 263/398] feat: +iflytek tts --- .well-known/metagpt_oas3_api.yaml | 57 +++++++++++++++++++++++++++++++ metagpt/learn/text_to_speech.py | 29 ++++++++++++++-- requirements.txt | 3 +- 3 files changed, 85 insertions(+), 4 deletions(-) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index 56c6f42d5..1e3cecb10 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -73,6 +73,63 @@ paths: '500': description: "Internal Server Error" + /tts/iflytek: + x-prerequisite: + - name: IFLYTEK_APP_ID + description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" + - name: IFLYTEK_API_KEY + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + - name: IFLYTEK_API_SECRET + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + post: + summary: "Convert Text to Base64-encoded .mp3 File Stream" + description: "For more details, check out: [iFlyTek](https://console.xfyun.cn/services/tts)" + operationId: iflytek_tts.oas3_iflytek_tts + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - text + properties: + text: + type: string + description: Text to convert + voice: + type: string + description: "Voice style, see: [iFlyTek Text-to_Speech](https://www.xfyun.cn/doc/tts/online_tts/API.html#%E6%8E%A5%E5%8F%A3%E8%B0%83%E7%94%A8%E6%B5%81%E7%A8%8B)" + default: "xiaoyan" + app_id: + type: string + description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" + default: "" + api_key: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + default: "" + api_secret: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + default: "" + responses: + '200': + description: "Base64-encoded .mp3 file data if successful, otherwise an empty string." + content: + application/json: + schema: + type: object + properties: + wav_data: + type: string + format: base64 + '400': + description: "Bad Request" + '500': + description: "Internal Server Error" + + /txt2img/openai: x-prerequisite: - name: OPENAI_API_KEY diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 81bc8512b..7c085c02f 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -11,6 +11,7 @@ import openai from metagpt.config import CONFIG from metagpt.const import BASE64_FORMAT from metagpt.tools.azure_tts import oas3_azsure_tts +from metagpt.tools.iflytek_tts import oas3_iflytek_tts from metagpt.utils.s3 import S3 @@ -22,6 +23,9 @@ async def text_to_speech( role="Girl", subscription_key="", region="", + iflytek_app_id="", + iflytek_api_key="", + iflytek_api_secret="", **kwargs, ): """Text to speech @@ -34,16 +38,35 @@ async def text_to_speech( :param text: The text used for voice conversion. :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. - :return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string. + :param iflytek_app_id: Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts` + :param iflytek_api_key: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + :param iflytek_api_secret: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + :return: Returns the Base64-encoded .wav/.mp3 file data if successful, otherwise an empty string. """ - audio_declaration = "data:audio/wav;base64," + if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region): + audio_declaration = "data:audio/wav;base64," base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) s3 = S3() url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) if url: return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data + if (CONFIG.IFLYTEK_APP_ID and CONFIG.IFLYTEK_API_KEY and CONFIG.IFLYTEK_API_SECRET) or ( + iflytek_app_id and iflytek_api_key and iflytek_api_secret + ): + audio_declaration = "data:audio/mp3;base64," + base64_data = await oas3_iflytek_tts( + text=text, app_id=iflytek_app_id, api_key=iflytek_api_key, api_secret=iflytek_api_secret + ) + s3 = S3() + url = await s3.cache(data=base64_data, file_ext=".mp3", format=BASE64_FORMAT) + if url: + return f"[{text}]({url})" + return audio_declaration + base64_data if base64_data else base64_data - raise openai.error.InvalidRequestError(message="AZURE_TTS_SUBSCRIPTION_KEY and AZURE_TTS_REGION error", param={}) + raise openai.error.InvalidRequestError( + message="AZURE_TTS_SUBSCRIPTION_KEY, AZURE_TTS_REGION, IFLYTEK_APP_ID, IFLYTEK_API_KEY, IFLYTEK_API_SECRET error", + param={}, + ) diff --git a/requirements.txt b/requirements.txt index 588b29e0b..2dd767026 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,4 +42,5 @@ connexion[swagger-ui] aiohttp_jinja2 azure-cognitiveservices-speech==1.31.0 aioboto3~=11.3.0 -redis==4.3.5 \ No newline at end of file +redis==4.3.5 +websocket-client \ No newline at end of file From f8aea281a85fde07459780f3e1f7e3b5a1e27e5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:11:22 +0800 Subject: [PATCH 264/398] feat: +iflytek tts --- metagpt/tools/iflytek_tts.py | 162 +++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 metagpt/tools/iflytek_tts.py diff --git a/metagpt/tools/iflytek_tts.py b/metagpt/tools/iflytek_tts.py new file mode 100644 index 000000000..a91d8091b --- /dev/null +++ b/metagpt/tools/iflytek_tts.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : iflytek_tts.py +@Desc : iFLYTEK TTS OAS3 api, which provides text-to-speech functionality +""" +import asyncio +import base64 +import hashlib +import hmac +import json +import uuid +from datetime import datetime +from enum import Enum +from pathlib import Path +from time import mktime +from typing import Optional +from urllib.parse import urlencode +from wsgiref.handlers import format_date_time + +import aiofiles +import websockets as websockets +from pydantic import BaseModel + +from metagpt.config import CONFIG +from metagpt.logs import logger + + +class IFlyTekTTSStatus(Enum): + STATUS_FIRST_FRAME = 0 # The first frame + STATUS_CONTINUE_FRAME = 1 # The intermediate frame + STATUS_LAST_FRAME = 2 # The last frame + + +class AudioData(BaseModel): + audio: str + status: int + ced: str + + +class IFlyTekTTSResponse(BaseModel): + code: int + message: str + data: Optional[AudioData] = None + sid: str + + +DEFAULT_IFLYTEK_VOICE = "xiaoyan" + + +class IFlyTekTTS(object): + def __init__(self, app_id: str, api_key: str, api_secret: str): + """ + :param app_id: Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts` + :param api_key: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + :param api_secret: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + """ + self.app_id = app_id or CONFIG.IFLYTEK_APP_ID + self.api_key = api_key or CONFIG.IFLYTEK_API_KEY + self.api_secret = api_secret or CONFIG.API_SECRET + + async def synthesize_speech(self, text, output_file: str, voice=DEFAULT_IFLYTEK_VOICE): + url = self._create_url() + data = { + "common": {"app_id": self.app_id}, + "business": {"aue": "lame", "sfl": 1, "auf": "audio/L16;rate=16000", "vcn": voice, "tte": "utf8"}, + "data": {"status": 2, "text": str(base64.b64encode(text.encode("utf-8")), "UTF8")}, + } + req = json.dumps(data) + async with websockets.connect(url) as websocket: + # send request + await websocket.send(req) + + # receive frames + async with aiofiles.open(str(output_file), "w") as writer: + while True: + v = await websocket.recv() + rsp = IFlyTekTTSResponse(**json.loads(v)) + if rsp.data: + await writer.write(rsp.data.audio) + if rsp.data.status != IFlyTekTTSStatus.STATUS_LAST_FRAME.value: + continue + break + + def _create_url(self): + """Create request url""" + url = "wss://tts-api.xfyun.cn/v2/tts" + # Generate a timestamp in RFC1123 format + now = datetime.now() + date = format_date_time(mktime(now.timetuple())) + + signature_origin = "host: " + "ws-api.xfyun.cn" + "\n" + signature_origin += "date: " + date + "\n" + signature_origin += "GET " + "/v2/tts " + "HTTP/1.1" + # Perform HMAC-SHA256 encryption + signature_sha = hmac.new( + self.api_secret.encode("utf-8"), signature_origin.encode("utf-8"), digestmod=hashlib.sha256 + ).digest() + signature_sha = base64.b64encode(signature_sha).decode(encoding="utf-8") + + authorization_origin = 'api_key="%s", algorithm="%s", headers="%s", signature="%s"' % ( + self.api_key, + "hmac-sha256", + "host date request-line", + signature_sha, + ) + authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(encoding="utf-8") + # Combine the authentication parameters of the request into a dictionary. + v = {"authorization": authorization, "date": date, "host": "ws-api.xfyun.cn"} + # Concatenate the authentication parameters to generate the URL. + url = url + "?" + urlencode(v) + return url + + +# Export +async def oas3_iflytek_tts(text: str, voice: str = "", app_id: str = "", api_key: str = "", api_secret: str = ""): + """Text to speech + For more details, check out:`https://www.xfyun.cn/doc/tts/online_tts/API.html` + + :param voice: Default `xiaoyan`. For more details, checkout: `https://www.xfyun.cn/doc/tts/online_tts/API.html#%E6%8E%A5%E5%8F%A3%E8%B0%83%E7%94%A8%E6%B5%81%E7%A8%8B` + :param text: The text used for voice conversion. + :param app_id: Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts` + :param api_key: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + :param api_secret: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + :return: Returns the Base64-encoded .mp3 file data if successful, otherwise an empty string. + + """ + if not app_id: + app_id = CONFIG.IFLYTEK_APP_ID + if not api_key: + api_key = CONFIG.IFLYTEK_API_KEY + if not api_secret: + api_secret = CONFIG.IFLYTEK_API_SECRET + if not voice: + voice = CONFIG.IFLYTEK_VOICE or DEFAULT_IFLYTEK_VOICE + + filename = Path(__file__).parent / (str(uuid.uuid4()).replace("-", "") + ".mp3") + try: + tts = IFlyTekTTS(app_id=app_id, api_key=api_key, api_secret=api_secret) + await tts.synthesize_speech(text=text, output_file=str(filename), voice=voice) + async with aiofiles.open(str(filename), mode="r") as reader: + base64_string = await reader.read() + except Exception as e: + logger.error(f"text:{text}, error:{e}") + base64_string = "" + finally: + filename.unlink() + + return base64_string + + +if __name__ == "__main__": + asyncio.get_event_loop().run_until_complete( + oas3_iflytek_tts( + text="你好，hello", + app_id="f7acef62", + api_key="fda72e3aa286042a492525816a5efa08", + api_secret="ZDk3NjdiMDBkODJlOWQ1NjRjMGI2NDY4", + ) + ) From 96aad1ce7745e7e39ae5dc82fbd2f59bf7ff144a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:25:50 +0800 Subject: [PATCH 265/398] feat: +log --- metagpt/tools/metagpt_oas3_api_svc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index 5c23f6566..2ff4c8225 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -7,8 +7,8 @@ @Desc : MetaGPT OpenAPI Specification 3.0 REST API service """ import asyncio -from pathlib import Path import sys +from pathlib import Path import connexion @@ -17,7 +17,7 @@ sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: def oas_http_svc(): """Start the OAS 3.0 OpenAPI HTTP service""" - app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') + app = connexion.AioHttpApp(__name__, specification_dir="../../.well-known/") app.add_api("metagpt_oas3_api.yaml") app.add_api("openapi.yaml") app.run(port=8080) @@ -35,6 +35,7 @@ async def async_main(): def main(): + print("http://localhost:8080/oas3/ui/") oas_http_svc() From c800ad02d18bff6295af1a0d3a0fc1f50e9092a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:35:48 +0800 Subject: [PATCH 266/398] feat: +example --- .well-known/skills.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 009368dbe..d08d7aced 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -21,7 +21,9 @@ entities: - ask: 'A boy affectionate says "hello world"' answer: 'text_to_speech(text="hello world", role="Boy", style="affectionate")' - ask: 'A boy says "你好"' - answer: 'text_to_speech(text="hello world", role="Boy", lang="Chinese")' + answer: 'text_to_speech(text="你好", role="Boy", lang="Chinese")' + - ask: 'How to speak "你好"?' + answer: 'text_to_speech(text="你好", lang="Chinese")' returns: type: string format: base64 @@ -42,6 +44,10 @@ entities: answer: 'text_to_image(text="Draw a girl", size_type="512x512")' - ask: 'Draw an apple' answer: 'text_to_image(text="Draw an apple", size_type="512x512")' + - ask: 'Draw an apple picture' + answer: 'text_to_image(text="Draw an apple", size_type="512x512")' + - ask: 'Draw an apple image' + answer: 'text_to_image(text="Draw an apple", size_type="512x512")' returns: type: string format: base64 From f60b68f1c54bec7bd787e0620828887cc1a6ed09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:39:38 +0800 Subject: [PATCH 267/398] refactor: think prompt --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 87127cbab..a988572f4 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -62,7 +62,7 @@ class Assistant(Role): ) prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" - prompt += "Otherwise, rewrite and return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" + prompt += "Otherwise, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(rsp) From a71708addcdc19575b6ef7f5e36cbf871655867c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:56:41 +0800 Subject: [PATCH 268/398] feat: +ver --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2dd767026..3b2dc3106 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,4 +43,4 @@ aiohttp_jinja2 azure-cognitiveservices-speech==1.31.0 aioboto3~=11.3.0 redis==4.3.5 -websocket-client \ No newline at end of file +websocket-client==1.6.2 \ No newline at end of file From 50835b8c472b23238d351aadade7acf3b79e428d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 20:04:44 +0800 Subject: [PATCH 269/398] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index bf2ca7f14..06e06df69 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,7 +276,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return response async def is_related(self, text1, text2): - command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"Paragraph 1:{text1}\n\nParagraph 2:{text2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From 246bf5ce00ab6a71fe8f97a297bbd44ed47a5bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 20:07:55 +0800 Subject: [PATCH 270/398] refactor: think --- metagpt/provider/openai_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 06e06df69..68b0e4171 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,7 +276,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return response async def is_related(self, text1, text2): - command = f"Paragraph 1:{text1}\n\nParagraph 2:{text2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + p1 = text1.replace("\n", " ") + p2 = text2.replace("\n", " ") + command = f"Paragraph 1:{p1}\n\nParagraph 2:{p2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From 0b412008c4e10626d124c2939dfcb9c43e529bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 20:11:51 +0800 Subject: [PATCH 271/398] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 68b0e4171..353ae46a0 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -278,7 +278,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): p1 = text1.replace("\n", " ") p2 = text2.replace("\n", " ") - command = f"Paragraph 1:{p1}\n\nParagraph 2:{p2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"Paragraph 1: {p1}\n\nParagraph 2: {p2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From caff43e1965acb87baf1011f54a9a77f68b4d041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 20:24:46 +0800 Subject: [PATCH 272/398] refactor: think --- metagpt/provider/openai_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 353ae46a0..fdf95f68c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -280,14 +280,14 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): p2 = text2.replace("\n", " ") command = f"Paragraph 1: {p1}\n\nParagraph 2: {p2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) + logger.info(f"RELATED:{rsp}\n\n{p1}\n{p2}") result, _ = self.extract_info(rsp) return result == "TRUE" async def rewrite(self, sentence: str, context: str): - command = ( - f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" - ) + command = f"{context}\n\nTaking into account the information above, please rephrase and provide the revised sentence:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) + logger.info(f"REWRITE:{rsp}\nFROM\n\n{sentence}") return rsp @staticmethod From b76ab1943656353eabde3320e7b8d4ffa1b24172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 20:32:02 +0800 Subject: [PATCH 273/398] refactor: think --- metagpt/provider/openai_api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index fdf95f68c..827a2e399 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -278,16 +278,14 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): p1 = text1.replace("\n", " ") p2 = text2.replace("\n", " ") - command = f"Paragraph 1: {p1}\n\nParagraph 2: {p2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"Paragraph 1: {p2}\n\nParagraph 2: {p1}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) - logger.info(f"RELATED:{rsp}\n\n{p1}\n{p2}") result, _ = self.extract_info(rsp) return result == "TRUE" async def rewrite(self, sentence: str, context: str): command = f"{context}\n\nTaking into account the information above, please rephrase and provide the revised sentence:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) - logger.info(f"REWRITE:{rsp}\nFROM\n\n{sentence}") return rsp @staticmethod From 508fff69209ce0d34699bc4ac37dc13382f2b19e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:02:51 +0800 Subject: [PATCH 274/398] refactor: think --- metagpt/provider/openai_api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 827a2e399..90fcd7ab3 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,15 +276,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return response async def is_related(self, text1, text2): - p1 = text1.replace("\n", " ") - p2 = text2.replace("\n", " ") - command = f"Paragraph 1: {p2}\n\nParagraph 2: {p1}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" async def rewrite(self, sentence: str, context: str): - command = f"{context}\n\nTaking into account the information above, please rephrase and provide the revised sentence:\n{sentence}" + # command = ( + # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + # ) + command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) return rsp From a9b56a6f56e1950e2b86f3c9c06f0c6f7bfed269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:04:25 +0800 Subject: [PATCH 275/398] refactor: think --- metagpt/roles/assistant.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index a988572f4..7fd1b1236 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -129,9 +129,10 @@ class Assistant(Role): await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS ) - if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. - last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) - return last_talk + # if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. + # last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) + # return last_talk + last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk From f450b61bc215ad70fbafb14e153a0cd905e203e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:13:10 +0800 Subject: [PATCH 276/398] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 90fcd7ab3..462d9d12d 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -286,7 +286,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # command = ( # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" # ) - command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text:\n{sentence}" + command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) return rsp From cb17a17b4aa02552c6d99af6e18dbb8946ace33f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:20:39 +0800 Subject: [PATCH 277/398] refactor: think --- metagpt/roles/assistant.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 7fd1b1236..a988572f4 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -129,10 +129,9 @@ class Assistant(Role): await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS ) - # if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. - # last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) - # return last_talk - last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) + if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. + last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) + return last_talk return last_talk From 1e39618b972bb9b9d55d53b1256c413451ecd289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:29:49 +0800 Subject: [PATCH 278/398] refactor: think --- metagpt/roles/assistant.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index a988572f4..0a6237f42 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -60,8 +60,7 @@ class Assistant(Role): prompt += ( f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" ) - prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" - prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" + prompt += "If the user's intent is unclear, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon\n" prompt += "Otherwise, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" logger.info(prompt) rsp = await self._llm.aask(prompt, []) @@ -90,7 +89,6 @@ class Assistant(Role): skill, text = Assistant.extract_info(input_string=rsp) handlers = { MessageType.Talk.value: self.talk_handler, - MessageType.Problem.value: self.talk_handler, MessageType.Skill.value: self.skill_handler, } handler = handlers.get(skill, self.talk_handler) From 80b934d41ac5e6cdc559586fdfe5a699bad0c149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:33:55 +0800 Subject: [PATCH 279/398] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 462d9d12d..7139c4946 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -277,7 +277,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." - command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From fa0b0b15114899e6724f081b7aa8f0dbfd9fbb6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:43:21 +0800 Subject: [PATCH 280/398] refactor: think --- metagpt/roles/assistant.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 0a6237f42..c0d1c3240 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -57,10 +57,7 @@ class Assistant(Role): prompt = f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): - prompt += ( - f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" - ) - prompt += "If the user's intent is unclear, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon\n" + prompt += f"If explicitly want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += "Otherwise, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" logger.info(prompt) rsp = await self._llm.aask(prompt, []) From e2ffba863127b376afa53f3165816544206572bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:51:14 +0800 Subject: [PATCH 281/398] refactor: think --- metagpt/roles/assistant.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index c0d1c3240..86a27cb18 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -58,7 +58,9 @@ class Assistant(Role): skills = self.skills.get_skill_list() for desc, name in skills.items(): prompt += f"If explicitly want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" - prompt += "Otherwise, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" + prompt += ( + 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx' + ) logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(rsp) From e25e19eb8fe6a4392766adf14f6456a649f023d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:59:36 +0800 Subject: [PATCH 282/398] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 7139c4946..949b252b2 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -277,7 +277,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." - command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From 703b2a9a2418f3184ecad157b76e28112983cbec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 23:15:11 +0800 Subject: [PATCH 283/398] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 949b252b2..e352ff54f 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -277,7 +277,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." - command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE]:1 brief and clear. Otherwise, return [FALSE]:1 brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From a147bdf92a306553bc93580b085102fb0efd7295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 23:18:18 +0800 Subject: [PATCH 284/398] refactor: think --- metagpt/provider/openai_api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index e352ff54f..bbceac1d2 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -318,8 +318,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return windows @staticmethod - def extract_info(input_string): - pattern = r"\[([A-Z]+)\]:\s*(.+)" + def extract_info(input_string, pattern=r"\[([A-Z]+)\]:\s*(.+)"): match = re.match(pattern, input_string) if match: return match.group(1), match.group(2) From c8e24aa39b60cdea52664a29f6c52180c04d31be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 23:20:13 +0800 Subject: [PATCH 285/398] refactor: think --- metagpt/provider/openai_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index bbceac1d2..30b82b8dc 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -277,9 +277,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." - command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE]:1 brief and clear. Otherwise, return [FALSE]:1 brief and clear." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) - result, _ = self.extract_info(rsp) + result, _ = self.extract_info(rsp, pattern=r"\[([A-Z]+)\]\s*(.+)") return result == "TRUE" async def rewrite(self, sentence: str, context: str): From 558f80b238a1da513046351dab31c97598fa3282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 23:35:24 +0800 Subject: [PATCH 286/398] refactor: think --- metagpt/roles/assistant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 86a27cb18..428c1a70f 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -54,10 +54,10 @@ class Assistant(Role): last_talk = await self.refine_memory() if not last_talk: return False - prompt = f"Refer to this sentence:\n {last_talk}\n" + prompt = f"Refer to this text:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): - prompt += f"If explicitly want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" + prompt += f"If the text explicitly want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += ( 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx' ) From 04231088c7717241df1da275f1c553854188897c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:14:31 +0800 Subject: [PATCH 287/398] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 428c1a70f..6530a3cac 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -57,7 +57,7 @@ class Assistant(Role): prompt = f"Refer to this text:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): - prompt += f"If the text explicitly want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" + prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += ( 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx' ) From ac211ae3a6ed9df419585b70d6a6765223a6aaf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:17:21 +0800 Subject: [PATCH 288/398] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 6530a3cac..516f78b0e 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -63,7 +63,7 @@ class Assistant(Role): ) logger.info(prompt) rsp = await self._llm.aask(prompt, []) - logger.info(rsp) + logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") return await self._plan(rsp, last_talk=last_talk) async def act(self) -> ActionOutput: From 092243670f7e9e716187be27843c7d11aff6b832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:23:29 +0800 Subject: [PATCH 289/398] feat: +log --- metagpt/provider/openai_api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 30b82b8dc..99f281964 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -280,6 +280,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp, pattern=r"\[([A-Z]+)\]\s*(.+)") + p2 = text2.replace("\n", "") + p1 = text1.replace("\n", "") + logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}") return result == "TRUE" async def rewrite(self, sentence: str, context: str): @@ -288,6 +291,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # ) command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) + logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}") return rsp @staticmethod From 6f55709ec599f804dcaefd86b4260e6ec6024f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:31:21 +0800 Subject: [PATCH 290/398] feat: +log --- metagpt/provider/openai_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 99f281964..d84109f6a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -279,11 +279,11 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) - result, _ = self.extract_info(rsp, pattern=r"\[([A-Z]+)\]\s*(.+)") + result = True if "TRUE" in rsp else False p2 = text2.replace("\n", "") p1 = text1.replace("\n", "") logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}") - return result == "TRUE" + return result async def rewrite(self, sentence: str, context: str): # command = ( From 8f8a5e185a84ebccc5bad58fa0a21fa963613cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:41:30 +0800 Subject: [PATCH 291/398] refactor: think --- metagpt/roles/assistant.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 516f78b0e..bae1b6c79 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -54,13 +54,12 @@ class Assistant(Role): last_talk = await self.refine_memory() if not last_talk: return False - prompt = f"Refer to this text:\n {last_talk}\n" + prompt = "" skills = self.skills.get_skill_list() for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" - prompt += ( - 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx' - ) + prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' + prompt = f"Now the text is: {last_talk}" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") From 8695a042e99cc61efeedbf2bde9c2db0525f5751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:49:06 +0800 Subject: [PATCH 292/398] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index bae1b6c79..a615c3933 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -59,7 +59,7 @@ class Assistant(Role): for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' - prompt = f"Now the text is: {last_talk}" + prompt += f"Now the text is: {last_talk}" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") From 2ff563e6b6261bf04116991f92ba0c3bacad920d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 11:01:21 +0800 Subject: [PATCH 293/398] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index a615c3933..743ec7c43 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -59,7 +59,7 @@ class Assistant(Role): for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' - prompt += f"Now the text is: {last_talk}" + prompt += f"Now determine the appropriate pattern for the text: {last_talk}" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") From 5f5fda42730cbc2e6441e20260bf246a6ee98e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 11:03:25 +0800 Subject: [PATCH 294/398] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 743ec7c43..07991da1a 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -59,7 +59,7 @@ class Assistant(Role): for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' - prompt += f"Now determine the appropriate pattern for the text: {last_talk}" + prompt += f"Now determine the appropriate pattern for the text: {last_talk}\n" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") From db72848965b29400a9235f7432a0a85cd3206ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 11:17:32 +0800 Subject: [PATCH 295/398] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 07991da1a..d310fca7c 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -59,7 +59,7 @@ class Assistant(Role): for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' - prompt += f"Now determine the appropriate pattern for the text: {last_talk}\n" + prompt += f"Now what specific action does the text explicitly ask for: {last_talk}\n" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") From 341bbbe4ba8a1e959158724196b9a8529d4211dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 11:33:40 +0800 Subject: [PATCH 296/398] refactor: think --- metagpt/roles/assistant.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index d310fca7c..bef2cf53c 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -59,8 +59,7 @@ class Assistant(Role): for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' - prompt += f"Now what specific action does the text explicitly ask for: {last_talk}\n" - logger.info(prompt) + prompt += f"Now what specific action is explicitly mentioned in the text: {last_talk}\n" rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") return await self._plan(rsp, last_talk=last_talk) From 03019a304bfd342b2a0d5ed62b5a262bb513e8e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 12:13:13 +0800 Subject: [PATCH 297/398] refactor: think --- metagpt/actions/talk_action.py | 1 + metagpt/provider/openai_api.py | 21 +++++++++++++-------- metagpt/roles/assistant.py | 14 +++++--------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 81caef013..4afed8014 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -44,6 +44,7 @@ class TalkAction(Action): f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n " f"{self._talk}" ) + logger.info(f"PROMPT: {prompt}") return prompt @property diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index bf2ca7f14..d84109f6a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,16 +276,22 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return response async def is_related(self, text1, text2): - command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) - result, _ = self.extract_info(rsp) - return result == "TRUE" + result = True if "TRUE" in rsp else False + p2 = text2.replace("\n", "") + p1 = text1.replace("\n", "") + logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}") + return result async def rewrite(self, sentence: str, context: str): - command = ( - f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" - ) + # command = ( + # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + # ) + command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) + logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}") return rsp @staticmethod @@ -316,8 +322,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return windows @staticmethod - def extract_info(input_string): - pattern = r"\[([A-Z]+)\]:\s*(.+)" + def extract_info(input_string, pattern=r"\[([A-Z]+)\]:\s*(.+)"): match = re.match(pattern, input_string) if match: return match.group(1), match.group(2) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 87127cbab..ac80a4bc8 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -54,18 +54,15 @@ class Assistant(Role): last_talk = await self.refine_memory() if not last_talk: return False - prompt = f"Refer to this sentence:\n {last_talk}\n" + prompt = "" skills = self.skills.get_skill_list() for desc, name in skills.items(): - prompt += ( - f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" - ) - prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" - prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" - prompt += "Otherwise, rewrite and return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" + prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" + prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' + prompt += f"Now what specific action is explicitly mentioned in the text: {last_talk}\n" logger.info(prompt) rsp = await self._llm.aask(prompt, []) - logger.info(rsp) + logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") return await self._plan(rsp, last_talk=last_talk) async def act(self) -> ActionOutput: @@ -90,7 +87,6 @@ class Assistant(Role): skill, text = Assistant.extract_info(input_string=rsp) handlers = { MessageType.Talk.value: self.talk_handler, - MessageType.Problem.value: self.talk_handler, MessageType.Skill.value: self.skill_handler, } handler = handlers.get(skill, self.talk_handler) From 4e0b2898a6a54993738448991699e81dd58bd577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 12:24:20 +0800 Subject: [PATCH 298/398] refactor: think --- metagpt/roles/assistant.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index ac80a4bc8..bef2cf53c 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -60,7 +60,6 @@ class Assistant(Role): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' prompt += f"Now what specific action is explicitly mentioned in the text: {last_talk}\n" - logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") return await self._plan(rsp, last_talk=last_talk) From c792cf09ecb642faf9bc628edf920b43847f83f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 13:18:03 +0800 Subject: [PATCH 299/398] refactor: think --- metagpt/roles/assistant.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index bef2cf53c..cd1932f82 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -93,6 +93,7 @@ class Assistant(Role): async def talk_handler(self, text, **kwargs) -> bool: history = self.memory.history_text + text = kwargs.get("last_talk") or text action = TalkAction( talk=text, knowledge=self.memory.get_knowledge(), history_summary=history, llm=self._llm, **kwargs ) From eec0fbde6d0a4563b166b5cab929a65bc70c518b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 15:02:36 +0800 Subject: [PATCH 300/398] refactor: disable log --- metagpt/memory/brain_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 2195da566..f309b532e 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -72,7 +72,7 @@ class BrainMemory(pydantic.BaseModel): if not redis.is_valid() or not redis_key: return BrainMemory() v = await redis.get(key=redis_key) - logger.info(f"REDIS GET {redis_key} {v}") + logger.debug(f"REDIS GET {redis_key} {v}") if v: data = json.loads(v) bm = BrainMemory(**data) @@ -86,7 +86,7 @@ class BrainMemory(pydantic.BaseModel): return False v = self.json() await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) - logger.info(f"REDIS SET {redis_key} {v}") + logger.debug(f"REDIS SET {redis_key} {v}") self.is_dirty = False @staticmethod From b3be30bdad534836d1bdaa168ae2a8a9d9e42245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 15:12:41 +0800 Subject: [PATCH 301/398] refactor: log --- metagpt/actions/skill_action.py | 3 +-- metagpt/actions/talk_action.py | 5 ++--- metagpt/provider/openai_api.py | 12 ++++++------ 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index 758591fdd..f629cfcbf 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -40,9 +40,8 @@ class ArgumentsParingAction(Action): async def run(self, *args, **kwargs) -> ActionOutput: prompt = self.prompt - logger.info(prompt) rsp = await self.llm.aask(msg=prompt, system_msgs=[]) - logger.info(rsp) + logger.debug(f"SKILL:{prompt}\n, RESULT:{rsp}") self.args = ArgumentsParingAction.parse_arguments(skill_name=self.skill.name, txt=rsp) self.rsp = ActionOutput(content=rsp) return self.rsp diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 4afed8014..0e3762798 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -44,7 +44,7 @@ class TalkAction(Action): f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n " f"{self._talk}" ) - logger.info(f"PROMPT: {prompt}") + logger.debug(f"PROMPT: {prompt}") return prompt @property @@ -64,9 +64,8 @@ class TalkAction(Action): async def run(self, *args, **kwargs) -> ActionOutput: prompt = self.prompt - logger.info(prompt) rsp = await self.llm.aask(msg=prompt, system_msgs=[]) - logger.info(rsp) + logger.debug(f"PROMPT:{prompt}\nRESULT:{rsp}\n") self._rsp = ActionOutput(content=rsp) return self._rsp diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index d84109f6a..863475f52 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -257,9 +257,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): else: command = f"Translate the above content into a summary of less than {max_words} words." msg = text + "\n\n" + command - logger.info(f"summary ask:{msg}") + logger.debug(f"summary ask:{msg}") response = await self.aask(msg=msg, system_msgs=[]) - logger.info(f"summary rsp: {response}") + logger.debug(f"summary rsp: {response}") return response async def get_context_title(self, text: str, max_words=5) -> str: @@ -270,9 +270,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): command = f"Translate the above summary into a {language} title of less than {max_words} words." summaries = [summary, command] msg = "\n".join(summaries) - logger.info(f"title ask:{msg}") + logger.debug(f"title ask:{msg}") response = await self.aask(msg=msg, system_msgs=[]) - logger.info(f"title rsp: {response}") + logger.debug(f"title rsp: {response}") return response async def is_related(self, text1, text2): @@ -282,7 +282,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): result = True if "TRUE" in rsp else False p2 = text2.replace("\n", "") p1 = text1.replace("\n", "") - logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}") + logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}\n") return result async def rewrite(self, sentence: str, context: str): @@ -291,7 +291,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # ) command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) - logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}") + logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}\n") return rsp @staticmethod From 832294809b097793dff3472b1183aed37f8f5c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 11:01:27 +0800 Subject: [PATCH 302/398] feat: + LLMType --- metagpt/llm.py | 24 ++++++++++++++++++++++-- metagpt/provider/__init__.py | 4 +++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 6a9a9132f..0ef23d0be 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -4,17 +4,37 @@ @Time : 2023/5/11 14:45 @Author : alexanderwu @File : llm.py +@Modified By: mashenquan, 2023 """ +from enum import Enum from metagpt.provider.anthropic_api import Claude2 as Claude from metagpt.provider.openai_api import OpenAIGPTAPI as LLM + +class LLMType(Enum): + OPENAI = "OpenAI" + METAGPT = "MetaGPT" + UNKNOWN = "UNKNOWN" + + @classmethod + def get(cls, value): + for member in cls: + if member.value == value: + return member + return cls.UNKNOWN + + @property + def UNKNOWN(self): + return LLMType.UNKNOWN + + DEFAULT_LLM = LLM() CLAUDE_LLM = Claude() async def ai_func(prompt): """使用LLM进行QA - QA with LLMs - """ + QA with LLMs + """ return await DEFAULT_LLM.aask(prompt) diff --git a/metagpt/provider/__init__.py b/metagpt/provider/__init__.py index 56dc19b4b..9895aa7fc 100644 --- a/metagpt/provider/__init__.py +++ b/metagpt/provider/__init__.py @@ -4,9 +4,11 @@ @Time : 2023/5/5 22:59 @Author : alexanderwu @File : __init__.py +@Modified By: mashenquan, 2023/9/8. Add `MetaGPTLLMAPI` """ from metagpt.provider.openai_api import OpenAIGPTAPI +from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI -__all__ = ["OpenAIGPTAPI"] +__all__ = ["OpenAIGPTAPI", "MetaGPTLLMAPI"] From 154f67c5e32467a9a21b5cb979aed25fa7e32520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 11:06:39 +0800 Subject: [PATCH 303/398] feat: + LLMType --- metagpt/llm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 0ef23d0be..e31eee908 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -9,7 +9,8 @@ from enum import Enum from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM +from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI as MetaGPT_LLM +from metagpt.provider.openai_api import OpenAIGPTAPI as OpenAI_LLM class LLMType(Enum): @@ -29,7 +30,8 @@ class LLMType(Enum): return LLMType.UNKNOWN -DEFAULT_LLM = LLM() +DEFAULT_LLM = OpenAI_LLM() +DEFAULT_METAGPT_LLM = MetaGPT_LLM() CLAUDE_LLM = Claude() From e316fe4d60ac8b2de96729fdc998d0db6069d1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 11:20:27 +0800 Subject: [PATCH 304/398] feat: + LLMType --- metagpt/llm.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/llm.py b/metagpt/llm.py index e31eee908..87ce8fa5b 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -8,6 +8,7 @@ """ from enum import Enum +from metagpt.config import CONFIG from metagpt.provider.anthropic_api import Claude2 as Claude from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI as MetaGPT_LLM from metagpt.provider.openai_api import OpenAIGPTAPI as OpenAI_LLM @@ -40,3 +41,10 @@ async def ai_func(prompt): QA with LLMs """ return await DEFAULT_LLM.aask(prompt) + + +class LLMFactory: + @staticmethod + async def new_llm() -> object: + llm = OpenAI_LLM() if CONFIG.LLM_TYPE == LLMType.OPENAI.value else MetaGPT_LLM() + return llm From c513712928e58ae3782819b29accf515ff366de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 11:43:34 +0800 Subject: [PATCH 305/398] feat: + kwargs --- metagpt/provider/openai_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 863475f52..64267975e 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -223,7 +223,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - async def get_summary(self, text: str, max_words=200, keep_language: bool = False): + async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 text_length = len(text) @@ -262,7 +262,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): logger.debug(f"summary rsp: {response}") return response - async def get_context_title(self, text: str, max_words=5) -> str: + async def get_context_title(self, text: str, max_words=5, **kwargs) -> str: """Generate text title""" summary = await self.get_summary(text, max_words=500) From a41fe2494e34af249ebdd530f0a8cecb2b3a259c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 11:55:36 +0800 Subject: [PATCH 306/398] feat: +LLMType --- metagpt/llm.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 87ce8fa5b..93cbcaaf6 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -26,10 +26,6 @@ class LLMType(Enum): return member return cls.UNKNOWN - @property - def UNKNOWN(self): - return LLMType.UNKNOWN - DEFAULT_LLM = OpenAI_LLM() DEFAULT_METAGPT_LLM = MetaGPT_LLM() From 3a4f31b51787f2e60bed4efda45f63d49e1637ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 12:00:30 +0800 Subject: [PATCH 307/398] feat: +LLMType --- metagpt/actions/action.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index e4b9613ad..c52caaa40 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -5,6 +5,7 @@ @Author : alexanderwu @File : action.py @Modified By: mashenquan, 2023/8/20. Add function return annotations. +@Modified By: mashenquan, 2023/9/8. Replace LLM with LLMFactory """ from __future__ import annotations @@ -14,16 +15,17 @@ from typing import Optional from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput -from metagpt.llm import LLM +from metagpt.llm import LLMFactory from metagpt.logs import logger +from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.common import OutputParser class Action(ABC): - def __init__(self, name: str = "", context=None, llm: LLM = None): + def __init__(self, name: str = "", context=None, llm: BaseGPTAPI = None): self.name: str = name if llm is None: - llm = LLM() + llm = LLMFactory.new_llm() self.llm = llm self.context = context self.prefix = "" From c7bc975cf20c926dcc52756efcc3038c9b6b30f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 12:02:43 +0800 Subject: [PATCH 308/398] fixbug: LLM() --- metagpt/roles/role.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index b1ace19fa..6d774b0b4 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -16,7 +16,7 @@ from pydantic import BaseModel, Field from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG from metagpt.const import OPTIONS -from metagpt.llm import LLM +from metagpt.llm import LLMFactory from metagpt.logs import logger from metagpt.memory import LongTermMemory, Memory from metagpt.schema import Message, MessageTag @@ -113,7 +113,7 @@ class Role: constraints = Role.format_value(constraints) desc = Role.format_value(desc) - self._llm = LLM() + self._llm = LLMFactory.new_llm() self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) self._states = [] self._actions = [] From 2324c1c6dcc334cfe368b3e4252db3060dbfbba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 12:44:49 +0800 Subject: [PATCH 309/398] fixbug: LLM() --- metagpt/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 93cbcaaf6..4772d2e6e 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -41,6 +41,6 @@ async def ai_func(prompt): class LLMFactory: @staticmethod - async def new_llm() -> object: + def new_llm() -> object: llm = OpenAI_LLM() if CONFIG.LLM_TYPE == LLMType.OPENAI.value else MetaGPT_LLM() return llm From ef485e7400546e5577f3ca59fcc089c811e13f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 13:52:36 +0800 Subject: [PATCH 310/398] feat: + summary --- metagpt/provider/metagpt_llm_api.py | 21 +++++++++++++++++++++ metagpt/roles/assistant.py | 4 +++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index c27e7132d..f8c4ac1ed 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -5,10 +5,13 @@ @File : metagpt_llm_api.py @Desc : MetaGPT LLM related APIs """ +import json import openai +from pydantic import BaseModel from metagpt.config import CONFIG +from metagpt.memory.brain_memory import BrainMemory from metagpt.provider import OpenAIGPTAPI from metagpt.provider.openai_api import RateLimiter @@ -31,3 +34,21 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): openai.api_type = CONFIG.METAGPT_API_TYPE openai.api_version = CONFIG.METAGPT_API_VERSION self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 + + async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs): + summary = [] + + class QuweryAnswerPair(BaseModel): + ask: str + answer: str + + rh = reversed(memory.history) + ix = 0 + while ix < len(rh): + t = rh[ix] + print(t) + # 如果 t是ask, continue + pass + + data = json.dumps(summary) + return data diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index cd1932f82..0a796ac11 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -121,7 +121,9 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_summary(history_text, max_words=800, keep_language=True) + history_summary = await self._llm.get_summary( + history_text, max_words=800, keep_language=True, memory=self.memory + ) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS ) From 6848d189cfd5c5d5df05d53fb825c64a85121090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 14:15:03 +0800 Subject: [PATCH 311/398] feat: + summary --- metagpt/provider/metagpt_llm_api.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index f8c4ac1ed..0688e1878 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -27,12 +27,12 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): RateLimiter.__init__(self, rpm=self.rpm) def __init_openai(self, *args, **kwargs): - openai.api_key = CONFIG.METAGPT_API_KEY - if CONFIG.METAGPT_API_BASE: - openai.api_base = CONFIG.METAGPT_API_BASE - if CONFIG.METAGPT_API_TYPE: - openai.api_type = CONFIG.METAGPT_API_TYPE - openai.api_version = CONFIG.METAGPT_API_VERSION + # openai.api_key = CONFIG.METAGPT_API_KEY + # if CONFIG.METAGPT_API_BASE: + # openai.api_base = CONFIG.METAGPT_API_BASE + # if CONFIG.METAGPT_API_TYPE: + # openai.api_type = CONFIG.METAGPT_API_TYPE + # openai.api_version = CONFIG.METAGPT_API_VERSION self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs): From 451b3510552fc22eef69b4e6f44e0a4caea7f75a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 14:18:19 +0800 Subject: [PATCH 312/398] feat: + summary --- metagpt/provider/metagpt_llm_api.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 0688e1878..17c2b3ab8 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -20,20 +20,11 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): - self.__init_openai() self.llm = openai self.model = CONFIG.METAGPT_API_MODEL self.auto_max_tokens = False - RateLimiter.__init__(self, rpm=self.rpm) - - def __init_openai(self, *args, **kwargs): - # openai.api_key = CONFIG.METAGPT_API_KEY - # if CONFIG.METAGPT_API_BASE: - # openai.api_base = CONFIG.METAGPT_API_BASE - # if CONFIG.METAGPT_API_TYPE: - # openai.api_type = CONFIG.METAGPT_API_TYPE - # openai.api_version = CONFIG.METAGPT_API_VERSION self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 + RateLimiter.__init__(self, rpm=self.rpm) async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs): summary = [] From 098027d249e709c4a939d8feb042e76f26ef0116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 14:23:36 +0800 Subject: [PATCH 313/398] feat: + summary --- metagpt/const.py | 6 ------ metagpt/provider/metagpt_llm_api.py | 9 +-------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index e9fa118d7..2323e3b6d 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -48,12 +48,6 @@ BRAIN_MEMORY = "BRAIN_MEMORY" SKILL_PATH = "SKILL_PATH" SERPER_API_KEY = "SERPER_API_KEY" -# Key Definitions for MetaGPT LLM -METAGPT_API_MODEL = "METAGPT_API_MODEL" -METAGPT_API_KEY = "METAGPT_API_KEY" -METAGPT_API_BASE = "METAGPT_API_BASE" -METAGPT_API_TYPE = "METAGPT_API_TYPE" -METAGPT_API_VERSION = "METAGPT_API_VERSION" # format BASE64_FORMAT = "base64" diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 17c2b3ab8..c21ffd650 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -7,24 +7,17 @@ """ import json -import openai from pydantic import BaseModel -from metagpt.config import CONFIG from metagpt.memory.brain_memory import BrainMemory from metagpt.provider import OpenAIGPTAPI -from metagpt.provider.openai_api import RateLimiter class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): - self.llm = openai - self.model = CONFIG.METAGPT_API_MODEL - self.auto_max_tokens = False - self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 - RateLimiter.__init__(self, rpm=self.rpm) + super().__init__() async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs): summary = [] From 239f68d40d1c49b94736344a94e8459fee43c535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 14:30:36 +0800 Subject: [PATCH 314/398] feat: + summary --- metagpt/actions/action.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index c52caaa40..92608f448 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -15,7 +15,6 @@ from typing import Optional from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput -from metagpt.llm import LLMFactory from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.common import OutputParser @@ -25,6 +24,8 @@ class Action(ABC): def __init__(self, name: str = "", context=None, llm: BaseGPTAPI = None): self.name: str = name if llm is None: + from metagpt.llm import LLMFactory + llm = LLMFactory.new_llm() self.llm = llm self.context = context From bda4132a9062a995616b1fd6ac93d35d218812dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 14:59:40 +0800 Subject: [PATCH 315/398] feat: + summary --- metagpt/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 8f8e4030f..9bf85174b 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -76,6 +76,7 @@ class Message: "sent_from": self.sent_from, "send_to": self.send_to, "tags": self.tags, + "id": self.id, } m = {"content": self.content} From 7723df1455b2a646e431d721ca9326d7872bb67a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 15:55:50 +0800 Subject: [PATCH 316/398] feat: + summary --- metagpt/provider/metagpt_llm_api.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index c21ffd650..06476f63b 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -6,10 +6,10 @@ @Desc : MetaGPT LLM related APIs """ import json +from typing import Dict, List from pydantic import BaseModel -from metagpt.memory.brain_memory import BrainMemory from metagpt.provider import OpenAIGPTAPI @@ -19,17 +19,22 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): def __init__(self): super().__init__() - async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs): + async def get_summary(self, history: List[Dict], max_words=200, keep_language: bool = False, **kwargs): summary = [] + class HisMsg(BaseModel): + content: str + tags: set + id: str + class QuweryAnswerPair(BaseModel): ask: str answer: str - rh = reversed(memory.history) + rh = reversed(history) ix = 0 while ix < len(rh): - t = rh[ix] + t = HisMsg(**rh[ix]) print(t) # 如果 t是ask, continue pass From a0ad7872f7dfc946ddc27d5e106fc3ab82130dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 15:57:24 +0800 Subject: [PATCH 317/398] feat: + summary --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 0a796ac11..5d04c2d6f 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -122,7 +122,7 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_summary( - history_text, max_words=800, keep_language=True, memory=self.memory + history_text, max_words=800, keep_language=True, history=self.memory.history ) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS From 8a0644a496fb956106dbcbc5697cd48617b85009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 15:58:13 +0800 Subject: [PATCH 318/398] feat: + summary --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 5d04c2d6f..2f9059210 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -122,7 +122,7 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_summary( - history_text, max_words=800, keep_language=True, history=self.memory.history + text=history_text, max_words=800, keep_language=True, history=self.memory.history ) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS From 4c9a5d8dda1238dbe0056243d3ef1860f6be0d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 16:45:01 +0800 Subject: [PATCH 319/398] feat: truncated history --- metagpt/provider/metagpt_llm_api.py | 53 ++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 06476f63b..d8d06aeaa 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -10,34 +10,55 @@ from typing import Dict, List from pydantic import BaseModel +from metagpt.memory.brain_memory import MessageType from metagpt.provider import OpenAIGPTAPI +class HisMsg(BaseModel): + content: str + tags: set + id: str + + +class Conversion(BaseModel): + """See: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb""" + + role: str + content: str + + class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): super().__init__() - async def get_summary(self, history: List[Dict], max_words=200, keep_language: bool = False, **kwargs): + async def get_summary(self, history: List[Dict], max_words=200, keep_language: bool = False, **kwargs) -> str: + """ + Return string in the following format： + [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Knock knock."}, + {"role": "assistant", "content": "Who's there?"}, + {"role": "user", "content": "Orange."}, + ] + """ summary = [] - class HisMsg(BaseModel): - content: str - tags: set - id: str + total_length = 0 + for m in reversed(history): + msg = HisMsg(**m) + c = Conversion(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) + length_delta = len(msg.content) + if total_length + length_delta > max_words: + left = max_words - total_length + if left > 0: + c.content = msg.content[0:left] + summary.insert(0, c.dict()) + break - class QuweryAnswerPair(BaseModel): - ask: str - answer: str - - rh = reversed(history) - ix = 0 - while ix < len(rh): - t = HisMsg(**rh[ix]) - print(t) - # 如果 t是ask, continue - pass + total_length += length_delta + summary.insert(0, c.dict()) data = json.dumps(summary) return data From 05532426c08a250ec4d7661fbecf79bd918b1ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 17:02:11 +0800 Subject: [PATCH 320/398] feat: truncated history --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index f309b532e..d83611af1 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -107,7 +107,7 @@ class BrainMemory(pydantic.BaseModel): def add_history(self, msg: Message): if msg.id: - if self.to_int(msg.id, 0) < self.to_int(self.last_history_id, -1): + if self.to_int(msg.id, 0) <= self.to_int(self.last_history_id, -1): return self.history.append(msg.dict()) self.is_dirty = True From 92402bedd4e2fe171e9ee9732b9ad120075e0da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 17:09:12 +0800 Subject: [PATCH 321/398] feat: truncated history --- metagpt/memory/brain_memory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index d83611af1..04ae6593a 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -110,6 +110,7 @@ class BrainMemory(pydantic.BaseModel): if self.to_int(msg.id, 0) <= self.to_int(self.last_history_id, -1): return self.history.append(msg.dict()) + self.last_history_id = str(msg.id) self.is_dirty = True def exists(self, text) -> bool: From 4c82298e8864f9e8f3712aa9bb6333079a015749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 18:21:10 +0800 Subject: [PATCH 322/398] feat: truncated history --- metagpt/memory/brain_memory.py | 62 ++++++++++++++++++++++++----- metagpt/provider/metagpt_llm_api.py | 41 ++----------------- metagpt/roles/assistant.py | 2 +- 3 files changed, 56 insertions(+), 49 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 04ae6593a..e8a98c55b 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -15,6 +15,7 @@ import pydantic from metagpt import Message from metagpt.logs import logger +from metagpt.schema import RawMessage from metagpt.utils.redis import Redis @@ -54,17 +55,21 @@ class BrainMemory(pydantic.BaseModel): def history_text(self): if len(self.history) == 0 and not self.historical_summary: return "" - texts = [self.historical_summary] if self.historical_summary else [] - for m in self.history[:-1]: - if isinstance(m, Dict): - t = Message(**m).content - elif isinstance(m, Message): - t = m.content - else: - continue - texts.append(t) + try: + self.loads_raw_messages() + return self.dumps_raw_messages() + except: + texts = [self.historical_summary] if self.historical_summary else [] + for m in self.history[:-1]: + if isinstance(m, Dict): + t = Message(**m).content + elif isinstance(m, Message): + t = m.content + else: + continue + texts.append(t) - return "\n".join(texts) + return "\n".join(texts) @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": @@ -130,3 +135,40 @@ class BrainMemory(pydantic.BaseModel): v = self.last_talk self.last_talk = None return v + + def loads_raw_messages(self): + if not self.historical_summary: + return + vv = json.loads(self.historical_summary) + msgs = [] + for v in vv: + tag = set([MessageType.Talk.value]) if v.get("role") == "user" else set([MessageType.Answer.value]) + m = Message(content=v.get("content"), tags=tag) + msgs.append(m) + msgs.extend(self.history) + self.history = msgs + self.is_dirty = True + + def dumps_raw_messages(self, max_length: int = 0) -> str: + summary = [] + + total_length = 0 + for m in reversed(self.history): + msg = Message(**m) + c = RawMessage(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) + length_delta = len(msg.content) + if max_length > 0: + if total_length + length_delta > max_length: + left = max_length - total_length + if left > 0: + c.content = msg.content[0:left] + summary.insert(0, c) + break + + total_length += length_delta + summary.insert(0, c) + + self.historical_summary = json.dumps(summary) + self.history = [] + self.is_dirty = True + return self.historical_summary diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index d8d06aeaa..3ae65a623 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -5,35 +5,18 @@ @File : metagpt_llm_api.py @Desc : MetaGPT LLM related APIs """ -import json -from typing import Dict, List -from pydantic import BaseModel - -from metagpt.memory.brain_memory import MessageType +from metagpt.memory.brain_memory import BrainMemory from metagpt.provider import OpenAIGPTAPI -class HisMsg(BaseModel): - content: str - tags: set - id: str - - -class Conversion(BaseModel): - """See: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb""" - - role: str - content: str - - class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): super().__init__() - async def get_summary(self, history: List[Dict], max_words=200, keep_language: bool = False, **kwargs) -> str: + async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs) -> str: """ Return string in the following format： [ @@ -43,22 +26,4 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): {"role": "user", "content": "Orange."}, ] """ - summary = [] - - total_length = 0 - for m in reversed(history): - msg = HisMsg(**m) - c = Conversion(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) - length_delta = len(msg.content) - if total_length + length_delta > max_words: - left = max_words - total_length - if left > 0: - c.content = msg.content[0:left] - summary.insert(0, c.dict()) - break - - total_length += length_delta - summary.insert(0, c.dict()) - - data = json.dumps(summary) - return data + return memory.dumps_raw_messages(max_length=max_words) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 2f9059210..2fcb6f584 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -122,7 +122,7 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_summary( - text=history_text, max_words=800, keep_language=True, history=self.memory.history + text=history_text, max_words=800, keep_language=True, memory=self.memory ) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS From 530d2f5b308a9c280853a20f51c2fac929c95134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:03:41 +0800 Subject: [PATCH 323/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 113 +++++++++++++++++++++++++++++++++ metagpt/provider/openai_api.py | 110 -------------------------------- metagpt/roles/assistant.py | 20 +++--- 3 files changed, 123 insertions(+), 120 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index e8a98c55b..7eda9c601 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -8,12 +8,16 @@ @Modified By: mashenquan, 2023/9/4. + redis memory cache. """ import json +import re from enum import Enum from typing import Dict, List +import openai import pydantic from metagpt import Message +from metagpt.config import CONFIG +from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS from metagpt.logs import logger from metagpt.schema import RawMessage from metagpt.utils.redis import Redis @@ -36,6 +40,7 @@ class BrainMemory(pydantic.BaseModel): last_history_id: str = "" is_dirty: bool = False last_talk: str = None + llm_type: str def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -172,3 +177,111 @@ class BrainMemory(pydantic.BaseModel): self.history = [] self.is_dirty = True return self.historical_summary + + async def get_summary(self, text: str, llm, max_words=200, keep_language: bool = False, **kwargs): + max_token_count = DEFAULT_MAX_TOKENS + max_count = 100 + text_length = len(text) + while max_count > 0: + if text_length < max_token_count: + return await self._get_summary(text=text, llm=llm, max_words=max_words, keep_language=keep_language) + + padding_size = 20 if max_token_count > 20 else 0 + text_windows = self.split_texts(text, window_size=max_token_count - padding_size) + part_max_words = min(int(max_words / len(text_windows)) + 1, 100) + summaries = [] + for ws in text_windows: + response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) + summaries.append(response) + if len(summaries) == 1: + return summaries[0] + + # Merged and retry + text = "\n".join(summaries) + text_length = len(text) + + max_count -= 1 # safeguard + raise openai.error.InvalidRequestError("text too long") + + async def _get_summary(self, text: str, llm, max_words=20, keep_language: bool = False): + """Generate text summary""" + if len(text) < max_words: + return text + if keep_language: + command = f".Translate the above content into a summary of less than {max_words} words in language of the content strictly." + else: + command = f"Translate the above content into a summary of less than {max_words} words." + msg = text + "\n\n" + command + logger.debug(f"summary ask:{msg}") + response = await llm.aask(msg=msg, system_msgs=[]) + logger.debug(f"summary rsp: {response}") + return response + + async def get_title(self, text: str, llm, max_words=5, **kwargs) -> str: + """Generate text title""" + summary = await self.get_summary(text, max_words=500) + + language = CONFIG.language or DEFAULT_LANGUAGE + command = f"Translate the above summary into a {language} title of less than {max_words} words." + summaries = [summary, command] + msg = "\n".join(summaries) + logger.debug(f"title ask:{msg}") + response = await llm.aask(msg=msg, system_msgs=[]) + logger.debug(f"title rsp: {response}") + return response + + async def is_related(self, text1, text2, llm): + # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." + rsp = await llm.aask(msg=command, system_msgs=[]) + result = True if "TRUE" in rsp else False + p2 = text2.replace("\n", "") + p1 = text1.replace("\n", "") + logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}\n") + return result + + async def rewrite(self, sentence: str, context: str, llm): + # command = ( + # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + # ) + command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" + rsp = await llm.aask(msg=command, system_msgs=[]) + logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}\n") + return rsp + + @staticmethod + def split_texts(text: str, window_size) -> List[str]: + """Splitting long text into sliding windows text""" + if window_size <= 0: + window_size = BrainMemory.DEFAULT_TOKEN_SIZE + total_len = len(text) + if total_len <= window_size: + return [text] + + padding_size = 20 if window_size > 20 else 0 + windows = [] + idx = 0 + data_len = window_size - padding_size + while idx < total_len: + if window_size + idx > total_len: # 不足一个滑窗 + windows.append(text[idx:]) + break + # 每个窗口少算padding_size自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] + # window_size=3, padding_size=1： + # [1, 2, 3], [3, 4, 5], [5, 6, 7], .... + # idx=2, | idx=5 | idx=8 | ... + w = text[idx : idx + window_size] + windows.append(w) + idx += data_len + + return windows + + @staticmethod + def extract_info(input_string, pattern=r"\[([A-Z]+)\]:\s*(.+)"): + match = re.match(pattern, input_string) + if match: + return match.group(1), match.group(2) + else: + return None, input_string + + DEFAULT_TOKEN_SIZE = 500 diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 64267975e..231b568c7 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -8,10 +8,8 @@ """ import asyncio import random -import re import time import traceback -from typing import List import openai from openai.error import APIConnectionError @@ -24,7 +22,6 @@ from tenacity import ( ) from metagpt.config import CONFIG -from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.cost_manager import Costs @@ -223,112 +220,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs): - max_token_count = DEFAULT_MAX_TOKENS - max_count = 100 - text_length = len(text) - while max_count > 0: - if text_length < max_token_count: - return await self._get_summary(text=text, max_words=max_words, keep_language=keep_language) - - padding_size = 20 if max_token_count > 20 else 0 - text_windows = self.split_texts(text, window_size=max_token_count - padding_size) - part_max_words = min(int(max_words / len(text_windows)) + 1, 100) - summaries = [] - for ws in text_windows: - response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) - summaries.append(response) - if len(summaries) == 1: - return summaries[0] - - # Merged and retry - text = "\n".join(summaries) - text_length = len(text) - - max_count -= 1 # safeguard - raise openai.error.InvalidRequestError("text too long") - - async def _get_summary(self, text: str, max_words=20, keep_language: bool = False): - """Generate text summary""" - if len(text) < max_words: - return text - if keep_language: - command = f".Translate the above content into a summary of less than {max_words} words in language of the content strictly." - else: - command = f"Translate the above content into a summary of less than {max_words} words." - msg = text + "\n\n" + command - logger.debug(f"summary ask:{msg}") - response = await self.aask(msg=msg, system_msgs=[]) - logger.debug(f"summary rsp: {response}") - return response - - async def get_context_title(self, text: str, max_words=5, **kwargs) -> str: - """Generate text title""" - summary = await self.get_summary(text, max_words=500) - - language = CONFIG.language or DEFAULT_LANGUAGE - command = f"Translate the above summary into a {language} title of less than {max_words} words." - summaries = [summary, command] - msg = "\n".join(summaries) - logger.debug(f"title ask:{msg}") - response = await self.aask(msg=msg, system_msgs=[]) - logger.debug(f"title rsp: {response}") - return response - - async def is_related(self, text1, text2): - # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." - command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." - rsp = await self.aask(msg=command, system_msgs=[]) - result = True if "TRUE" in rsp else False - p2 = text2.replace("\n", "") - p1 = text1.replace("\n", "") - logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}\n") - return result - - async def rewrite(self, sentence: str, context: str): - # command = ( - # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" - # ) - command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" - rsp = await self.aask(msg=command, system_msgs=[]) - logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}\n") - return rsp - - @staticmethod - def split_texts(text: str, window_size) -> List[str]: - """Splitting long text into sliding windows text""" - if window_size <= 0: - window_size = OpenAIGPTAPI.DEFAULT_TOKEN_SIZE - total_len = len(text) - if total_len <= window_size: - return [text] - - padding_size = 20 if window_size > 20 else 0 - windows = [] - idx = 0 - data_len = window_size - padding_size - while idx < total_len: - if window_size + idx > total_len: # 不足一个滑窗 - windows.append(text[idx:]) - break - # 每个窗口少算padding_size自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] - # window_size=3, padding_size=1： - # [1, 2, 3], [3, 4, 5], [5, 6, 7], .... - # idx=2, | idx=5 | idx=8 | ... - w = text[idx : idx + window_size] - windows.append(w) - idx += data_len - - return windows - - @staticmethod - def extract_info(input_string, pattern=r"\[([A-Z]+)\]:\s*(.+)"): - match = re.match(pattern, input_string) - if match: - return match.group(1), match.group(2) - else: - return None, input_string - @staticmethod async def async_retry_call(func, *args, **kwargs): for i in range(OpenAIGPTAPI.MAX_TRY): @@ -371,7 +262,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 - DEFAULT_TOKEN_SIZE = 500 if __name__ == "__main__": diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 2fcb6f584..d5467cafb 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -121,23 +121,23 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_summary( - text=history_text, max_words=800, keep_language=True, memory=self.memory + history_summary = await self.memory.get_summary( + text=history_text, max_words=800, keep_language=True, llm=self._llm ) - await self.memory.set_history_summary( - history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS - ) - if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. - last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) + # await self.memory.set_history_summary( + # history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS + # ) + if last_talk and await self.memory.is_related( + text1=last_talk, text2=history_summary, llm=self._llm + ): # Merge relevant content. + last_talk = await self.memory.rewrite(sentence=last_talk, context=history_text, llm=self._llm) return last_talk return last_talk @staticmethod def extract_info(input_string): - from metagpt.provider.openai_api import OpenAIGPTAPI - - return OpenAIGPTAPI.extract_info(input_string) + return BrainMemory.extract_info(input_string) def get_memory(self) -> str: return self.memory.json() From 4c873a91584286ea8bfb37a635a37b82eb5b3b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:13:23 +0800 Subject: [PATCH 324/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 14 +++++++++++--- metagpt/roles/assistant.py | 20 +++++--------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 7eda9c601..fea3b2512 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -178,13 +178,16 @@ class BrainMemory(pydantic.BaseModel): self.is_dirty = True return self.historical_summary - async def get_summary(self, text: str, llm, max_words=200, keep_language: bool = False, **kwargs): + async def summerize(self, llm, max_words=200, keep_language: bool = False, **kwargs): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 + text = self.history_text text_length = len(text) + summary = "" while max_count > 0: if text_length < max_token_count: - return await self._get_summary(text=text, llm=llm, max_words=max_words, keep_language=keep_language) + summary = await self._get_summary(text=text, llm=llm, max_words=max_words, keep_language=keep_language) + break padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) @@ -194,13 +197,18 @@ class BrainMemory(pydantic.BaseModel): response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) summaries.append(response) if len(summaries) == 1: - return summaries[0] + summary = summaries[0] + break # Merged and retry text = "\n".join(summaries) text_length = len(text) max_count -= 1 # safeguard + if not summary: + await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS) + return summary + raise openai.error.InvalidRequestError("text too long") async def _get_summary(self, text: str, llm, max_words=20, keep_language: bool = False): diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index d5467cafb..26711486f 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -45,7 +45,7 @@ class Assistant(Role): name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs ) brain_memory = CONFIG.BRAIN_MEMORY - self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory() + self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory(llm_type=CONFIG.LLM_TYPE) skill_path = Path(CONFIG.SKILL_PATH) if CONFIG.SKILL_PATH else None self.skills = SkillLoader(skill_yaml_file_name=skill_path) @@ -83,7 +83,7 @@ class Assistant(Role): self.memory.add_talk(Message(content=text)) async def _plan(self, rsp: str, **kwargs) -> bool: - skill, text = Assistant.extract_info(input_string=rsp) + skill, text = BrainMemory.extract_info(input_string=rsp) handlers = { MessageType.Talk.value: self.talk_handler, MessageType.Skill.value: self.skill_handler, @@ -121,24 +121,14 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self.memory.get_summary( - text=history_text, max_words=800, keep_language=True, llm=self._llm - ) - # await self.memory.set_history_summary( - # history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS - # ) - if last_talk and await self.memory.is_related( - text1=last_talk, text2=history_summary, llm=self._llm - ): # Merge relevant content. + history_summary = await self.memory.summerize(max_words=800, keep_language=True, llm=self._llm) + if last_talk and await self.memory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): + # Merge relevant content. last_talk = await self.memory.rewrite(sentence=last_talk, context=history_text, llm=self._llm) return last_talk return last_talk - @staticmethod - def extract_info(input_string): - return BrainMemory.extract_info(input_string) - def get_memory(self) -> str: return self.memory.json() From 44706ba1416805083caec6683787157ec8df38ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:23:09 +0800 Subject: [PATCH 325/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index fea3b2512..adb1f0114 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -80,7 +80,7 @@ class BrainMemory(pydantic.BaseModel): async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": redis = Redis(conf=redis_conf) if not redis.is_valid() or not redis_key: - return BrainMemory() + return BrainMemory(llm_type=CONFIG.LLM_TYPE) v = await redis.get(key=redis_key) logger.debug(f"REDIS GET {redis_key} {v}") if v: @@ -88,9 +88,11 @@ class BrainMemory(pydantic.BaseModel): bm = BrainMemory(**data) bm.is_dirty = False return bm - return BrainMemory() + return BrainMemory(llm_type=CONFIG.LLM_TYPE) async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60, redis_conf: Dict = None): + if not self.is_dirty: + return redis = Redis(conf=redis_conf) if not redis.is_valid() or not redis_key: return False From 948d1577e4a51f673768f8c16c51e378e435c732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:30:35 +0800 Subject: [PATCH 326/398] refactor: brain memory --- metagpt/memory/memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index bf9f0541c..f9dd5c1a3 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -8,7 +8,6 @@ from collections import defaultdict from typing import Iterable, Type -from metagpt.actions import Action from metagpt.schema import Message @@ -17,6 +16,8 @@ class Memory: def __init__(self): """Initialize an empty storage list and an empty index dictionary""" + from metagpt.actions import Action + self.storage: list[Message] = [] self.index: dict[Type[Action], list[Message]] = defaultdict(list) From c66012d087b9c80b207a256238142e6daf8e4a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:45:42 +0800 Subject: [PATCH 327/398] refactor: brain memory --- metagpt/memory/memory.py | 3 +-- metagpt/provider/metagpt_llm_api.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index f9dd5c1a3..bf9f0541c 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -8,6 +8,7 @@ from collections import defaultdict from typing import Iterable, Type +from metagpt.actions import Action from metagpt.schema import Message @@ -16,8 +17,6 @@ class Memory: def __init__(self): """Initialize an empty storage list and an empty index dictionary""" - from metagpt.actions import Action - self.storage: list[Message] = [] self.index: dict[Type[Action], list[Message]] = defaultdict(list) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 3ae65a623..95514cf53 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -6,7 +6,6 @@ @Desc : MetaGPT LLM related APIs """ -from metagpt.memory.brain_memory import BrainMemory from metagpt.provider import OpenAIGPTAPI @@ -16,7 +15,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): def __init__(self): super().__init__() - async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs) -> str: + async def get_summary(self, memory, max_words=200, keep_language: bool = False, **kwargs) -> str: """ Return string in the following format： [ From 0c21aa810f64743ac3a484d53c005bf654cbf3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:49:51 +0800 Subject: [PATCH 328/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index adb1f0114..596928a4c 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -10,7 +10,7 @@ import json import re from enum import Enum -from typing import Dict, List +from typing import Dict, List, Optional import openai import pydantic @@ -40,7 +40,7 @@ class BrainMemory(pydantic.BaseModel): last_history_id: str = "" is_dirty: bool = False last_talk: str = None - llm_type: str + llm_type: Optional[str] = None def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) From 415e6d5686a231201cd9c2a92c0cefdda12893ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 20:25:50 +0800 Subject: [PATCH 329/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 38 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 596928a4c..4f99de3c7 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -56,25 +56,25 @@ class BrainMemory(pydantic.BaseModel): texts = [Message(**m).content for m in self.knowledge] return "\n".join(texts) - @property - def history_text(self): - if len(self.history) == 0 and not self.historical_summary: - return "" - try: - self.loads_raw_messages() - return self.dumps_raw_messages() - except: - texts = [self.historical_summary] if self.historical_summary else [] - for m in self.history[:-1]: - if isinstance(m, Dict): - t = Message(**m).content - elif isinstance(m, Message): - t = m.content - else: - continue - texts.append(t) - - return "\n".join(texts) + # @property + # def history_text(self): + # if len(self.history) == 0 and not self.historical_summary: + # return "" + # try: + # self.loads_raw_messages() + # return self.dumps_raw_messages() + # except: + # texts = [self.historical_summary] if self.historical_summary else [] + # for m in self.history[:-1]: + # if isinstance(m, Dict): + # t = Message(**m).content + # elif isinstance(m, Message): + # t = m.content + # else: + # continue + # texts.append(t) + # + # return "\n".join(texts) @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": From 7abb1a3b9368c704dbec747755e107a72cc138ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 20:29:59 +0800 Subject: [PATCH 330/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 4f99de3c7..805ef1b27 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -44,12 +44,12 @@ class BrainMemory(pydantic.BaseModel): def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) - self.history.append(msg.dict()) + self.add_history(msg) self.is_dirty = True def add_answer(self, msg: Message): msg.add_tag(MessageType.Answer.value) - self.history.append(msg.dict()) + self.add_history(msg) self.is_dirty = True def get_knowledge(self) -> str: From c36e1d6f1a85c7d1fb4ad124efcfe2d40917d7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 20:41:46 +0800 Subject: [PATCH 331/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 805ef1b27..45a7c0691 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -103,7 +103,7 @@ class BrainMemory(pydantic.BaseModel): @staticmethod def to_redis_key(prefix: str, user_id: str, chat_id: str): - return f"{prefix}:{chat_id}:{user_id}" + return f"{prefix}:{user_id}:{chat_id}" async def set_history_summary(self, history_summary, redis_key, redis_conf): if self.historical_summary == history_summary: @@ -294,4 +294,9 @@ class BrainMemory(pydantic.BaseModel): else: return None, input_string + def set_llm_type(self, v): + if v: + self.llm_type = v + self.is_dirty = True + DEFAULT_TOKEN_SIZE = 500 From 2be79730a020e0c455810087eb2e771df9d59f11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 20:49:36 +0800 Subject: [PATCH 332/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 9 +++++++-- metagpt/roles/assistant.py | 7 +++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 45a7c0691..b06bf1036 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -240,7 +240,8 @@ class BrainMemory(pydantic.BaseModel): logger.debug(f"title rsp: {response}") return response - async def is_related(self, text1, text2, llm): + @staticmethod + async def is_related(text1, text2, llm): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await llm.aask(msg=command, system_msgs=[]) @@ -295,8 +296,12 @@ class BrainMemory(pydantic.BaseModel): return None, input_string def set_llm_type(self, v): - if v: + if v and v != self.llm_type: self.llm_type = v self.is_dirty = True + @property + def is_history_available(self): + return self.history or self.historical_summary + DEFAULT_TOKEN_SIZE = 500 diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 26711486f..54c1e2f43 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -115,16 +115,15 @@ class Assistant(Role): return True async def refine_memory(self) -> str: - history_text = self.memory.history_text last_talk = self.memory.pop_last_talk() if last_talk is None: # No user feedback, unsure if past conversation is finished. return None - if history_text == "": + if not self.memory.is_history_available: return last_talk history_summary = await self.memory.summerize(max_words=800, keep_language=True, llm=self._llm) - if last_talk and await self.memory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): + if last_talk and await BrainMemory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): # Merge relevant content. - last_talk = await self.memory.rewrite(sentence=last_talk, context=history_text, llm=self._llm) + last_talk = await self.memory.rewrite(sentence=last_talk, llm=self._llm) return last_talk return last_talk From 0703c29030587cb0c0b6a57907c5112c8fe84d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:21:03 +0800 Subject: [PATCH 333/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 103 ++++++++++++++------------------- 1 file changed, 44 insertions(+), 59 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index b06bf1036..a9677bd66 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -18,6 +18,7 @@ import pydantic from metagpt import Message from metagpt.config import CONFIG from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS +from metagpt.llm import LLMType from metagpt.logs import logger from metagpt.schema import RawMessage from metagpt.utils.redis import Redis @@ -56,26 +57,6 @@ class BrainMemory(pydantic.BaseModel): texts = [Message(**m).content for m in self.knowledge] return "\n".join(texts) - # @property - # def history_text(self): - # if len(self.history) == 0 and not self.historical_summary: - # return "" - # try: - # self.loads_raw_messages() - # return self.dumps_raw_messages() - # except: - # texts = [self.historical_summary] if self.historical_summary else [] - # for m in self.history[:-1]: - # if isinstance(m, Dict): - # t = Message(**m).content - # elif isinstance(m, Message): - # t = m.content - # else: - # continue - # texts.append(t) - # - # return "\n".join(texts) - @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": redis = Redis(conf=redis_conf) @@ -143,47 +124,19 @@ class BrainMemory(pydantic.BaseModel): self.last_talk = None return v - def loads_raw_messages(self): - if not self.historical_summary: - return - vv = json.loads(self.historical_summary) - msgs = [] - for v in vv: - tag = set([MessageType.Talk.value]) if v.get("role") == "user" else set([MessageType.Answer.value]) - m = Message(content=v.get("content"), tags=tag) - msgs.append(m) - msgs.extend(self.history) - self.history = msgs - self.is_dirty = True + async def summarize(self, llm, max_words=200, keep_language: bool = False, **kwargs): + if self.llm_type == LLMType.METAGPT.value: + return await self._metagpt_summarize(llm=llm, max_words=max_words, keep_language=keep_language, **kwargs) - def dumps_raw_messages(self, max_length: int = 0) -> str: - summary = [] + return await self._openai_summarize(llm=llm, max_words=max_words, keep_language=keep_language, **kwargs) - total_length = 0 - for m in reversed(self.history): - msg = Message(**m) - c = RawMessage(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) - length_delta = len(msg.content) - if max_length > 0: - if total_length + length_delta > max_length: - left = max_length - total_length - if left > 0: - c.content = msg.content[0:left] - summary.insert(0, c) - break - - total_length += length_delta - summary.insert(0, c) - - self.historical_summary = json.dumps(summary) - self.history = [] - self.is_dirty = True - return self.historical_summary - - async def summerize(self, llm, max_words=200, keep_language: bool = False, **kwargs): + async def _openai_summarize(self, llm, max_words=200, keep_language: bool = False, **kwargs): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 - text = self.history_text + texts = [self.historical_summary] + for m in self.history: + texts.append(m.content) + text = "\n".join(texts) text_length = len(text) summary = "" while max_count > 0: @@ -210,9 +163,41 @@ class BrainMemory(pydantic.BaseModel): if not summary: await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS) return summary - raise openai.error.InvalidRequestError("text too long") + async def _metagpt_summarize(self, max_words=200, **kwargs): + if not self.history: + return "" + + total_length = 0 + msgs = [] + for m in reversed(self.history): + delta = len(m.content) + if total_length + delta > max_words: + left = max_words - total_length + if left == 0: + break + m.content = m.content[0:left] + msgs.append(m) + break + msgs.append(m) + total_length += delta + self.history = msgs + self.is_dirty = True + await self.dumps(redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS_CONF) + self.is_dirty = False + + return BrainMemory.to_metagpt_history_format(self.history) + + @staticmethod + def to_metagpt_history_format(history) -> str: + mmsg = [] + for m in reversed(history): + msg = Message(**m) + r = RawMessage(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) + mmsg.append(r) + return json.dumps(mmsg) + async def _get_summary(self, text: str, llm, max_words=20, keep_language: bool = False): """Generate text summary""" if len(text) < max_words: @@ -302,6 +287,6 @@ class BrainMemory(pydantic.BaseModel): @property def is_history_available(self): - return self.history or self.historical_summary + return bool(self.history or self.historical_summary) DEFAULT_TOKEN_SIZE = 500 From 8e30dfd84a8e516fe7a6ad7d993a7883ec728b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:21:38 +0800 Subject: [PATCH 334/398] refactor: brain memory --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 54c1e2f43..66daef403 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -120,7 +120,7 @@ class Assistant(Role): return None if not self.memory.is_history_available: return last_talk - history_summary = await self.memory.summerize(max_words=800, keep_language=True, llm=self._llm) + history_summary = await self.memory.summarize(max_words=800, keep_language=True, llm=self._llm) if last_talk and await BrainMemory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): # Merge relevant content. last_talk = await self.memory.rewrite(sentence=last_talk, llm=self._llm) From 12b2fcd4be85b2dd013fc040d046dda938c38b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:30:15 +0800 Subject: [PATCH 335/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index a9677bd66..3d713ddfb 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -171,16 +171,17 @@ class BrainMemory(pydantic.BaseModel): total_length = 0 msgs = [] - for m in reversed(self.history): + for i in reversed(self.history): + m = Message(**i) delta = len(m.content) if total_length + delta > max_words: left = max_words - total_length if left == 0: break m.content = m.content[0:left] - msgs.append(m) + msgs.append(m.dict()) break - msgs.append(m) + msgs.append(m.dict()) total_length += delta self.history = msgs self.is_dirty = True @@ -198,7 +199,8 @@ class BrainMemory(pydantic.BaseModel): mmsg.append(r) return json.dumps(mmsg) - async def _get_summary(self, text: str, llm, max_words=20, keep_language: bool = False): + @staticmethod + async def _get_summary(text: str, llm, max_words=20, keep_language: bool = False): """Generate text summary""" if len(text) < max_words: return text @@ -214,7 +216,7 @@ class BrainMemory(pydantic.BaseModel): async def get_title(self, text: str, llm, max_words=5, **kwargs) -> str: """Generate text title""" - summary = await self.get_summary(text, max_words=500) + summary = await self.summarize(text, max_words=500) language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." From 24a3e725726338ed3b5a611489ce1af481692e2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:35:48 +0800 Subject: [PATCH 336/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 3d713ddfb..09a4915fc 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -181,7 +181,7 @@ class BrainMemory(pydantic.BaseModel): m.content = m.content[0:left] msgs.append(m.dict()) break - msgs.append(m.dict()) + msgs.append(i) total_length += delta self.history = msgs self.is_dirty = True From a4f36e0852f0804c327cc6cce016e00e28d0591c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:42:35 +0800 Subject: [PATCH 337/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 09a4915fc..e65459f1a 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -183,7 +183,7 @@ class BrainMemory(pydantic.BaseModel): break msgs.append(i) total_length += delta - self.history = msgs + self.history = msgs.reverse() self.is_dirty = True await self.dumps(redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS_CONF) self.is_dirty = False From 5b3f6e0b6857210dacf115e171418d5893afdcf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:43:28 +0800 Subject: [PATCH 338/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index e65459f1a..39e2ec43d 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -183,7 +183,8 @@ class BrainMemory(pydantic.BaseModel): break msgs.append(i) total_length += delta - self.history = msgs.reverse() + msgs.reverse() + self.history = msgs self.is_dirty = True await self.dumps(redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS_CONF) self.is_dirty = False From 1df4121b12863793b23dcd7a11d6855b35eb752d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 22:06:54 +0800 Subject: [PATCH 339/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 39e2ec43d..2d191ccaa 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -194,7 +194,7 @@ class BrainMemory(pydantic.BaseModel): @staticmethod def to_metagpt_history_format(history) -> str: mmsg = [] - for m in reversed(history): + for m in history: msg = Message(**m) r = RawMessage(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) mmsg.append(r) From 1ce9ad54fd6dbe52f726b8977b18aae19049f23c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 22:24:20 +0800 Subject: [PATCH 340/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 2d191ccaa..e0e2ae1a0 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -228,8 +228,17 @@ class BrainMemory(pydantic.BaseModel): logger.debug(f"title rsp: {response}") return response + async def is_related(self, text1, text2, llm): + if self.llm_type == LLMType.METAGPT.value: + return await self._metagpt_is_related(text1=text1, text2=text2, llm=llm) + return await self._openai_is_related(text1=text1, text2=text2, llm=llm) + @staticmethod - async def is_related(text1, text2, llm): + async def _metagpt_is_related(**kwargs): + return False + + @staticmethod + async def _openai_is_related(text1, text2, llm, **kwargs): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await llm.aask(msg=command, system_msgs=[]) @@ -240,6 +249,14 @@ class BrainMemory(pydantic.BaseModel): return result async def rewrite(self, sentence: str, context: str, llm): + if self.llm_type == LLMType.METAGPT.value: + return await self._metagpt_rewrite(sentence=sentence, context=context, llm=llm) + return await self._openai_rewrite(sentence=sentence, context=context, llm=llm) + + async def _metagpt_rewrite(self, sentence: str, **kwargs): + return sentence + + async def _openai_rewrite(self, sentence: str, context: str, llm, **kwargs): # command = ( # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" # ) From 7c6b0325d8a2001491d2ec25167ef638f417aa7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 22:27:49 +0800 Subject: [PATCH 341/398] refactor: brain memory --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 66daef403..397ddc94b 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -121,7 +121,7 @@ class Assistant(Role): if not self.memory.is_history_available: return last_talk history_summary = await self.memory.summarize(max_words=800, keep_language=True, llm=self._llm) - if last_talk and await BrainMemory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): + if last_talk and await self.memory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): # Merge relevant content. last_talk = await self.memory.rewrite(sentence=last_talk, llm=self._llm) return last_talk From f2da313548b07f81ce8e9299b2d96bb067ba7e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 22:58:00 +0800 Subject: [PATCH 342/398] refactor: brain memory --- metagpt/actions/talk_action.py | 11 +++++++++++ metagpt/memory/brain_memory.py | 24 ++++++++++++++++++++++++ metagpt/provider/base_gpt_api.py | 19 +++++++++++++++---- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 0e3762798..baef47eeb 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -6,10 +6,12 @@ @File : talk_action.py @Desc : Act as it’s a talk """ +import json from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG from metagpt.const import DEFAULT_LANGUAGE +from metagpt.llm import LLMType from metagpt.logs import logger @@ -63,6 +65,15 @@ class TalkAction(Action): return prompt async def run(self, *args, **kwargs) -> ActionOutput: + if CONFIG.LLM_TYPE == LLMType.METAGPT.value: + rsp = await self.llm.aask( + msg=self._talk, + knowledge_msgs=[{"knowledge": self._knowledge}] if self._knowledge else None, + history_msgs=json.loads(self._history_summary) if self._history_summary else None, + ) + self._rsp = ActionOutput(content=rsp) + return self._rsp + prompt = self.prompt rsp = await self.llm.aask(msg=prompt, system_msgs=[]) logger.debug(f"PROMPT:{prompt}\nRESULT:{rsp}\n") diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index e0e2ae1a0..0f9c1dbb6 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -309,4 +309,28 @@ class BrainMemory(pydantic.BaseModel): def is_history_available(self): return bool(self.history or self.historical_summary) + @property + def history_text(self): + if self.llm_type == LLMType.METAGPT.value: + return self._get_metagpt_history_text() + return self._get_openai_history_text() + + def _get_metagpt_history_text(self): + return BrainMemory.to_metagpt_history_format(self.history) + + def _get_openai_history_text(self): + if len(self.history) == 0 and not self.historical_summary: + return "" + texts = [self.historical_summary] if self.historical_summary else [] + for m in self.history[:-1]: + if isinstance(m, Dict): + t = Message(**m).content + elif isinstance(m, Message): + t = m.content + else: + continue + texts.append(t) + + return "\n".join(texts) + DEFAULT_TOKEN_SIZE = 500 diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 7351e6916..f405ae902 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -38,11 +38,22 @@ class BaseGPTAPI(BaseChatbot): rsp = self.completion(message) return self.get_choice_text(rsp) - async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, generator: bool = False) -> str: + async def aask( + self, + msg: str, + system_msgs: Optional[list[str]] = None, + history_msgs: Optional[list[dict[str, str]]] = None, + knowledge_msgs: Optional[list[dict[str, str]]] = None, + generator: bool = False, + ) -> str: + message = [] if system_msgs: - message = self._system_msgs(system_msgs) + [self._user_msg(msg)] - else: - message = [self._default_system_msg(), self._user_msg(msg)] + message = self._system_msgs(system_msgs) + if knowledge_msgs: + message.extend(knowledge_msgs) + if history_msgs: + message.extend(history_msgs) + message.append(self._user_msg(msg)) try: rsp = await self.acompletion_text(message, stream=True, generator=generator) except Exception as e: From f92aeb0e506e852d3551f7cf67b3574448e91712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:05:30 +0800 Subject: [PATCH 343/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 0f9c1dbb6..7677a9144 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -134,7 +134,8 @@ class BrainMemory(pydantic.BaseModel): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 texts = [self.historical_summary] - for m in self.history: + for i in self.history: + m = Message(**i) texts.append(m.content) text = "\n".join(texts) text_length = len(text) From 1b267d34dc986e8f18be63423783421d88e72eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:08:46 +0800 Subject: [PATCH 344/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 7677a9144..f3a3e3563 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -161,7 +161,7 @@ class BrainMemory(pydantic.BaseModel): text_length = len(text) max_count -= 1 # safeguard - if not summary: + if summary: await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS) return summary raise openai.error.InvalidRequestError("text too long") From 270b14e0360c41ede5fadaf71ea4b6e04b384dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:29:08 +0800 Subject: [PATCH 345/398] refactor: brain memory --- metagpt/actions/talk_action.py | 33 ++++++++++++++++++++++---------- metagpt/provider/base_gpt_api.py | 9 +++------ 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index baef47eeb..2d473c7cf 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -64,22 +64,35 @@ class TalkAction(Action): logger.info(f"PROMPT: {prompt}") return prompt - async def run(self, *args, **kwargs) -> ActionOutput: - if CONFIG.LLM_TYPE == LLMType.METAGPT.value: - rsp = await self.llm.aask( - msg=self._talk, - knowledge_msgs=[{"knowledge": self._knowledge}] if self._knowledge else None, - history_msgs=json.loads(self._history_summary) if self._history_summary else None, - ) - self._rsp = ActionOutput(content=rsp) - return self._rsp - + async def run_old(self, *args, **kwargs) -> ActionOutput: prompt = self.prompt rsp = await self.llm.aask(msg=prompt, system_msgs=[]) logger.debug(f"PROMPT:{prompt}\nRESULT:{rsp}\n") self._rsp = ActionOutput(content=rsp) return self._rsp + async def run(self, *args, **kwargs) -> ActionOutput: + language = CONFIG.language or DEFAULT_LANGUAGE + system_msgs = [ + f"You are {CONFIG.agent_description}.", + "Your responses should align with the role-play agreement, " + "maintaining the character's persona and habits. When faced with unrelated questions, playfully " + "decline to answer without revealing your AI nature to preserve the character's image.", + "If the information is insufficient, you can search in the context or knowledge.", + f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.", + ] + format_msgs = [] + if self._knowledge: + format_msgs.append({"knowledge": self._knowledge}) + if self._history_summary: + if CONFIG.LLM_TYPE == LLMType.METAGPT.value: + format_msgs.append(json.loads(self._history_summary)) + else: + format_msgs.append({"context": self._history_summary}) + rsp = await self.llm.aask(msg=self._talk, format_msgs=format_msgs, system_msgs=system_msgs) + self._rsp = ActionOutput(content=rsp) + return self._rsp + __FORMATION__ = """Formation: "Capacity and role" defines the role you are currently playing; "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index f405ae902..19f5fcfff 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -42,17 +42,14 @@ class BaseGPTAPI(BaseChatbot): self, msg: str, system_msgs: Optional[list[str]] = None, - history_msgs: Optional[list[dict[str, str]]] = None, - knowledge_msgs: Optional[list[dict[str, str]]] = None, + format_msgs: Optional[list[dict[str, str]]] = None, generator: bool = False, ) -> str: message = [] if system_msgs: message = self._system_msgs(system_msgs) - if knowledge_msgs: - message.extend(knowledge_msgs) - if history_msgs: - message.extend(history_msgs) + if format_msgs: + message.extend(format_msgs) message.append(self._user_msg(msg)) try: rsp = await self.acompletion_text(message, stream=True, generator=generator) From b49c7f2d70e7b7f45d4e632c203b9fcecbfe52ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:38:04 +0800 Subject: [PATCH 346/398] refactor: brain memory --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 2d473c7cf..3c3db0841 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -88,7 +88,7 @@ class TalkAction(Action): if CONFIG.LLM_TYPE == LLMType.METAGPT.value: format_msgs.append(json.loads(self._history_summary)) else: - format_msgs.append({"context": self._history_summary}) + format_msgs.append({"knowledge": self._history_summary}) rsp = await self.llm.aask(msg=self._talk, format_msgs=format_msgs, system_msgs=system_msgs) self._rsp = ActionOutput(content=rsp) return self._rsp From d906bd1c81d534dbd1edee9d03e9e556a2805c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:41:31 +0800 Subject: [PATCH 347/398] refactor: brain memory --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 3c3db0841..b5282c3e5 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -83,12 +83,12 @@ class TalkAction(Action): ] format_msgs = [] if self._knowledge: - format_msgs.append({"knowledge": self._knowledge}) + format_msgs.append({"role": "knowledge", "content": self._knowledge}) if self._history_summary: if CONFIG.LLM_TYPE == LLMType.METAGPT.value: format_msgs.append(json.loads(self._history_summary)) else: - format_msgs.append({"knowledge": self._history_summary}) + format_msgs.append({"role": "context", "content": self._history_summary}) rsp = await self.llm.aask(msg=self._talk, format_msgs=format_msgs, system_msgs=system_msgs) self._rsp = ActionOutput(content=rsp) return self._rsp From b1f7aa396895723b121f184d2fff559a72eb52be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:44:01 +0800 Subject: [PATCH 348/398] refactor: brain memory --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index b5282c3e5..85d99db49 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -83,12 +83,12 @@ class TalkAction(Action): ] format_msgs = [] if self._knowledge: - format_msgs.append({"role": "knowledge", "content": self._knowledge}) + format_msgs.append({"role": "assistant", "content": self._knowledge}) if self._history_summary: if CONFIG.LLM_TYPE == LLMType.METAGPT.value: format_msgs.append(json.loads(self._history_summary)) else: - format_msgs.append({"role": "context", "content": self._history_summary}) + format_msgs.append({"role": "assistant", "content": self._history_summary}) rsp = await self.llm.aask(msg=self._talk, format_msgs=format_msgs, system_msgs=system_msgs) self._rsp = ActionOutput(content=rsp) return self._rsp From f4eea02866cc76d9e3ceb809c466451abae91af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:54:56 +0800 Subject: [PATCH 349/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index f3a3e3563..cdf3d7fbb 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -164,7 +164,7 @@ class BrainMemory(pydantic.BaseModel): if summary: await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS) return summary - raise openai.error.InvalidRequestError("text too long") + raise openai.error.InvalidRequestError(message="text too long", param=None) async def _metagpt_summarize(self, max_words=200, **kwargs): if not self.history: From 20fb71b0a3a6f2abebea6d81edf73c0a59f26afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:58:30 +0800 Subject: [PATCH 350/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index cdf3d7fbb..3dfa050b3 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -216,9 +216,9 @@ class BrainMemory(pydantic.BaseModel): logger.debug(f"summary rsp: {response}") return response - async def get_title(self, text: str, llm, max_words=5, **kwargs) -> str: + async def get_title(self, llm, max_words=5, **kwargs) -> str: """Generate text title""" - summary = await self.summarize(text, max_words=500) + summary = await self.summarize(max_words=500) language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." From 42d0281fbbba5ccd8f5646c2e7303ba1d5aa6f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 00:00:41 +0800 Subject: [PATCH 351/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 3dfa050b3..a995244a6 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -218,7 +218,7 @@ class BrainMemory(pydantic.BaseModel): async def get_title(self, llm, max_words=5, **kwargs) -> str: """Generate text title""" - summary = await self.summarize(max_words=500) + summary = await self.summarize(llm=llm, max_words=500) language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." From 8b5d83956d7cbc852fbeeb6e4006bc8d0712088e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:05:44 +0800 Subject: [PATCH 352/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 10 +++++++--- metagpt/provider/openai_api.py | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index a995244a6..9878fa750 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -124,13 +124,15 @@ class BrainMemory(pydantic.BaseModel): self.last_talk = None return v - async def summarize(self, llm, max_words=200, keep_language: bool = False, **kwargs): + async def summarize(self, llm, max_words=200, keep_language: bool = False, limit: int = -1, **kwargs): if self.llm_type == LLMType.METAGPT.value: return await self._metagpt_summarize(llm=llm, max_words=max_words, keep_language=keep_language, **kwargs) - return await self._openai_summarize(llm=llm, max_words=max_words, keep_language=keep_language, **kwargs) + return await self._openai_summarize( + llm=llm, max_words=max_words, keep_language=keep_language, limit=limit, **kwargs + ) - async def _openai_summarize(self, llm, max_words=200, keep_language: bool = False, **kwargs): + async def _openai_summarize(self, llm, max_words=200, keep_language: bool = False, limit: int = -1, **kwargs): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 texts = [self.historical_summary] @@ -139,6 +141,8 @@ class BrainMemory(pydantic.BaseModel): texts.append(m.content) text = "\n".join(texts) text_length = len(text) + if limit > 0 and text_length < limit: + return text summary = "" while max_count > 0: if text_length < max_token_count: diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 231b568c7..9dbbaf7e5 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -22,7 +22,9 @@ from tenacity import ( ) from metagpt.config import CONFIG +from metagpt.llm import LLMType from metagpt.logs import logger +from metagpt.memory.brain_memory import BrainMemory from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.cost_manager import Costs from metagpt.utils.token_counter import ( @@ -261,6 +263,19 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise e raise openai.error.OpenAIError("Exceeds the maximum retries") + async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs) -> str: + """ + Return string in the following format： + [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Knock knock."}, + {"role": "assistant", "content": "Who's there?"}, + {"role": "user", "content": "Orange."}, + ] + """ + memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text) + return await memory.summarize(llm=self._llm, max_length=max_words, keep_language=keep_language) + MAX_TRY = 5 @@ -269,4 +284,3 @@ if __name__ == "__main__": as dfas sad lkf sdkl sakdfsdk sjd jsk sdl sk dd sd asd fa sdf sad dd - .gitlab-ci.yml & base_test.py """ - OpenAIGPTAPI.split_texts(txt, 30) From 827505fca9838f7df57970174ae018df911f258d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:07:46 +0800 Subject: [PATCH 353/398] refactor: brain memory --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 9dbbaf7e5..85dfe8436 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -274,7 +274,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): ] """ memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text) - return await memory.summarize(llm=self._llm, max_length=max_words, keep_language=keep_language) + return await memory.summarize(llm=self, max_length=max_words, keep_language=keep_language) MAX_TRY = 5 From 6942cc91619e35a626cbfc5b33f5e27f856ebc42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:16:07 +0800 Subject: [PATCH 354/398] refactor: brain memory --- metagpt/llm.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 4772d2e6e..67ae42d62 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -8,15 +8,15 @@ """ from enum import Enum +import openai + from metagpt.config import CONFIG -from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI as MetaGPT_LLM -from metagpt.provider.openai_api import OpenAIGPTAPI as OpenAI_LLM class LLMType(Enum): OPENAI = "OpenAI" METAGPT = "MetaGPT" + CLAUDE = "Claude" UNKNOWN = "UNKNOWN" @classmethod @@ -27,20 +27,18 @@ class LLMType(Enum): return cls.UNKNOWN -DEFAULT_LLM = OpenAI_LLM() -DEFAULT_METAGPT_LLM = MetaGPT_LLM() -CLAUDE_LLM = Claude() - - -async def ai_func(prompt): - """使用LLM进行QA - QA with LLMs - """ - return await DEFAULT_LLM.aask(prompt) - - class LLMFactory: @staticmethod def new_llm() -> object: - llm = OpenAI_LLM() if CONFIG.LLM_TYPE == LLMType.OPENAI.value else MetaGPT_LLM() - return llm + from metagpt.provider.anthropic_api import Claude2 as Claude + from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI as MetaGPT_LLM + from metagpt.provider.openai_api import OpenAIGPTAPI as OpenAI_LLM + + if CONFIG.LLM_TYPE == LLMType.OPENAI.value: + return OpenAI_LLM() + if CONFIG.LLM_TYPE == LLMType.METAGPT.value: + return MetaGPT_LLM() + if CONFIG.LLM_TYPE == LLMType.CLAUDE.value: + return Claude() + + raise openai.InvalidRequestError(message=f"Unsupported LLM TYPE: {CONFIG.LLM_TYPE}") From 525ca29c89d7279f082f0e7d237a6445dbdd61df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:17:52 +0800 Subject: [PATCH 355/398] refactor: brain memory --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 85dfe8436..de640aed7 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -24,7 +24,6 @@ from tenacity import ( from metagpt.config import CONFIG from metagpt.llm import LLMType from metagpt.logs import logger -from metagpt.memory.brain_memory import BrainMemory from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.cost_manager import Costs from metagpt.utils.token_counter import ( @@ -273,6 +272,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): {"role": "user", "content": "Orange."}, ] """ + from metagpt.memory.brain_memory import BrainMemory + memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text) return await memory.summarize(llm=self, max_length=max_words, keep_language=keep_language) From 1254f93467ca8cd9cad34e3c6791ce9ffef3d633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:22:31 +0800 Subject: [PATCH 356/398] refactor: brain memory --- metagpt/memory/brain_memory.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 9878fa750..b8f9a2a15 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -42,6 +42,7 @@ class BrainMemory(pydantic.BaseModel): is_dirty: bool = False last_talk: str = None llm_type: Optional[str] = None + cacheable: bool = True def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -78,8 +79,9 @@ class BrainMemory(pydantic.BaseModel): if not redis.is_valid() or not redis_key: return False v = self.json() - await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) - logger.debug(f"REDIS SET {redis_key} {v}") + if self.cacheable: + await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) + logger.debug(f"REDIS SET {redis_key} {v}") self.is_dirty = False @staticmethod From 348cafa0b86096f96b4cb41fef197f04b5814256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:24:08 +0800 Subject: [PATCH 357/398] refactor: brain memory --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index de640aed7..514671488 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -274,7 +274,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ from metagpt.memory.brain_memory import BrainMemory - memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text) + memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text, cacheable=False) return await memory.summarize(llm=self, max_length=max_words, keep_language=keep_language) MAX_TRY = 5 From bed3d8c841dcd1c6901e9ecfcac5e855b4413164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 11:54:26 +0800 Subject: [PATCH 358/398] refactor: brain memory --- metagpt/actions/talk_action.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 85d99db49..f9ff76015 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -71,7 +71,8 @@ class TalkAction(Action): self._rsp = ActionOutput(content=rsp) return self._rsp - async def run(self, *args, **kwargs) -> ActionOutput: + @property + def aask_args(self): language = CONFIG.language or DEFAULT_LANGUAGE system_msgs = [ f"You are {CONFIG.agent_description}.", @@ -89,7 +90,11 @@ class TalkAction(Action): format_msgs.append(json.loads(self._history_summary)) else: format_msgs.append({"role": "assistant", "content": self._history_summary}) - rsp = await self.llm.aask(msg=self._talk, format_msgs=format_msgs, system_msgs=system_msgs) + return self._talk, format_msgs, system_msgs + + async def run(self, *args, **kwargs) -> ActionOutput: + msg, format_msgs, system_msgs = self.aask_args + rsp = await self.llm.aask(msg=msg, format_msgs=format_msgs, system_msgs=system_msgs) self._rsp = ActionOutput(content=rsp) return self._rsp From 5f3931820ec17d2de2aeef77b4294bfd3dc67b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 12:08:05 +0800 Subject: [PATCH 359/398] refactor: brain memory --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index f9ff76015..eb619cb7e 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -87,7 +87,7 @@ class TalkAction(Action): format_msgs.append({"role": "assistant", "content": self._knowledge}) if self._history_summary: if CONFIG.LLM_TYPE == LLMType.METAGPT.value: - format_msgs.append(json.loads(self._history_summary)) + format_msgs.extend(json.loads(self._history_summary)) else: format_msgs.append({"role": "assistant", "content": self._history_summary}) return self._talk, format_msgs, system_msgs From 5903b3efbc33f3ec5ba68953980dac4c5c83dd3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 13:03:05 +0800 Subject: [PATCH 360/398] refactor: brain memory --- metagpt/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 67ae42d62..eeb665872 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -41,4 +41,4 @@ class LLMFactory: if CONFIG.LLM_TYPE == LLMType.CLAUDE.value: return Claude() - raise openai.InvalidRequestError(message=f"Unsupported LLM TYPE: {CONFIG.LLM_TYPE}") + raise openai.InvalidRequestError(message=f"Unsupported LLM TYPE: {CONFIG.LLM_TYPE}", param=None) From ce6619a10c5aac43a715cfb53a6844c3c732e7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 13:07:21 +0800 Subject: [PATCH 361/398] refactor: brain memory --- metagpt/provider/base_gpt_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 19f5fcfff..59da67d5b 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -48,6 +48,8 @@ class BaseGPTAPI(BaseChatbot): message = [] if system_msgs: message = self._system_msgs(system_msgs) + else: + message = [self._default_system_msg()] if format_msgs: message.extend(format_msgs) message.append(self._user_msg(msg)) From 1b71081c745f469a5f4529c30558f565a59bbe8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 13:08:29 +0800 Subject: [PATCH 362/398] refactor: brain memory --- metagpt/provider/base_gpt_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 59da67d5b..1b1187b72 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -45,7 +45,6 @@ class BaseGPTAPI(BaseChatbot): format_msgs: Optional[list[dict[str, str]]] = None, generator: bool = False, ) -> str: - message = [] if system_msgs: message = self._system_msgs(system_msgs) else: From 2c3ab2fae4be572d62e7fd5a54392e25993327b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 13:10:01 +0800 Subject: [PATCH 363/398] refactor: brain memory --- metagpt/provider/metagpt_llm_api.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 95514cf53..7e79f0ae5 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -14,15 +14,3 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): def __init__(self): super().__init__() - - async def get_summary(self, memory, max_words=200, keep_language: bool = False, **kwargs) -> str: - """ - Return string in the following format： - [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Knock knock."}, - {"role": "assistant", "content": "Who's there?"}, - {"role": "user", "content": "Orange."}, - ] - """ - return memory.dumps_raw_messages(max_length=max_words) From dda55aec96ee25b5f44297b42b2075939b63f683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 15:13:25 +0800 Subject: [PATCH 364/398] fixbug: llm missing --- metagpt/memory/brain_memory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index b8f9a2a15..59d108a7d 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -156,7 +156,9 @@ class BrainMemory(pydantic.BaseModel): part_max_words = min(int(max_words / len(text_windows)) + 1, 100) summaries = [] for ws in text_windows: - response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) + response = await self._get_summary( + text=ws, llm=llm, max_words=part_max_words, keep_language=keep_language + ) summaries.append(response) if len(summaries) == 1: summary = summaries[0] From b58d2ff2d3ff64b4fd6a7c2279a6520a04e8e958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 15:19:09 +0800 Subject: [PATCH 365/398] fixbug: llm missing --- metagpt/provider/openai_api.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 514671488..81be1975a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -263,15 +263,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs) -> str: - """ - Return string in the following format： - [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Knock knock."}, - {"role": "assistant", "content": "Who's there?"}, - {"role": "user", "content": "Orange."}, - ] - """ from metagpt.memory.brain_memory import BrainMemory memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text, cacheable=False) From 95a5f1b9f1edc484b280b2b24277c9bf52926d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 16:29:41 +0800 Subject: [PATCH 366/398] fixbug: context missing --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 397ddc94b..84ca07c9a 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -123,7 +123,7 @@ class Assistant(Role): history_summary = await self.memory.summarize(max_words=800, keep_language=True, llm=self._llm) if last_talk and await self.memory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): # Merge relevant content. - last_talk = await self.memory.rewrite(sentence=last_talk, llm=self._llm) + last_talk = await self.memory.rewrite(sentence=last_talk, context=history_summary, llm=self._llm) return last_talk return last_talk From 85dc0ad7d4522df3c2fc8bdb58c50f9029f25f33 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sat, 9 Sep 2023 14:28:46 +0800 Subject: [PATCH 367/398] wait_exponential if RateLimitError --- metagpt/provider/base_gpt_api.py | 9 +--- metagpt/provider/openai_api.py | 71 ++++++-------------------------- 2 files changed, 13 insertions(+), 67 deletions(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 1b1187b72..e334e8a5d 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -9,7 +9,6 @@ from abc import abstractmethod from typing import Optional -from metagpt.logs import logger from metagpt.provider.base_chatbot import BaseChatbot @@ -52,13 +51,7 @@ class BaseGPTAPI(BaseChatbot): if format_msgs: message.extend(format_msgs) message.append(self._user_msg(msg)) - try: - rsp = await self.acompletion_text(message, stream=True, generator=generator) - except Exception as e: - logger.exception(f"{e}") - logger.info(f"ask:{msg}, error:{e}") - raise e - logger.info(f"ask:{msg}, anwser:{rsp}") + rsp = await self.acompletion_text(message, stream=True, generator=generator) return rsp def _extract_assistant_rsp(self, context): diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 81be1975a..7fc8b867a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -7,17 +7,16 @@ Change cost control from global to company level. """ import asyncio -import random import time -import traceback import openai -from openai.error import APIConnectionError +from openai.error import APIConnectionError, RateLimitError from tenacity import ( after_log, retry, retry_if_exception_type, stop_after_attempt, + wait_exponential, wait_fixed, ) @@ -75,16 +74,13 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ def __init__(self): - self.llm = openai self.model = CONFIG.openai_api_model self.auto_max_tokens = False self.rpm = int(CONFIG.get("RPM", 10)) RateLimiter.__init__(self, rpm=self.rpm) async def _achat_completion_stream(self, messages: list[dict]) -> str: - response = await self.async_retry_call( - openai.ChatCompletion.acreate, **self._cons_kwargs(messages), stream=True - ) + response = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages), stream=True) # iterate through the stream of events async for chunk in response: chunk_message = chunk["choices"][0]["delta"] # extract the message @@ -118,12 +114,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return kwargs async def _achat_completion(self, messages: list[dict]) -> dict: - rsp = await self.async_retry_call(self.llm.ChatCompletion.acreate, **self._cons_kwargs(messages)) + rsp = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages)) self._update_costs(rsp.get("usage")) return rsp def _chat_completion(self, messages: list[dict]) -> dict: - rsp = self.retry_call(self.llm.ChatCompletion.create, **self._cons_kwargs(messages)) + rsp = openai.ChatCompletion.create(**self._cons_kwargs(messages)) self._update_costs(rsp) return rsp @@ -144,6 +140,13 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) + @retry( + stop=stop_after_attempt(6), + wait=wait_exponential(1), + after=after_log(logger, logger.level("WARNING").name), + retry=retry_if_exception_type(RateLimitError), + reraise=True, + ) async def acompletion_text(self, messages: list[dict], stream=False, generator: bool = False) -> str: """when streaming, print each token in place.""" if stream: @@ -221,58 +224,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - @staticmethod - async def async_retry_call(func, *args, **kwargs): - for i in range(OpenAIGPTAPI.MAX_TRY): - try: - rsp = await func(*args, **kwargs) - return rsp - except openai.error.RateLimitError as e: - random_time = random.uniform(0, 3) # 生成0到5秒之间的随机时间 - rounded_time = round(random_time, 1) # 保留一位小数，以实现0.1秒的精度 - logger.warning(f"Exception:{e}, sleeping for {rounded_time} seconds") - await asyncio.sleep(rounded_time) - continue - except Exception as e: - error_str = traceback.format_exc() - logger.error(f"Exception:{e}, stack:{error_str}") - raise e - raise openai.error.OpenAIError("Exceeds the maximum retries") - - @staticmethod - def retry_call(func, *args, **kwargs): - for i in range(OpenAIGPTAPI.MAX_TRY): - try: - rsp = func(*args, **kwargs) - return rsp - except openai.error.RateLimitError as e: - logger.warning(f"Exception:{e}") - continue - except ( - openai.error.AuthenticationError, - openai.error.PermissionError, - openai.error.InvalidAPIType, - openai.error.SignatureVerificationError, - ) as e: - logger.warning(f"Exception:{e}") - raise e - except Exception as e: - error_str = traceback.format_exc() - logger.error(f"Exception:{e}, stack:{error_str}") - raise e - raise openai.error.OpenAIError("Exceeds the maximum retries") - async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs) -> str: from metagpt.memory.brain_memory import BrainMemory memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text, cacheable=False) return await memory.summarize(llm=self, max_length=max_words, keep_language=keep_language) - - MAX_TRY = 5 - - -if __name__ == "__main__": - txt = """ -as dfas sad lkf sdkl sakdfsdk sjd jsk sdl sk dd sd asd fa sdf sad dd -- .gitlab-ci.yml & base_test.py - """ From 19e78ff13e109b55aebb59ca2da2c9f02bcd78a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 9 Sep 2023 16:38:43 +0800 Subject: [PATCH 368/398] fixbug: get_title --- metagpt/memory/brain_memory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 59d108a7d..78eeac758 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -226,6 +226,9 @@ class BrainMemory(pydantic.BaseModel): async def get_title(self, llm, max_words=5, **kwargs) -> str: """Generate text title""" + if self.llm_type == LLMType.METAGPT.value: + return self.history[0] if self.history else "New" + summary = await self.summarize(llm=llm, max_words=500) language = CONFIG.language or DEFAULT_LANGUAGE From 1b6b24077e2f4b9fa37af2ee742a3e578c8efeee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 9 Sep 2023 16:43:42 +0800 Subject: [PATCH 369/398] fixbug: get_title --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 78eeac758..be3736100 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -227,7 +227,7 @@ class BrainMemory(pydantic.BaseModel): async def get_title(self, llm, max_words=5, **kwargs) -> str: """Generate text title""" if self.llm_type == LLMType.METAGPT.value: - return self.history[0] if self.history else "New" + return Message(**self.history[0]).content if self.history else "New" summary = await self.summarize(llm=llm, max_words=500) From 768e934444bb0c2180240a9671eb61ce3218471d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 15 Sep 2023 17:32:45 +0800 Subject: [PATCH 370/398] refactor: uuid --- metagpt/tools/iflytek_tts.py | 2 +- metagpt/utils/s3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/tools/iflytek_tts.py b/metagpt/tools/iflytek_tts.py index a91d8091b..cb87d2e7f 100644 --- a/metagpt/tools/iflytek_tts.py +++ b/metagpt/tools/iflytek_tts.py @@ -136,7 +136,7 @@ async def oas3_iflytek_tts(text: str, voice: str = "", app_id: str = "", api_key if not voice: voice = CONFIG.IFLYTEK_VOICE or DEFAULT_IFLYTEK_VOICE - filename = Path(__file__).parent / (str(uuid.uuid4()).replace("-", "") + ".mp3") + filename = Path(__file__).parent / (uuid.uuid4().hex + ".mp3") try: tts = IFlyTekTTS(app_id=app_id, api_key=api_key, api_secret=api_secret) await tts.synthesize_speech(text=text, output_file=str(filename), voice=voice) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 96b457972..dde68f720 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -132,7 +132,7 @@ class S3: async def cache(self, data: str, file_ext: str, format: str = "") -> str: """Save data to remote S3 and return url""" - object_name = str(uuid.uuid4()).replace("-", "") + file_ext + object_name = uuid.uuid4().hex + file_ext path = Path(__file__).parent pathname = path / object_name try: From 89be81524c963a64e5e21c4cc05126bf289eb63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 15 Sep 2023 21:56:39 +0800 Subject: [PATCH 371/398] feat: update skill specification --- .well-known/skills.yaml | 213 +++++++++++++++++++++++----------- metagpt/learn/skill_loader.py | 61 +++++++--- 2 files changed, 189 insertions(+), 85 deletions(-) diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index d08d7aced..137bfcdb4 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -1,72 +1,149 @@ +skillapi: "0.1.0" + +info: + title: "Agent Skill Specification" + version: "1.0" + entities: Assistant: - skills: - - name: text_to_speech - description: Text-to-speech - id: text_to_speech.text_to_speech - x-prerequisite: - - name: AZURE_TTS_SUBSCRIPTION_KEY - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - - name: AZURE_TTS_REGION - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - arguments: - text: 'The text used for voice conversion. Required.' - lang: 'The value can contain a language code such as en (English), or a locale such as en-US (English - United States). The optional parameter are "English", "Chinese". Default value: "Chinese".' - voice: 'Default value: "zh-CN-XiaomoNeural".' - style: 'Speaking style to express different emotions like cheerfulness, empathy, and calm. The optional parameter values are "affectionate", "angry", "calm", "cheerful", "depressed", "disgruntled", "embarrassed", "envious", "fearful", "gentle", "sad", "serious". Default value: "affectionate".' - role: 'With roles, the same voice can act as a different age and gender. The optional parameter values are "Girl", "Boy", "OlderAdultFemale", "OlderAdultMale", "SeniorFemale", "SeniorMale", "YoungAdultFemale", "YoungAdultMale". Default value: "Girl".' - examples: - - ask: 'A girl says "hello world"' - answer: 'text_to_speech(text="hello world", role="Girl")' - - ask: 'A boy affectionate says "hello world"' - answer: 'text_to_speech(text="hello world", role="Boy", style="affectionate")' - - ask: 'A boy says "你好"' - answer: 'text_to_speech(text="你好", role="Boy", lang="Chinese")' - - ask: 'How to speak "你好"?' - answer: 'text_to_speech(text="你好", lang="Chinese")' - returns: - type: string - format: base64 + summary: assistant + description: assistant + skills: + - name: text_to_speech + description: Text-to-speech + id: text_to_speech.text_to_speech + required: + oneOf: + - schema: + type: object + properties: + AZURE_TTS_SUBSCRIPTION_KEY: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + AZURE_TTS_REGION: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + - schema: + type: object + properties: + IFLYTEK_APP_ID: + type: string + description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_KEY: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_SECRET: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + parameters: + text: + description: 'The text used for voice conversion.' + required: true + type: string + lang: + description: 'The value can contain a language code such as en (English), or a locale such as en-US (English - United States).' + type: string + enum: + - English + - Chinese + default: Chinese + voice: + description: Name of voice styles + type: string + default: zh-CN-XiaomoNeural + style: + type: string + description: Speaking style to express different emotions like cheerfulness, empathy, and calm. + enum: + - affectionate + - angry + - calm + - cheerful + - depressed + - disgruntled + - embarrassed + - envious + - fearful + - gentle + - sad + - serious + default: affectionate + role: + type: string + description: With roles, the same voice can act as a different age and gender. + enum: + - Girl + - Boy + - OlderAdultFemale + - OlderAdultMale + - SeniorFemale + - SeniorMale + - YoungAdultFemale + - YoungAdultMale + default: Girl + examples: + - ask: 'A girl says "hello world"' + answer: 'text_to_speech(text="hello world", role="Girl")' + - ask: 'A boy affectionate says "hello world"' + answer: 'text_to_speech(text="hello world", role="Boy", style="affectionate")' + - ask: 'A boy says "你好"' + answer: 'text_to_speech(text="hello world", role="Boy", lang="Chinese")' + returns: + type: string + format: base64 - - name: text_to_image - description: Create a drawing based on the text. - id: text_to_image.text_to_image - x-prerequisite: - - name: OPENAI_API_KEY - description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" - - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL - description: "Model url." - arguments: - text: 'The text used for image conversion. Required.' - size_type: 'Default value: "512x512".' - examples: - - ask: 'Draw a girl' - answer: 'text_to_image(text="Draw a girl", size_type="512x512")' - - ask: 'Draw an apple' - answer: 'text_to_image(text="Draw an apple", size_type="512x512")' - - ask: 'Draw an apple picture' - answer: 'text_to_image(text="Draw an apple", size_type="512x512")' - - ask: 'Draw an apple image' - answer: 'text_to_image(text="Draw an apple", size_type="512x512")' - returns: - type: string - format: base64 + - name: text_to_image + description: Create a drawing based on the text. + id: text_to_image.text_to_image + required: + oneOf: + - name: OPENAI_API_KEY + type: string + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL + type: string + description: "Model url." + parameters: + text: + description: 'The text used for image conversion.' + type: string + required: true + size_type: + description: size type + type: string + default: "512x512" + examples: + - ask: 'Draw a girl' + answer: 'text_to_image(text="Draw a girl", size_type="512x512")' + - ask: 'Draw an apple' + answer: 'text_to_image(text="Draw an apple", size_type="512x512")' + returns: + type: string + format: base64 - - name: web_search - description: Perform Google searches to provide real-time information. - id: web_search.web_search - x-prerequisite: - - name: SEARCH_ENGINE - description: "Supported values: serpapi/google/serper/ddg" - - name: SERPER_API_KEY - description: "SERPER API KEY, For more details, checkout: `https://serper.dev/api-key`" - arguments: - query: 'The search query. Required.' - max_results: 'The number of search results to retrieve. Default value: 6.' - examples: - - ask: 'Search for information about artificial intelligence' - answer: 'web_search(query="Search for information about artificial intelligence", max_results=6)' - - ask: 'Find news articles about climate change' - answer: 'web_search(query="Find news articles about climate change", max_results=6)' - returns: - type: string \ No newline at end of file + - name: web_search + description: Perform Google searches to provide real-time information. + id: web_search.web_search + required: + - name: SEARCH_ENGINE + type: string + description: "Supported values: serpapi/google/serper/ddg" + - name: SERPER_API_KEY + type: string + description: "SERPER API KEY, For more details, checkout: `https://serper.dev/api-key`" + parameters: + query: + type: string + description: 'The search query.' + required: true + max_results: + type: number + default: 6 + description: 'The number of search results to retrieve.' + examples: + - ask: 'Search for information about artificial intelligence' + answer: 'web_search(query="Search for information about artificial intelligence", max_results=6)' + - ask: 'Find news articles about climate change' + answer: 'web_search(query="Find news articles about climate change", max_results=6)' + returns: + type: string diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index 83200bca6..b1d27db92 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -7,10 +7,10 @@ @Desc : Skill YAML Configuration Loader. """ from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union import yaml -from pydantic import BaseModel, Field +from pydantic import BaseModel from metagpt.config import CONFIG @@ -25,29 +25,43 @@ class Returns(BaseModel): format: Optional[str] = None -class Prerequisite(BaseModel): - name: str - type: Optional[str] = None - description: Optional[str] = None - default: Optional[str] = None +class Parameter(BaseModel): + type: str + description: str = None class Skill(BaseModel): name: str - description: str - id: str - x_prerequisite: Optional[List[Prerequisite]] = Field(default=None, alias="x-prerequisite") - arguments: Dict + description: str = None + id: str = None + required: Optional[Union[List, Dict]] = None + parameters: Dict[str, Parameter] = None examples: List[Example] returns: Returns + @property + def arguments(self) -> Dict: + if not self.parameters: + return {} + ret = {} + for k, v in self.parameters.items(): + ret[k] = v.description if v.description else "" + return ret -class EntitySkills(BaseModel): + +class Entity(BaseModel): + name: str = None skills: List[Skill] +class Components(BaseModel): + pass + + class SkillsDeclaration(BaseModel): - entities: Dict[str, EntitySkills] + skillapi: str + entities: Dict[str, Entity] + components: Components = None class SkillLoader: @@ -60,8 +74,8 @@ class SkillLoader: def get_skill_list(self, entity_name: str = "Assistant") -> Dict: """Return the skill name based on the skill description.""" - entity_skills = self.get_entity(entity_name) - if not entity_skills: + entity = self.get_entity(entity_name) + if not entity: return {} agent_skills = CONFIG.agent_skills @@ -73,7 +87,7 @@ class SkillLoader: names = [AgentSkill(**i).name for i in agent_skills] description_to_name_mappings = {} - for s in entity_skills.skills: + for s in entity.skills: if s.name not in names: continue description_to_name_mappings[s.description] = s.name @@ -89,8 +103,21 @@ class SkillLoader: if sk.name == name: return sk - def get_entity(self, name) -> EntitySkills: + def get_entity(self, name) -> Entity: """Return a list of skills for the entity.""" if not self._skills: return None return self._skills.entities.get(name) + + +if __name__ == "__main__": + CONFIG.agent_skills = [ + {"id": 1, "name": "text_to_speech", "type": "builtin", "config": {}, "enabled": True}, + {"id": 2, "name": "text_to_image", "type": "builtin", "config": {}, "enabled": True}, + {"id": 3, "name": "ai_call", "type": "builtin", "config": {}, "enabled": True}, + {"id": 3, "name": "data_analysis", "type": "builtin", "config": {}, "enabled": True}, + {"id": 5, "name": "crawler", "type": "builtin", "config": {"engine": "ddg"}, "enabled": True}, + {"id": 6, "name": "knowledge", "type": "builtin", "config": {}, "enabled": True}, + ] + loader = SkillLoader() + print(loader.get_skill_list()) From 9fdf70658608d2a91d3648bf155d0ff4fa5b7d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 16 Sep 2023 10:37:27 +0800 Subject: [PATCH 372/398] feat: +type --- .well-known/metagpt_oas3_api.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index 1e3cecb10..e21cc2d01 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -14,8 +14,10 @@ paths: /tts/azsure: x-prerequisite: - name: AZURE_TTS_SUBSCRIPTION_KEY + type: string description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - name: AZURE_TTS_REGION + type: string description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" post: summary: "Convert Text to Base64-encoded .wav File Stream" @@ -76,10 +78,13 @@ paths: /tts/iflytek: x-prerequisite: - name: IFLYTEK_APP_ID + type: string description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" - name: IFLYTEK_API_KEY + type: string description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" - name: IFLYTEK_API_SECRET + type: string description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" post: summary: "Convert Text to Base64-encoded .mp3 File Stream" @@ -133,6 +138,7 @@ paths: /txt2img/openai: x-prerequisite: - name: OPENAI_API_KEY + type: string description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" post: summary: "Convert Text to Base64-encoded Image Data Stream" @@ -174,6 +180,7 @@ paths: /txt2embedding/openai: x-prerequisite: - name: OPENAI_API_KEY + type: string description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" post: summary: Text to embedding @@ -216,6 +223,7 @@ paths: /txt2image/metagpt: x-prerequisite: - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL + type: string description: "Model url." post: summary: "Text to Image" From b4493052e7a3eb2533e5a642491a5e9c1c0e5e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 16 Sep 2023 14:56:38 +0800 Subject: [PATCH 373/398] feat: +x-prerequisite --- .well-known/metagpt_oas3_api.yaml | 71 ++++++++++++++++--------- .well-known/skills.yaml | 86 ++++++++++++++++++------------- 2 files changed, 96 insertions(+), 61 deletions(-) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index e21cc2d01..0a702e8b6 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -13,12 +13,17 @@ servers: paths: /tts/azsure: x-prerequisite: - - name: AZURE_TTS_SUBSCRIPTION_KEY - type: string - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - - name: AZURE_TTS_REGION - type: string - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + configurations: + AZURE_TTS_SUBSCRIPTION_KEY: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + AZURE_TTS_REGION: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + required: + allOf: + - AZURE_TTS_SUBSCRIPTION_KEY + - AZURE_TTS_REGION post: summary: "Convert Text to Base64-encoded .wav File Stream" description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" @@ -77,15 +82,21 @@ paths: /tts/iflytek: x-prerequisite: - - name: IFLYTEK_APP_ID - type: string - description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" - - name: IFLYTEK_API_KEY - type: string - description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" - - name: IFLYTEK_API_SECRET - type: string - description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + configurations: + IFLYTEK_APP_ID: + type: string + description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_KEY: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_SECRET: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + required: + allOf: + - IFLYTEK_APP_ID + - IFLYTEK_API_KEY + - IFLYTEK_API_SECRET post: summary: "Convert Text to Base64-encoded .mp3 File Stream" description: "For more details, check out: [iFlyTek](https://console.xfyun.cn/services/tts)" @@ -137,9 +148,13 @@ paths: /txt2img/openai: x-prerequisite: - - name: OPENAI_API_KEY - type: string - description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + configurations: + OPENAI_API_KEY: + type: string + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + required: + allOf: + - OPENAI_API_KEY post: summary: "Convert Text to Base64-encoded Image Data Stream" operationId: openai_text_to_image.oas3_openai_text_to_image @@ -179,9 +194,13 @@ paths: description: "Internal Server Error" /txt2embedding/openai: x-prerequisite: - - name: OPENAI_API_KEY - type: string - description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + configurations: + OPENAI_API_KEY: + type: string + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + required: + allOf: + - OPENAI_API_KEY post: summary: Text to embedding operationId: openai_text_to_embedding.oas3_openai_text_to_embedding @@ -222,9 +241,13 @@ paths: /txt2image/metagpt: x-prerequisite: - - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL - type: string - description: "Model url." + configurations: + METAGPT_TEXT_TO_IMAGE_MODEL_URL: + type: string + description: "Model url." + required: + allOf: + - METAGPT_TEXT_TO_IMAGE_MODEL_URL post: summary: "Text to Image" description: "Generate an image from the provided text using the MetaGPT Text-to-Image API." diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 137bfcdb4..05465454a 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -12,29 +12,32 @@ entities: - name: text_to_speech description: Text-to-speech id: text_to_speech.text_to_speech - required: - oneOf: - - schema: - type: object - properties: - AZURE_TTS_SUBSCRIPTION_KEY: - type: string - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - AZURE_TTS_REGION: - type: string - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - - schema: - type: object - properties: - IFLYTEK_APP_ID: - type: string - description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" - IFLYTEK_API_KEY: - type: string - description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" - IFLYTEK_API_SECRET: - type: string - description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + x-prerequisite: + configurations: + AZURE_TTS_SUBSCRIPTION_KEY: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + AZURE_TTS_REGION: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + IFLYTEK_APP_ID: + type: string + description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_KEY: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_SECRET: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + required: + oneOf: + - allOf: + - AZURE_TTS_SUBSCRIPTION_KEY + - AZURE_TTS_REGION + - allOf: + - IFLYTEK_APP_ID + - IFLYTEK_API_KEY + - IFLYTEK_API_SECRET parameters: text: description: 'The text used for voice conversion.' @@ -51,9 +54,9 @@ entities: description: Name of voice styles type: string default: zh-CN-XiaomoNeural - style: + style: type: string - description: Speaking style to express different emotions like cheerfulness, empathy, and calm. + description: Speaking style to express different emotions like cheerfulness, empathy, and calm. enum: - affectionate - angry @@ -95,16 +98,20 @@ entities: - name: text_to_image description: Create a drawing based on the text. id: text_to_image.text_to_image - required: - oneOf: - - name: OPENAI_API_KEY + x-prerequisite: + configurations: + OPENAI_API_KEY: type: string description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" - - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL + METAGPT_TEXT_TO_IMAGE_MODEL_URL: type: string description: "Model url." + required: + oneOf: + - OPENAI_API_KEY + - METAGPT_TEXT_TO_IMAGE_MODEL_URL parameters: - text: + text: description: 'The text used for image conversion.' type: string required: true @@ -124,13 +131,18 @@ entities: - name: web_search description: Perform Google searches to provide real-time information. id: web_search.web_search - required: - - name: SEARCH_ENGINE - type: string - description: "Supported values: serpapi/google/serper/ddg" - - name: SERPER_API_KEY - type: string - description: "SERPER API KEY, For more details, checkout: `https://serper.dev/api-key`" + x-prerequisite: + configurations: + SEARCH_ENGINE: + type: string + description: "Supported values: serpapi/google/serper/ddg" + SERPER_API_KEY: + type: string + description: "SERPER API KEY, For more details, checkout: `https://serper.dev/api-key`" + required: + allOf: + - SEARCH_ENGINE + - SERPER_API_KEY parameters: query: type: string From ad71adb2091bbefb948cad48bc70c74891226bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 16 Sep 2023 15:02:24 +0800 Subject: [PATCH 374/398] feat: +x-prerequisite --- metagpt/learn/skill_loader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index b1d27db92..dff5e26ae 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -7,10 +7,10 @@ @Desc : Skill YAML Configuration Loader. """ from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional import yaml -from pydantic import BaseModel +from pydantic import BaseModel, Field from metagpt.config import CONFIG @@ -34,7 +34,7 @@ class Skill(BaseModel): name: str description: str = None id: str = None - required: Optional[Union[List, Dict]] = None + x_prerequisite: Dict = Field(default=None, alias="x-prerequisite") parameters: Dict[str, Parameter] = None examples: List[Example] returns: Returns From 4bf3510832e1114c9418b56d02f215c48334964f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 19 Sep 2023 14:13:28 +0800 Subject: [PATCH 375/398] feat: +unit test --- tests/metagpt/learn/test_skill_loader.py | 41 ++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/metagpt/learn/test_skill_loader.py diff --git a/tests/metagpt/learn/test_skill_loader.py b/tests/metagpt/learn/test_skill_loader.py new file mode 100644 index 000000000..5bc0e776f --- /dev/null +++ b/tests/metagpt/learn/test_skill_loader.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/19 +@Author : mashenquan +@File : test_skill_loader.py +@Desc : Unit tests. +""" + +from metagpt.config import CONFIG +from metagpt.learn.skill_loader import SkillLoader + + +def test_suite(): + CONFIG.agent_skills = [ + {"id": 1, "name": "text_to_speech", "type": "builtin", "config": {}, "enabled": True}, + {"id": 2, "name": "text_to_image", "type": "builtin", "config": {}, "enabled": True}, + {"id": 3, "name": "ai_call", "type": "builtin", "config": {}, "enabled": True}, + {"id": 3, "name": "data_analysis", "type": "builtin", "config": {}, "enabled": True}, + {"id": 5, "name": "crawler", "type": "builtin", "config": {"engine": "ddg"}, "enabled": True}, + {"id": 6, "name": "knowledge", "type": "builtin", "config": {}, "enabled": True}, + {"id": 6, "name": "web_search", "type": "builtin", "config": {}, "enabled": True}, + ] + loader = SkillLoader() + skills = loader.get_skill_list() + assert skills + assert len(skills) >= 3 + for desc, name in skills.items(): + assert desc + assert name + + entity = loader.get_entity("Assistant") + assert entity + assert entity.skills + for sk in entity.skills: + assert sk + assert sk.arguments + + +if __name__ == "__main__": + test_suite() From c69928a1745a84bb9a25a040ac50a59a849807ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 19 Sep 2023 21:33:23 +0800 Subject: [PATCH 376/398] refactor: example --- .well-known/skills.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 05465454a..c19a9501e 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -10,7 +10,7 @@ entities: description: assistant skills: - name: text_to_speech - description: Text-to-speech + description: Generate a voice file from the input text, text-to-speech id: text_to_speech.text_to_speech x-prerequisite: configurations: @@ -90,7 +90,7 @@ entities: - ask: 'A boy affectionate says "hello world"' answer: 'text_to_speech(text="hello world", role="Boy", style="affectionate")' - ask: 'A boy says "你好"' - answer: 'text_to_speech(text="hello world", role="Boy", lang="Chinese")' + answer: 'text_to_speech(text="你好", role="Boy", lang="Chinese")' returns: type: string format: base64 From 49f55ad3746da6c71587535e6fa9f85695bffb11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 20 Sep 2023 11:37:33 +0800 Subject: [PATCH 377/398] feat: +LLM_TYPE: OpenAI --- config/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.yaml b/config/config.yaml index 5c8dea03e..71744aa7f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -11,6 +11,7 @@ OPENAI_API_BASE: "https://api.openai.com/v1" OPENAI_API_MODEL: "gpt-4" MAX_TOKENS: 1500 RPM: 10 +LLM_TYPE: OpenAI #### if Anthropic #Anthropic_API_KEY: "YOUR_API_KEY" From 56bf0b9b97c69e0aa0a49ddf35d576945e38d236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 20 Sep 2023 17:45:47 +0800 Subject: [PATCH 378/398] fixbug: max_words --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 7fc8b867a..953043912 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -228,4 +228,4 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): from metagpt.memory.brain_memory import BrainMemory memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text, cacheable=False) - return await memory.summarize(llm=self, max_length=max_words, keep_language=keep_language) + return await memory.summarize(llm=self, max_words=max_words, keep_language=keep_language) From 45aa451ec6daf7f6690aabb75e1b305b29925514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Dec 2023 13:55:12 +0800 Subject: [PATCH 379/398] feat: upgrade openai to 1.x --- metagpt/llm.py | 9 +- metagpt/provider/base_chatbot.py | 7 +- metagpt/provider/base_gpt_api.py | 57 ++++---- metagpt/provider/human_provider.py | 22 ++- metagpt/provider/openai_api.py | 213 +++++++++++++++-------------- requirements.txt | 10 +- 6 files changed, 174 insertions(+), 144 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index d8d06c0a1..dce33b9db 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 14:45 @Author : alexanderwu @File : llm.py +@Modified By: mashenquan, 2023-12-4. Upgrade openai to 1.x """ from metagpt.config import CONFIG @@ -11,7 +12,9 @@ from metagpt.provider.anthropic_api import Claude2 as Claude from metagpt.provider.human_provider import HumanProvider from metagpt.provider.openai_api import OpenAIGPTAPI from metagpt.provider.spark_api import SparkAPI -from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI +# openai v1.x removed the 'api_requestor', making interfaces built on it no longer functional. +# More: https://github.com/openai/openai-python/discussions/742 +# from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI _ = HumanProvider() # Avoid pre-commit error @@ -25,8 +28,8 @@ def LLM() -> "BaseGPTAPI": llm = Claude() elif CONFIG.spark_api_key: llm = SparkAPI() - elif CONFIG.zhipuai_api_key: - llm = ZhiPuAIGPTAPI() + # elif CONFIG.zhipuai_api_key: # openai v1.x removed the 'api_requestor' + # llm = ZhiPuAIGPTAPI() else: raise RuntimeError("You should config a LLM configuration first") diff --git a/metagpt/provider/base_chatbot.py b/metagpt/provider/base_chatbot.py index a6950f144..535130de7 100644 --- a/metagpt/provider/base_chatbot.py +++ b/metagpt/provider/base_chatbot.py @@ -4,6 +4,7 @@ @Time : 2023/5/5 23:00 @Author : alexanderwu @File : base_chatbot.py +@Modified By: mashenquan, 2023/11/21. Add `timeout`. """ from abc import ABC, abstractmethod from dataclasses import dataclass @@ -17,13 +18,13 @@ class BaseChatbot(ABC): use_system_prompt: bool = True @abstractmethod - def ask(self, msg: str) -> str: + def ask(self, msg: str, timeout=3) -> str: """Ask GPT a question and get an answer""" @abstractmethod - def ask_batch(self, msgs: list) -> str: + def ask_batch(self, msgs: list, timeout=3) -> str: """Ask GPT multiple questions and get a series of answers""" @abstractmethod - def ask_code(self, msgs: list) -> str: + def ask_code(self, msgs: list, timeout=3) -> str: """Ask GPT multiple questions and get a piece of code""" diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 565ae94f7..75cebed77 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -33,23 +33,27 @@ class BaseGPTAPI(BaseChatbot): def _default_system_msg(self): return self._system_msg(self.system_prompt) - def ask(self, msg: str) -> str: + def ask(self, msg: str, timeout=3) -> str: message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] - rsp = self.completion(message) + rsp = self.completion(message, timeout=timeout) return self.get_choice_text(rsp) - async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: + async def aask( + self, + msg: str, + system_msgs: Optional[list[str]] = None, + format_msgs: Optional[list[dict[str, str]]] = None, + generator: bool = False, + timeout=3, + ) -> str: if system_msgs: - message = ( - self._system_msgs(system_msgs) + [self._user_msg(msg)] - if self.use_system_prompt - else [self._user_msg(msg)] - ) + message = self._system_msgs(system_msgs) else: - message = ( - [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] - ) - rsp = await self.acompletion_text(message, stream=True) + message = [self._default_system_msg()] + if format_msgs: + message.extend(format_msgs) + message.append(self._user_msg(msg)) + rsp = await self.acompletion_text(message, stream=True, generator=generator, timeout=timeout) logger.debug(message) # logger.debug(rsp) return rsp @@ -57,38 +61,38 @@ class BaseGPTAPI(BaseChatbot): def _extract_assistant_rsp(self, context): return "\n".join([i["content"] for i in context if i["role"] == "assistant"]) - def ask_batch(self, msgs: list) -> str: + def ask_batch(self, msgs: list, timeout=3) -> str: context = [] for msg in msgs: umsg = self._user_msg(msg) context.append(umsg) - rsp = self.completion(context) + rsp = self.completion(context, timeout=timeout) rsp_text = self.get_choice_text(rsp) context.append(self._assistant_msg(rsp_text)) return self._extract_assistant_rsp(context) - async def aask_batch(self, msgs: list) -> str: + async def aask_batch(self, msgs: list, timeout=3) -> str: """Sequential questioning""" context = [] for msg in msgs: umsg = self._user_msg(msg) context.append(umsg) - rsp_text = await self.acompletion_text(context) + rsp_text = await self.acompletion_text(context, timeout=timeout) context.append(self._assistant_msg(rsp_text)) return self._extract_assistant_rsp(context) - def ask_code(self, msgs: list[str]) -> str: + def ask_code(self, msgs: list[str], timeout=3) -> str: """FIXME: No code segment filtering has been done here, and all results are actually displayed""" - rsp_text = self.ask_batch(msgs) + rsp_text = self.ask_batch(msgs, timeout=timeout) return rsp_text - async def aask_code(self, msgs: list[str]) -> str: + async def aask_code(self, msgs: list[str], timeout=3) -> str: """FIXME: No code segment filtering has been done here, and all results are actually displayed""" - rsp_text = await self.aask_batch(msgs) + rsp_text = await self.aask_batch(msgs, timeout=timeout) return rsp_text @abstractmethod - def completion(self, messages: list[dict]): + def completion(self, messages: list[dict], timeout=3): """All GPTAPIs are required to provide the standard OpenAI completion interface [ {"role": "system", "content": "You are a helpful assistant."}, @@ -98,7 +102,7 @@ class BaseGPTAPI(BaseChatbot): """ @abstractmethod - async def acompletion(self, messages: list[dict]): + async def acompletion(self, messages: list[dict], timeout=3): """Asynchronous version of completion All GPTAPIs are required to provide the standard OpenAI completion interface [ @@ -109,7 +113,7 @@ class BaseGPTAPI(BaseChatbot): """ @abstractmethod - async def acompletion_text(self, messages: list[dict], stream=False) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str: """Asynchronous version of completion. Return str. Support stream-print""" def get_choice_text(self, rsp: dict) -> str: @@ -145,7 +149,7 @@ class BaseGPTAPI(BaseChatbot): :return dict: return first function of choice, for exmaple, {'name': 'execute', 'arguments': '{\n "language": "python",\n "code": "print(\'Hello, World!\')"\n}'} """ - return rsp.get("choices")[0]["message"]["tool_calls"][0]["function"].to_dict() + return rsp.get("choices")[0]["message"]["tool_calls"][0]["function"] def get_choice_function_arguments(self, rsp: dict) -> dict: """Required to provide the first function arguments of choice. @@ -163,3 +167,8 @@ class BaseGPTAPI(BaseChatbot): def messages_to_dict(self, messages): """objects to [{"role": "user", "content": msg}] etc.""" return [i.to_dict() for i in messages] + + @abstractmethod + async def close(self): + """Close connection""" + pass diff --git a/metagpt/provider/human_provider.py b/metagpt/provider/human_provider.py index c70a7f1a6..ba9c93c88 100644 --- a/metagpt/provider/human_provider.py +++ b/metagpt/provider/human_provider.py @@ -14,24 +14,32 @@ class HumanProvider(BaseGPTAPI): This enables replacing LLM anywhere in the framework with a human, thus introducing human interaction """ - def ask(self, msg: str) -> str: + def ask(self, msg: str, timeout=3) -> str: logger.info("It's your turn, please type in your response. You may also refer to the context below") rsp = input(msg) if rsp in ["exit", "quit"]: exit() return rsp - async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: - return self.ask(msg) + async def aask(self, msg: str, + system_msgs: Optional[list[str]] = None, + format_msgs: Optional[list[dict[str, str]]] = None, + generator: bool = False, + timeout=3,) -> str: + return self.ask(msg, timeout=timeout) - def completion(self, messages: list[dict]): + def completion(self, messages: list[dict], timeout=3): """dummy implementation of abstract method in base""" return [] - async def acompletion(self, messages: list[dict]): + async def acompletion(self, messages: list[dict], timeout=3): """dummy implementation of abstract method in base""" return [] - async def acompletion_text(self, messages: list[dict], stream=False) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str: """dummy implementation of abstract method in base""" - return [] + return "" + + async def close(self): + """Close connection""" + pass diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 8ac0c4b21..45fc763be 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -3,18 +3,23 @@ @Time : 2023/5/5 23:08 @Author : alexanderwu @File : openai.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. +@Modified By: mashenquan, 2023/11/21. Fix bug: ReadTimeout. +@Modified By: mashenquan, 2023/12/1. Fix bug: Unclosed connection caused by openai 0.x. """ import asyncio import time from typing import NamedTuple, Union -import openai -from openai.error import APIConnectionError +from openai import APIConnectionError, AsyncAzureOpenAI, AsyncOpenAI, RateLimitError +from openai.types import CompletionUsage from tenacity import ( after_log, retry, retry_if_exception_type, stop_after_attempt, + wait_exponential, wait_fixed, ) @@ -143,47 +148,31 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ def __init__(self): - self.__init_openai(CONFIG) - self.llm = openai self.model = CONFIG.openai_api_model self.auto_max_tokens = False + self.rpm = int(CONFIG.get("RPM", 10)) + if CONFIG.openai_api_type == "azure": + # https://learn.microsoft.com/zh-cn/azure/ai-services/openai/how-to/migration?tabs=python-new%2Cdalle-fix + self._client = AsyncAzureOpenAI( + api_key=CONFIG.openai_api_key, + api_version=CONFIG.openai_api_version, + azure_endpoint=CONFIG.openai_api_base, + ) + else: + # https://github.com/openai/openai-python#async-usage + self._client = AsyncOpenAI(api_key=CONFIG.openai_api_key, base_url=CONFIG.openai_api_base) self._cost_manager = CostManager() RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self, config): - openai.api_key = config.openai_api_key - if config.openai_api_base: - openai.api_base = config.openai_api_base - if config.openai_api_type: - openai.api_type = config.openai_api_type - openai.api_version = config.openai_api_version - if config.openai_proxy: - openai.proxy = config.openai_proxy - self.rpm = int(config.get("RPM", 10)) - - async def _achat_completion_stream(self, messages: list[dict]) -> str: - response = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages), stream=True) - - # create variables to collect the stream of chunks - collected_chunks = [] - collected_messages = [] + async def _achat_completion_stream(self, messages: list[dict], timeout=3) -> str: + kwargs = self._cons_kwargs(messages, timeout=timeout) + response = await self._client.chat.completions.create(**kwargs, stream=True) # iterate through the stream of events async for chunk in response: - collected_chunks.append(chunk) # save the event response - choices = chunk["choices"] - if len(choices) > 0: - chunk_message = chunk["choices"][0].get("delta", {}) # extract the message - collected_messages.append(chunk_message) # save the message - if "content" in chunk_message: - print(chunk_message["content"], end="") - print() + chunk_message = chunk.choices[0].delta.content or "" # extract the message + yield chunk_message - full_reply_content = "".join([m.get("content", "") for m in collected_messages]) - usage = self._calc_usage(messages, full_reply_content) - self._update_costs(usage) - return full_reply_content - - def _cons_kwargs(self, messages: list[dict], **configs) -> dict: + def _cons_kwargs(self, messages: list[dict], timeout=3, **configs) -> dict: kwargs = { "messages": messages, "max_tokens": self.get_max_tokens(messages), @@ -196,39 +185,27 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): kwargs.update(configs) if CONFIG.openai_api_type == "azure": - if CONFIG.deployment_name and CONFIG.deployment_id: - raise ValueError("You can only use one of the `deployment_id` or `deployment_name` model") - elif not CONFIG.deployment_name and not CONFIG.deployment_id: - raise ValueError("You must specify `DEPLOYMENT_NAME` or `DEPLOYMENT_ID` parameter") - kwargs_mode = ( - {"engine": CONFIG.deployment_name} - if CONFIG.deployment_name - else {"deployment_id": CONFIG.deployment_id} - ) + kwargs["model"] = CONFIG.deployment_id else: - kwargs_mode = {"model": self.model} - kwargs.update(kwargs_mode) + kwargs["model"] = self.model + kwargs["timeout"] = max(CONFIG.TIMEOUT, timeout) if CONFIG.TIMEOUT is not None else timeout + return kwargs - async def _achat_completion(self, messages: list[dict]) -> dict: - rsp = await self.llm.ChatCompletion.acreate(**self._cons_kwargs(messages)) - self._update_costs(rsp.get("usage")) - return rsp + async def _achat_completion(self, messages: list[dict], timeout=3) -> dict: + kwargs = self._cons_kwargs(messages, timeout=timeout) + rsp = await self._client.chat.completions.create(**kwargs) + self._update_costs(rsp.usage) + return rsp.dict() - def _chat_completion(self, messages: list[dict]) -> dict: - rsp = self.llm.ChatCompletion.create(**self._cons_kwargs(messages)) - self._update_costs(rsp) - return rsp + def completion(self, messages: list[dict], timeout=3) -> dict: + loop = self.get_event_loop() + return loop.run_until_complete(self.acompletion(messages, timeout=timeout)) - def completion(self, messages: list[dict]) -> dict: + async def acompletion(self, messages: list[dict], timeout=3) -> dict: # if isinstance(messages[0], Message): # messages = self.messages_to_dict(messages) - return self._chat_completion(messages) - - async def acompletion(self, messages: list[dict]) -> dict: - # if isinstance(messages[0], Message): - # messages = self.messages_to_dict(messages) - return await self._achat_completion(messages) + return await self._achat_completion(messages, timeout=timeout) @retry( stop=stop_after_attempt(3), @@ -237,14 +214,34 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream=False) -> str: + @retry( + stop=stop_after_attempt(6), + wait=wait_exponential(1), + after=after_log(logger, logger.level("WARNING").name), + retry=retry_if_exception_type(RateLimitError), + reraise=True, + ) + async def acompletion_text(self, messages: list[dict], stream=False, generator: bool = False, timeout=3) -> str: """when streaming, print each token in place.""" if stream: - return await self._achat_completion_stream(messages) - rsp = await self._achat_completion(messages) + resp = self._achat_completion_stream(messages, timeout=timeout) + if generator: + return resp + + collected_messages = [] + async for i in resp: + print(i, end="") + collected_messages.append(i) + + full_reply_content = "".join(collected_messages) + usage = self._calc_usage(messages, full_reply_content) + self._update_costs(usage) + return full_reply_content + + rsp = await self._achat_completion(messages, timeout=timeout) return self.get_choice_text(rsp) - def _func_configs(self, messages: list[dict], **kwargs) -> dict: + def _func_configs(self, messages: list[dict], timeout=3, **kwargs) -> dict: """ Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create """ @@ -255,17 +252,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): } kwargs.update(configs) - return self._cons_kwargs(messages, **kwargs) + return self._cons_kwargs(messages=messages, timeout=timeout, **kwargs) - def _chat_completion_function(self, messages: list[dict], **kwargs) -> dict: - rsp = self.llm.ChatCompletion.create(**self._func_configs(messages, **kwargs)) - self._update_costs(rsp.get("usage")) - return rsp + def _chat_completion_function(self, messages: list[dict], timeout=3, **kwargs) -> dict: + loop = self.get_event_loop() + return loop.run_until_complete(self._achat_completion_function(messages=messages, timeout=timeout, **kwargs)) - async def _achat_completion_function(self, messages: list[dict], **chat_configs) -> dict: - rsp = await self.llm.ChatCompletion.acreate(**self._func_configs(messages, **chat_configs)) - self._update_costs(rsp.get("usage")) - return rsp + async def _achat_completion_function(self, messages: list[dict], timeout=3, **chat_configs) -> dict: + kwargs = self._func_configs(messages=messages, timeout=timeout, **chat_configs) + rsp = await self._client.chat.completions.create(**kwargs) + self._update_costs(rsp.usage) + return rsp.dict() def _process_message(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: """convert messages to list[dict].""" @@ -319,21 +316,22 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): rsp = await self._achat_completion_function(messages, **kwargs) return self.get_choice_function_arguments(rsp) - def _calc_usage(self, messages: list[dict], rsp: str) -> dict: - usage = {} + def _calc_usage(self, messages: list[dict], rsp: str) -> CompletionUsage: if CONFIG.calc_usage: try: prompt_tokens = count_message_tokens(messages, self.model) completion_tokens = count_string_tokens(rsp, self.model) - usage["prompt_tokens"] = prompt_tokens - usage["completion_tokens"] = completion_tokens + usage = CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) return usage except Exception as e: logger.error("usage calculation failed!", e) - else: - return usage + return CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0) - async def acompletion_batch(self, batch: list[list[dict]]) -> list[dict]: + async def acompletion_batch(self, batch: list[list[dict]], timeout=3) -> list[dict]: """Return full JSON""" split_batches = self.split_batches(batch) all_results = [] @@ -342,16 +340,16 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): logger.info(small_batch) await self.wait_if_needed(len(small_batch)) - future = [self.acompletion(prompt) for prompt in small_batch] + future = [self.acompletion(prompt, timeout=timeout) for prompt in small_batch] results = await asyncio.gather(*future) logger.info(results) all_results.extend(results) return all_results - async def acompletion_batch_text(self, batch: list[list[dict]]) -> list[str]: + async def acompletion_batch_text(self, batch: list[list[dict]], timeout=3) -> list[str]: """Only return plain text""" - raw_results = await self.acompletion_batch(batch) + raw_results = await self.acompletion_batch(batch, timeout=timeout) results = [] for idx, raw_result in enumerate(raw_results, start=1): result = self.get_choice_text(raw_result) @@ -359,14 +357,11 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): logger.info(f"Result of task {idx}: {result}") return results - def _update_costs(self, usage: dict): + def _update_costs(self, usage: CompletionUsage): if CONFIG.calc_usage: - try: - prompt_tokens = int(usage["prompt_tokens"]) - completion_tokens = int(usage["completion_tokens"]) - self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) - except Exception as e: - logger.error("updating costs failed!", e) + prompt_tokens = usage.prompt_tokens + completion_tokens = usage.completion_tokens + self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) def get_costs(self) -> Costs: return self._cost_manager.get_costs() @@ -377,18 +372,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) def moderation(self, content: Union[str, list[str]]): - try: - if not content: - logger.error("content cannot be empty!") - else: - rsp = self._moderation(content=content) - return rsp - except Exception as e: - logger.error(f"moderating failed:{e}") - - def _moderation(self, content: Union[str, list[str]]): - rsp = self.llm.Moderation.create(input=content) - return rsp + loop = self.get_event_loop() + loop.run_until_complete(self.amoderation(content=content)) async def amoderation(self, content: Union[str, list[str]]): try: @@ -401,5 +386,25 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): logger.error(f"moderating failed:{e}") async def _amoderation(self, content: Union[str, list[str]]): - rsp = await self.llm.Moderation.acreate(input=content) + rsp = await self._client.moderations.create(input=content) return rsp + + async def close(self): + """Close connection""" + if not self._client: + return + await self._client.close() + self._client = None + + @staticmethod + def get_event_loop(): + try: + return asyncio.get_event_loop() + except RuntimeError as e: + if "There is no current event loop in thread" in str(e): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return loop + else: + raise e + diff --git a/requirements.txt b/requirements.txt index 99f738448..bcd2db243 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ langchain==0.0.231 loguru==0.6.0 meilisearch==0.21.0 numpy==1.24.3 -openai>=0.28.1 +openai>=1.3.6 openpyxl beautifulsoup4==4.12.2 pandas==2.0.3 @@ -42,9 +42,13 @@ qdrant-client==1.4.0 pytest-mock==3.11.1 open-interpreter==0.1.7; python_version>"3.9" ta==0.10.2 -semantic-kernel==0.3.13.dev0 +semantic-kernel wrapt==1.15.0 -websocket-client==0.58.0 +#aiohttp_jinja2 +#azure-cognitiveservices-speech~=1.31.0 +#aioboto3~=11.3.0 +#redis==4.3.5 +websocket-client==1.6.2 aiofiles==23.2.1 gitpython==3.1.40 zhipuai==1.0.7 From f7fd3e4ab8435a2421b5a84af2e1be6b70bd49fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Dec 2023 23:04:07 +0800 Subject: [PATCH 380/398] feat: +SummarizeCode, refactor project_name --- metagpt/actions/design_api.py | 55 +++-------- metagpt/actions/project_management.py | 10 +- metagpt/actions/summarize_code.py | 8 +- metagpt/actions/write_code.py | 7 +- metagpt/actions/write_prd.py | 40 ++++++-- metagpt/const.py | 3 + metagpt/roles/engineer.py | 131 ++++++++++++++++++-------- metagpt/roles/qa_engineer.py | 12 ++- metagpt/roles/role.py | 3 +- metagpt/schema.py | 9 +- metagpt/utils/file_repository.py | 11 +++ metagpt/utils/git_repository.py | 28 +++++- 12 files changed, 219 insertions(+), 98 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index c5787ba20..605b871a1 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023/11/27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ import json from pathlib import Path @@ -43,7 +44,7 @@ Requirement: Fill in the following missing information based on the context, eac ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Constant text. ## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -58,15 +59,15 @@ and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Implementation approach": "We will ...", - "project_name": "snake_game", + "project_name": "{project_name}", "File list": ["main.py"], "Data structures and interfaces": ' classDiagram - class Game{ + class Game{{ +int score - } + }} ... Game "1" -- "1" Food: has ', @@ -77,7 +78,7 @@ and only output the json inside this tag, nothing else G->>M: end game ', "Anything UNCLEAR": "The requirement is clear to me." -} +}} [/CONTENT] """, }, @@ -96,7 +97,7 @@ ATTENTION: Output carefully referenced "Format example" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Constant text. ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -114,7 +115,7 @@ We will ... ## project_name ```python -"snake_game" +"{project_name}" ``` ## File list @@ -173,7 +174,7 @@ ATTENTION: Output carefully referenced "Old Design" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Constant text "{project_name}". ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -229,50 +230,20 @@ class WriteDesign(Action): async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) + format_example = format_example.format(project_name=CONFIG.project_name) prompt = prompt_template.format(context=context, format_example=format_example) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - self._rename_project_name(system_design=system_design) - await self._rename_workspace(system_design) return system_design async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): - prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content) + prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content, + project_name=CONFIG.project_name) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python # package name" contain space, have to use setattr - self._rename_project_name(system_design=system_design) system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) return system_design_doc - @staticmethod - def _rename_project_name(system_design): - # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" - # contain space, have to use setattr - if CONFIG.project_name: - setattr( - system_design.instruct_content, - "project_name", - CONFIG.project_name, - ) - return - setattr( - system_design.instruct_content, - "project_name", - system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), - ) - - @staticmethod - async def _rename_workspace(system_design): - if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to - # Section 2.2.3.10 of RFC 135 - return - - if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["project_name"] - else: - ws_name = CodeParser.parse_str(block="project_name", text=system_design) - CONFIG.git_repo.rename_root(ws_name) - async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: prd = await prds_file_repo.get(filename) old_system_design_doc = await system_design_file_repo.get(filename) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 3d59daeed..95da0d65a 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -183,6 +183,10 @@ MERGE_PROMPT = """ ## Old Tasks {old_tasks} ----- + +## Format example +{format_example} +----- Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules. Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. @@ -201,7 +205,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format, +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Format example" format, and only output the json inside this tag, nothing else """ @@ -264,7 +268,9 @@ class WriteTasks(Action): return rsp async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content) + _, format_example = get_template(templates, format) + prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content, + format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) task_doc.content = rsp.instruct_content.json(ensure_ascii=False) return task_doc diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py index 88a37536b..d9cb47021 100644 --- a/metagpt/actions/summarize_code.py +++ b/metagpt/actions/summarize_code.py @@ -3,7 +3,9 @@ """ @Author : alexanderwu @File : summarize_code.py +@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode. """ +from pathlib import Path from tenacity import retry, stop_after_attempt, wait_fixed @@ -95,8 +97,10 @@ class SummarizeCode(Action): return code_rsp async def run(self): - design_doc = await FileRepository.get_file(self.context.design_filename) - task_doc = await FileRepository.get_file(self.context.task_filename) + design_pathname = Path(self.context.design_filename) + design_doc = await FileRepository.get_file(filename=design_pathname.name, relative_path=design_pathname.parent) + task_pathname = Path(self.context.task_filename) + task_doc = await FileRepository.get_file(filename=task_pathname.name, relative_path=task_pathname.parent) src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) code_blocks = [] for filename in self.context.codes_filenames: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 59ccb49a5..86cd24e33 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -19,7 +19,7 @@ from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action -from metagpt.const import TEST_OUTPUTS_FILE_REPO +from metagpt.const import TEST_OUTPUTS_FILE_REPO, CODE_SUMMARIES_FILE_REPO from metagpt.logs import logger from metagpt.schema import CodingContext, RunCodeResult from metagpt.utils.common import CodeParser @@ -50,6 +50,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc # Debug logs ```text {logs} + +{summary_log} ``` ----- @@ -90,6 +92,8 @@ class WriteCode(Action): test_doc = await FileRepository.get_file( filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO ) + summary_doc = await FileRepository.get_file(filename=coding_context.design_doc.filename, + relative_path=CODE_SUMMARIES_FILE_REPO) logs = "" if test_doc: test_detail = RunCodeResult.loads(test_doc.content) @@ -100,6 +104,7 @@ class WriteCode(Action): code=coding_context.code_doc.content, logs=logs, filename=self.context.filename, + summary_log=summary_doc.content if summary_doc else "" ) logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 3967a0578..ed133abfd 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -8,6 +8,7 @@ 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. 3. Move the document storage operations related to WritePRD from the save operation of WriteDesign. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ from __future__ import annotations @@ -27,6 +28,7 @@ from metagpt.const import ( ) from metagpt.logs import logger from metagpt.schema import Document, Documents +from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -53,7 +55,7 @@ ATTENTION: Output carefully referenced "Format example" in format. {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "project_name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -85,9 +87,10 @@ and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Language": "", "Original Requirements": "", + "project_name": "{project_name}", "Search Information": "", "Requirements": "", "Product Goals": [], @@ -111,7 +114,7 @@ and only output the json inside this tag, nothing else "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], "UI Design draft": "", "Anything UNCLEAR": "", -} +}} [/CONTENT] """, }, @@ -228,6 +231,7 @@ There are no unclear points. OUTPUT_MAPPING = { "Language": (str, ...), "Original Requirements": (str, ...), + "project_name": (str, ...), "Product Goals": (List[str], ...), "User Stories": (List[str], ...), "Competitive Analysis": (List[str], ...), @@ -270,7 +274,7 @@ ATTENTION: Output carefully referenced "Old PRD" in format. {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "project_name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -343,14 +347,18 @@ class WritePRD(Action): # logger.info(format) prompt_template, format_example = get_template(templates, format) + project_name = CONFIG.project_name if CONFIG.project_name else "" + format_example = format_example.format(project_name=project_name) # logger.info(prompt_template) # logger.info(format_example) prompt = prompt_template.format( - requirements=requirements, search_information=info, format_example=format_example + requirements=requirements, search_information=info, format_example=format_example, + project_name=project_name ) # logger.info(prompt) # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) + await self._rename_workspace(prd) return prd async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: @@ -366,9 +374,13 @@ class WritePRD(Action): return False async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content, + project_name=CONFIG.project_name) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) prd_doc.content = prd.instruct_content.json(ensure_ascii=False) + await self._rename_workspace(prd) return prd_doc async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: @@ -404,3 +416,19 @@ class WritePRD(Action): @staticmethod async def _save_pdf(prd_doc): await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO) + + @staticmethod + async def _rename_workspace(prd): + if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to + # Section 2.2.3.10 of RFC 135 + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + return + + if not CONFIG.project_name: + if isinstance(prd, ActionOutput): + ws_name = prd.instruct_content.dict()["project_name"] + else: + ws_name = CodeParser.parse_str(block="project_name", text=prd) + CONFIG.project_name = ws_name + CONFIG.git_repo.rename_root(CONFIG.project_name) \ No newline at end of file diff --git a/metagpt/const.py b/metagpt/const.py index a646cea7a..bd735a5e1 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for common properties in the Message. @Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. +@Modified By: mashenquan, 2023/12/5. Add directories for code summarization.. """ import contextvars import os @@ -87,5 +88,7 @@ PRD_PDF_FILE_REPO = "resources/prd" TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" TEST_CODES_FILE_REPO = "tests" TEST_OUTPUTS_FILE_REPO = "test_outputs" +CODE_SUMMARIES_FILE_REPO = "docs/code_summaries" +CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries" YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index d42835a1b..caff1c680 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -13,6 +13,8 @@ @Modified By: mashenquan, 2023-11-27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.5 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from __future__ import annotations @@ -23,7 +25,8 @@ from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG -from metagpt.const import MESSAGE_ROUTE_TO_NONE, SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, CODE_SUMMARIES_FILE_REPO, \ + CODE_SUMMARIES_PDF_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import ( @@ -33,6 +36,16 @@ from metagpt.schema import ( Documents, Message, ) +from metagpt.utils.common import any_to_str_set, any_to_str + +IS_PASS_PROMPT = """ +{context} + +---- +Does the above log indicate anything that needs to be done? +If there are any tasks to be completed, please answer 'NO' along with the to-do list in JSON format; +otherwise, answer 'YES' in JSON format. +""" class Engineer(Role): @@ -49,18 +62,18 @@ class Engineer(Role): """ def __init__( - self, - name: str = "Alex", - profile: str = "Engineer", - goal: str = "Write elegant, readable, extensible, efficient code", - constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", - n_borg: int = 1, - use_code_review: bool = False, + self, + name: str = "Alex", + profile: str = "Engineer", + goal: str = "Write elegant, readable, extensible, efficient code", + constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", + n_borg: int = 1, + use_code_review: bool = False, ) -> None: """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self.use_code_review = use_code_review - self._watch([WriteTasks]) + self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview]) self.code_todos = [] self.summarize_todos = [] self.n_borg = n_borg @@ -105,43 +118,85 @@ class Engineer(Role): if self._rc.todo is None: return None if isinstance(self._rc.todo, WriteCode): - changed_files = await self._act_sp_with_cr(review=self.use_code_review) - # Unit tests only. - if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files: - changed_files.add(CONFIG.REQA_FILENAME) - return Message( - content="\n".join(changed_files), - role=self.profile, - cause_by=WriteCodeReview if self.use_code_review else WriteCode, - send_to="Edward", # The name of QaEngineer - ) + return await self._act_write_code() if isinstance(self._rc.todo, SummarizeCode): - summaries = [] - for todo in self.summarize_todos: - summary = await todo.run() - summaries.append(summary.json(ensure_ascii=False)) + return await self._act_summarize() + return None + + async def _act_write_code(self): + changed_files = await self._act_sp_with_cr(review=self.use_code_review) + return Message( + content="\n".join(changed_files), + role=self.profile, + cause_by=WriteCodeReview if self.use_code_review else WriteCode, + send_to=self, + sent_from=self + ) + + async def _act_summarize(self): + code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO) + code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO) + tasks = [] + src_relative_path = CONFIG.src_workspace.relative_to(CONFIG.git_repo.workdir) + for todo in self.summarize_todos: + summary = await todo.run() + summary_filename = Path(todo.context.design_filename).with_suffix(".md").name + dependencies = {todo.context.design_filename, todo.context.task_filename} + for filename in todo.context.codes_filenames: + rpath = src_relative_path / filename + dependencies.add(str(rpath)) + await code_summaries_pdf_file_repo.save(filename=summary_filename, content=summary, + dependencies=dependencies) + is_pass, reason = await self._is_pass(summary) + if not is_pass: + todo.context.reason = reason + tasks.append(todo.context.dict()) + await code_summaries_file_repo.save(filename=Path(todo.context.design_filename).name, + content=todo.context.json(), dependencies=dependencies) + + if not tasks: return Message( - content="\n".join(summaries), + content="", role=self.profile, cause_by=SummarizeCode, - send_to=MESSAGE_ROUTE_TO_NONE, + sent_from=self, + send_to="Edward", # The name of QaEngineer ) - return None + return Message( + content=json.dumps(tasks), + role=self.profile, + cause_by=SummarizeCode, + send_to=self, + sent_from=self + ) + + async def _is_pass(self, summary) -> (str, str): + msgs = [{"role": "user", "content": IS_PASS_PROMPT.format(context=summary)}] + rsp = await self._llm.acompletion_text(messages=msgs, stream=False) + logger.info(rsp) + if "YES" in rsp: + return True, rsp + return False, rsp async def _think(self) -> Action | None: if not CONFIG.src_workspace: CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name - if not self.code_todos: - await self._new_code_actions() - elif not self.summarize_todos: - await self._new_summarize_actions() - else: + write_code_filters = any_to_str_set([WriteTasks, SummarizeCode]) + summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) + if not self._rc.news: return None - return self._rc.todo # For agent store + msg = self._rc.news[0] + if msg.cause_by in write_code_filters: + await self._new_code_actions() + return self._rc.todo + if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): + await self._new_summarize_actions() + return self._rc.todo + return None @staticmethod async def _new_coding_context( - filename, src_file_repo, task_file_repo, design_file_repo, dependency + filename, src_file_repo, task_file_repo, design_file_repo, dependency ) -> CodingContext: old_code_doc = await src_file_repo.get(filename) if not old_code_doc: @@ -216,16 +271,16 @@ class Engineer(Role): async def _new_summarize_actions(self): src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - changed_src_files = src_file_repo.changed_files + src_files = src_file_repo.all_files # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). summarizations = {} - for filename in changed_src_files: - dependencies = src_file_repo.get_dependency(filename=filename) + for filename in src_files: + dependencies = await src_file_repo.get_dependency(filename=filename) ctx = CodeSummarizeContext.loads(filenames=dependencies) if ctx not in summarizations: - summarizations[ctx] = set() + summarizations[ctx] = [] srcs = summarizations.get(ctx) - srcs.add(filename) + srcs.append(filename) for ctx, filenames in summarizations.items(): ctx.codes_filenames = filenames self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm)) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 41a3213dc..15a01b9e9 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -11,10 +11,13 @@ WriteTest/RunCode/DebugError object, rather than passing them in when calling the run function. 2. According to Section 2.2.3.5.7 of RFC 135, change the method of transferring files from using the Message to using file references. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest # from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( MESSAGE_ROUTE_TO_NONE, @@ -40,13 +43,16 @@ class QaEngineer(Role): self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates - self._watch([WriteCode, WriteCodeReview, WriteTest, RunCode, DebugError]) + self._watch([SummarizeCode, WriteTest, RunCode, DebugError]) self.test_round = 0 self.test_round_allowed = test_round_allowed async def _write_test(self, message: Message) -> None: - changed_files = message.content.splitlines() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + changed_files = set(src_file_repo.changed_files.keys()) + # Unit tests only. + if CONFIG.reqa_file and CONFIG.reqa_file not in changed_files: + changed_files.add(CONFIG.reqa_file) tests_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) for filename in changed_files: # write tests @@ -146,7 +152,7 @@ class QaEngineer(Role): ) return result_msg - code_filters = any_to_str_set({WriteCode, WriteCodeReview}) + code_filters = any_to_str_set({SummarizeCode}) test_filters = any_to_str_set({WriteTest, DebugError}) run_filters = any_to_str_set({RunCode}) for msg in self._rc.news: diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1e99cc1ff..2651be7eb 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -284,9 +284,10 @@ class Role: instruct_content=response.instruct_content, role=self.profile, cause_by=self._rc.todo, + sent_from=self, ) else: - msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) + msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self) self._rc.memory.add(msg) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index d1174799a..51f395e65 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -324,10 +324,11 @@ class RunCodeResult(BaseModel): class CodeSummarizeContext(BaseModel): design_filename: str = "" task_filename: str = "" - codes_filenames: Set[str] = Field(default_factory=set) + codes_filenames: List[str] = Field(default_factory=list) + reason: str = "" @staticmethod - def loads(filenames: Set) -> CodeSummarizeContext: + def loads(filenames: List) -> CodeSummarizeContext: ctx = CodeSummarizeContext() for filename in filenames: if Path(filename).is_relative_to(SYSTEM_DESIGN_FILE_REPO): @@ -337,3 +338,7 @@ class CodeSummarizeContext(BaseModel): ctx.task_filename = str(filename) continue return ctx + + def __hash__(self): + return hash((self.design_filename, self.task_filename)) + diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 0815bf90a..a435a6b8e 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -151,6 +151,17 @@ class FileRepository: relative_files[str(rf)] = ct return relative_files + @property + def all_files(self) -> List: + """Get a dictionary of all files in the repository. + + The dictionary includes file paths relative to the current FileRepository. + + :return: A dictionary where keys are file paths and values are file information. + :rtype: List + """ + return self._git_repo.get_files(relative_path=self._relative_path) + def get_change_dir_files(self, dir: Path | str) -> List: """Get the files in a directory that have changed. diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 7c9ec645f..090b7319d 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -11,7 +11,7 @@ from __future__ import annotations import shutil from enum import Enum from pathlib import Path -from typing import Dict +from typing import Dict, List from git.repo import Repo from git.repo.fun import is_git_dir @@ -200,6 +200,32 @@ class GitRepository: logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) + def get_files(self, relative_path: Path | str) -> List: + """Retrieve a list of files in the specified relative path. + + The method returns a list of file paths relative to the current FileRepository. + + :param relative_path: The relative path within the repository. + :type relative_path: Path or str + :return: A list of file paths in the specified directory. + :rtype: List[str] + """ + try: + relative_path = Path(relative_path).relative_to(self.workdir) + except ValueError: + relative_path = Path(relative_path) + + files = [] + try: + directory_path = Path(self.workdir) / relative_path + for file_path in directory_path.iterdir(): + if file_path.is_file(): + rpath = file_path.relative_to(directory_path) + files.append(str(rpath)) + except Exception as e: + logger.error(f"Error: {e}") + return files + if __name__ == "__main__": path = DEFAULT_WORKSPACE_ROOT / "git" From dac4be4b3e04f656c4f073e2161bd2c79c8eb242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Dec 2023 23:04:07 +0800 Subject: [PATCH 381/398] feat: +SummarizeCode, refactor project_name --- metagpt/actions/design_api.py | 55 +++-------- metagpt/actions/project_management.py | 10 +- metagpt/actions/summarize_code.py | 8 +- metagpt/actions/write_code.py | 7 +- metagpt/actions/write_prd.py | 48 +++++++--- metagpt/const.py | 3 + metagpt/roles/engineer.py | 133 ++++++++++++++++++-------- metagpt/roles/qa_engineer.py | 12 ++- metagpt/roles/role.py | 3 +- metagpt/schema.py | 9 +- metagpt/utils/file_repository.py | 11 +++ metagpt/utils/git_repository.py | 28 +++++- 12 files changed, 224 insertions(+), 103 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index c5787ba20..605b871a1 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023/11/27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ import json from pathlib import Path @@ -43,7 +44,7 @@ Requirement: Fill in the following missing information based on the context, eac ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Constant text. ## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -58,15 +59,15 @@ and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Implementation approach": "We will ...", - "project_name": "snake_game", + "project_name": "{project_name}", "File list": ["main.py"], "Data structures and interfaces": ' classDiagram - class Game{ + class Game{{ +int score - } + }} ... Game "1" -- "1" Food: has ', @@ -77,7 +78,7 @@ and only output the json inside this tag, nothing else G->>M: end game ', "Anything UNCLEAR": "The requirement is clear to me." -} +}} [/CONTENT] """, }, @@ -96,7 +97,7 @@ ATTENTION: Output carefully referenced "Format example" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Constant text. ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -114,7 +115,7 @@ We will ... ## project_name ```python -"snake_game" +"{project_name}" ``` ## File list @@ -173,7 +174,7 @@ ATTENTION: Output carefully referenced "Old Design" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Constant text "{project_name}". ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -229,50 +230,20 @@ class WriteDesign(Action): async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) + format_example = format_example.format(project_name=CONFIG.project_name) prompt = prompt_template.format(context=context, format_example=format_example) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - self._rename_project_name(system_design=system_design) - await self._rename_workspace(system_design) return system_design async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): - prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content) + prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content, + project_name=CONFIG.project_name) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python # package name" contain space, have to use setattr - self._rename_project_name(system_design=system_design) system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) return system_design_doc - @staticmethod - def _rename_project_name(system_design): - # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" - # contain space, have to use setattr - if CONFIG.project_name: - setattr( - system_design.instruct_content, - "project_name", - CONFIG.project_name, - ) - return - setattr( - system_design.instruct_content, - "project_name", - system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), - ) - - @staticmethod - async def _rename_workspace(system_design): - if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to - # Section 2.2.3.10 of RFC 135 - return - - if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["project_name"] - else: - ws_name = CodeParser.parse_str(block="project_name", text=system_design) - CONFIG.git_repo.rename_root(ws_name) - async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: prd = await prds_file_repo.get(filename) old_system_design_doc = await system_design_file_repo.get(filename) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 3d59daeed..95da0d65a 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -183,6 +183,10 @@ MERGE_PROMPT = """ ## Old Tasks {old_tasks} ----- + +## Format example +{format_example} +----- Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules. Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. @@ -201,7 +205,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format, +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Format example" format, and only output the json inside this tag, nothing else """ @@ -264,7 +268,9 @@ class WriteTasks(Action): return rsp async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content) + _, format_example = get_template(templates, format) + prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content, + format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) task_doc.content = rsp.instruct_content.json(ensure_ascii=False) return task_doc diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py index 88a37536b..d9cb47021 100644 --- a/metagpt/actions/summarize_code.py +++ b/metagpt/actions/summarize_code.py @@ -3,7 +3,9 @@ """ @Author : alexanderwu @File : summarize_code.py +@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode. """ +from pathlib import Path from tenacity import retry, stop_after_attempt, wait_fixed @@ -95,8 +97,10 @@ class SummarizeCode(Action): return code_rsp async def run(self): - design_doc = await FileRepository.get_file(self.context.design_filename) - task_doc = await FileRepository.get_file(self.context.task_filename) + design_pathname = Path(self.context.design_filename) + design_doc = await FileRepository.get_file(filename=design_pathname.name, relative_path=design_pathname.parent) + task_pathname = Path(self.context.task_filename) + task_doc = await FileRepository.get_file(filename=task_pathname.name, relative_path=task_pathname.parent) src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) code_blocks = [] for filename in self.context.codes_filenames: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 59ccb49a5..86cd24e33 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -19,7 +19,7 @@ from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action -from metagpt.const import TEST_OUTPUTS_FILE_REPO +from metagpt.const import TEST_OUTPUTS_FILE_REPO, CODE_SUMMARIES_FILE_REPO from metagpt.logs import logger from metagpt.schema import CodingContext, RunCodeResult from metagpt.utils.common import CodeParser @@ -50,6 +50,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc # Debug logs ```text {logs} + +{summary_log} ``` ----- @@ -90,6 +92,8 @@ class WriteCode(Action): test_doc = await FileRepository.get_file( filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO ) + summary_doc = await FileRepository.get_file(filename=coding_context.design_doc.filename, + relative_path=CODE_SUMMARIES_FILE_REPO) logs = "" if test_doc: test_detail = RunCodeResult.loads(test_doc.content) @@ -100,6 +104,7 @@ class WriteCode(Action): code=coding_context.code_doc.content, logs=logs, filename=self.context.filename, + summary_log=summary_doc.content if summary_doc else "" ) logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 3967a0578..eb89f1ad1 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -8,6 +8,7 @@ 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. 3. Move the document storage operations related to WritePRD from the save operation of WriteDesign. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ from __future__ import annotations @@ -27,6 +28,7 @@ from metagpt.const import ( ) from metagpt.logs import logger from metagpt.schema import Document, Documents +from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -53,7 +55,7 @@ ATTENTION: Output carefully referenced "Format example" in format. {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "project_name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -85,9 +87,10 @@ and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Language": "", "Original Requirements": "", + "project_name": "{project_name}", "Search Information": "", "Requirements": "", "Product Goals": [], @@ -111,7 +114,7 @@ and only output the json inside this tag, nothing else "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], "UI Design draft": "", "Anything UNCLEAR": "", -} +}} [/CONTENT] """, }, @@ -228,6 +231,7 @@ There are no unclear points. OUTPUT_MAPPING = { "Language": (str, ...), "Original Requirements": (str, ...), + "project_name": (str, ...), "Product Goals": (List[str], ...), "User Stories": (List[str], ...), "Competitive Analysis": (List[str], ...), @@ -270,7 +274,7 @@ ATTENTION: Output carefully referenced "Old PRD" in format. {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "project_name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -320,6 +324,7 @@ class WritePRD(Action): if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"REWRITE PRD:{prd_doc.filename}") # If there is no existing PRD, generate one using 'docs/requirement.txt'. if not change_files.docs: prd_doc = await self._update_prd( @@ -327,6 +332,7 @@ class WritePRD(Action): ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"NEW PRD:{prd_doc.filename}") # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the # 'publish' message to transition the workflow to the next stage. This design allows room for global # optimization in subsequent steps. @@ -343,32 +349,36 @@ class WritePRD(Action): # logger.info(format) prompt_template, format_example = get_template(templates, format) + project_name = CONFIG.project_name if CONFIG.project_name else "" + format_example = format_example.format(project_name=project_name) # logger.info(prompt_template) # logger.info(format_example) prompt = prompt_template.format( - requirements=requirements, search_information=info, format_example=format_example + requirements=requirements, search_information=info, format_example=format_example, + project_name=project_name ) # logger.info(prompt) # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) + await self._rename_workspace(prd) return prd async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: - m = json.loads(old_prd_doc.content) - if m.get("Original Requirements") == new_requirement_doc.content: - # There have been no changes in the requirements, so they are considered unrelated. - return False prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) res = await self._aask(prompt=prompt) - logger.info(f"[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") + logger.info(f"REQ-RELATIVE:[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") if "YES" in res: return True return False async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content, + project_name=CONFIG.project_name) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) prd_doc.content = prd.instruct_content.json(ensure_ascii=False) + await self._rename_workspace(prd) return prd_doc async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: @@ -404,3 +414,19 @@ class WritePRD(Action): @staticmethod async def _save_pdf(prd_doc): await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO) + + @staticmethod + async def _rename_workspace(prd): + if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to + # Section 2.2.3.10 of RFC 135 + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + return + + if not CONFIG.project_name: + if isinstance(prd, ActionOutput): + ws_name = prd.instruct_content.dict()["project_name"] + else: + ws_name = CodeParser.parse_str(block="project_name", text=prd) + CONFIG.project_name = ws_name + CONFIG.git_repo.rename_root(CONFIG.project_name) \ No newline at end of file diff --git a/metagpt/const.py b/metagpt/const.py index a646cea7a..bd735a5e1 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for common properties in the Message. @Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. +@Modified By: mashenquan, 2023/12/5. Add directories for code summarization.. """ import contextvars import os @@ -87,5 +88,7 @@ PRD_PDF_FILE_REPO = "resources/prd" TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" TEST_CODES_FILE_REPO = "tests" TEST_OUTPUTS_FILE_REPO = "test_outputs" +CODE_SUMMARIES_FILE_REPO = "docs/code_summaries" +CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries" YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index d42835a1b..59279c402 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -13,6 +13,8 @@ @Modified By: mashenquan, 2023-11-27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.5 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from __future__ import annotations @@ -23,7 +25,8 @@ from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG -from metagpt.const import MESSAGE_ROUTE_TO_NONE, SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, CODE_SUMMARIES_FILE_REPO, \ + CODE_SUMMARIES_PDF_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import ( @@ -33,6 +36,16 @@ from metagpt.schema import ( Documents, Message, ) +from metagpt.utils.common import any_to_str_set, any_to_str + +IS_PASS_PROMPT = """ +{context} + +---- +Does the above log indicate anything that needs to be done? +If there are any tasks to be completed, please answer 'NO' along with the to-do list in JSON format; +otherwise, answer 'YES' in JSON format. +""" class Engineer(Role): @@ -49,18 +62,18 @@ class Engineer(Role): """ def __init__( - self, - name: str = "Alex", - profile: str = "Engineer", - goal: str = "Write elegant, readable, extensible, efficient code", - constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", - n_borg: int = 1, - use_code_review: bool = False, + self, + name: str = "Alex", + profile: str = "Engineer", + goal: str = "Write elegant, readable, extensible, efficient code", + constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", + n_borg: int = 1, + use_code_review: bool = False, ) -> None: """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self.use_code_review = use_code_review - self._watch([WriteTasks]) + self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview]) self.code_todos = [] self.summarize_todos = [] self.n_borg = n_borg @@ -105,43 +118,87 @@ class Engineer(Role): if self._rc.todo is None: return None if isinstance(self._rc.todo, WriteCode): - changed_files = await self._act_sp_with_cr(review=self.use_code_review) - # Unit tests only. - if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files: - changed_files.add(CONFIG.REQA_FILENAME) - return Message( - content="\n".join(changed_files), - role=self.profile, - cause_by=WriteCodeReview if self.use_code_review else WriteCode, - send_to="Edward", # The name of QaEngineer - ) + return await self._act_write_code() if isinstance(self._rc.todo, SummarizeCode): - summaries = [] - for todo in self.summarize_todos: - summary = await todo.run() - summaries.append(summary.json(ensure_ascii=False)) + return await self._act_summarize() + return None + + async def _act_write_code(self): + changed_files = await self._act_sp_with_cr(review=self.use_code_review) + return Message( + content="\n".join(changed_files), + role=self.profile, + cause_by=WriteCodeReview if self.use_code_review else WriteCode, + send_to=self, + sent_from=self + ) + + async def _act_summarize(self): + code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO) + code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO) + tasks = [] + src_relative_path = CONFIG.src_workspace.relative_to(CONFIG.git_repo.workdir) + for todo in self.summarize_todos: + summary = await todo.run() + summary_filename = Path(todo.context.design_filename).with_suffix(".md").name + dependencies = {todo.context.design_filename, todo.context.task_filename} + for filename in todo.context.codes_filenames: + rpath = src_relative_path / filename + dependencies.add(str(rpath)) + await code_summaries_pdf_file_repo.save(filename=summary_filename, content=summary, + dependencies=dependencies) + is_pass, reason = await self._is_pass(summary) + if not is_pass: + todo.context.reason = reason + tasks.append(todo.context.dict()) + await code_summaries_file_repo.save(filename=Path(todo.context.design_filename).name, + content=todo.context.json(), dependencies=dependencies) + + if not tasks: return Message( - content="\n".join(summaries), + content="", role=self.profile, cause_by=SummarizeCode, - send_to=MESSAGE_ROUTE_TO_NONE, + sent_from=self, + send_to="Edward", # The name of QaEngineer ) - return None + return Message( + content=json.dumps(tasks), + role=self.profile, + cause_by=SummarizeCode, + send_to=self, + sent_from=self + ) + + async def _is_pass(self, summary) -> (str, str): + msgs = [{"role": "user", "content": IS_PASS_PROMPT.format(context=summary)}] + rsp = await self._llm.acompletion_text(messages=msgs, stream=False) + logger.info(rsp) + if "YES" in rsp: + return True, rsp + return False, rsp async def _think(self) -> Action | None: if not CONFIG.src_workspace: CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name - if not self.code_todos: - await self._new_code_actions() - elif not self.summarize_todos: - await self._new_summarize_actions() - else: + write_code_filters = any_to_str_set([WriteTasks, SummarizeCode]) + summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) + if not self._rc.news: return None - return self._rc.todo # For agent store + msg = self._rc.news[0] + if msg.cause_by in write_code_filters: + logger.info(f"TODO WriteCode:{msg.json()}") + await self._new_code_actions() + return self._rc.todo + if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): + logger.info(f"TODO SummarizeCode:{msg.json()}") + await self._new_summarize_actions() + return self._rc.todo + return None @staticmethod async def _new_coding_context( - filename, src_file_repo, task_file_repo, design_file_repo, dependency + filename, src_file_repo, task_file_repo, design_file_repo, dependency ) -> CodingContext: old_code_doc = await src_file_repo.get(filename) if not old_code_doc: @@ -216,16 +273,16 @@ class Engineer(Role): async def _new_summarize_actions(self): src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - changed_src_files = src_file_repo.changed_files + src_files = src_file_repo.all_files # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). summarizations = {} - for filename in changed_src_files: - dependencies = src_file_repo.get_dependency(filename=filename) + for filename in src_files: + dependencies = await src_file_repo.get_dependency(filename=filename) ctx = CodeSummarizeContext.loads(filenames=dependencies) if ctx not in summarizations: - summarizations[ctx] = set() + summarizations[ctx] = [] srcs = summarizations.get(ctx) - srcs.add(filename) + srcs.append(filename) for ctx, filenames in summarizations.items(): ctx.codes_filenames = filenames self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm)) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 41a3213dc..15a01b9e9 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -11,10 +11,13 @@ WriteTest/RunCode/DebugError object, rather than passing them in when calling the run function. 2. According to Section 2.2.3.5.7 of RFC 135, change the method of transferring files from using the Message to using file references. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest # from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( MESSAGE_ROUTE_TO_NONE, @@ -40,13 +43,16 @@ class QaEngineer(Role): self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates - self._watch([WriteCode, WriteCodeReview, WriteTest, RunCode, DebugError]) + self._watch([SummarizeCode, WriteTest, RunCode, DebugError]) self.test_round = 0 self.test_round_allowed = test_round_allowed async def _write_test(self, message: Message) -> None: - changed_files = message.content.splitlines() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + changed_files = set(src_file_repo.changed_files.keys()) + # Unit tests only. + if CONFIG.reqa_file and CONFIG.reqa_file not in changed_files: + changed_files.add(CONFIG.reqa_file) tests_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) for filename in changed_files: # write tests @@ -146,7 +152,7 @@ class QaEngineer(Role): ) return result_msg - code_filters = any_to_str_set({WriteCode, WriteCodeReview}) + code_filters = any_to_str_set({SummarizeCode}) test_filters = any_to_str_set({WriteTest, DebugError}) run_filters = any_to_str_set({RunCode}) for msg in self._rc.news: diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1e99cc1ff..2651be7eb 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -284,9 +284,10 @@ class Role: instruct_content=response.instruct_content, role=self.profile, cause_by=self._rc.todo, + sent_from=self, ) else: - msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) + msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self) self._rc.memory.add(msg) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index d1174799a..51f395e65 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -324,10 +324,11 @@ class RunCodeResult(BaseModel): class CodeSummarizeContext(BaseModel): design_filename: str = "" task_filename: str = "" - codes_filenames: Set[str] = Field(default_factory=set) + codes_filenames: List[str] = Field(default_factory=list) + reason: str = "" @staticmethod - def loads(filenames: Set) -> CodeSummarizeContext: + def loads(filenames: List) -> CodeSummarizeContext: ctx = CodeSummarizeContext() for filename in filenames: if Path(filename).is_relative_to(SYSTEM_DESIGN_FILE_REPO): @@ -337,3 +338,7 @@ class CodeSummarizeContext(BaseModel): ctx.task_filename = str(filename) continue return ctx + + def __hash__(self): + return hash((self.design_filename, self.task_filename)) + diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 0815bf90a..a435a6b8e 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -151,6 +151,17 @@ class FileRepository: relative_files[str(rf)] = ct return relative_files + @property + def all_files(self) -> List: + """Get a dictionary of all files in the repository. + + The dictionary includes file paths relative to the current FileRepository. + + :return: A dictionary where keys are file paths and values are file information. + :rtype: List + """ + return self._git_repo.get_files(relative_path=self._relative_path) + def get_change_dir_files(self, dir: Path | str) -> List: """Get the files in a directory that have changed. diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 7c9ec645f..090b7319d 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -11,7 +11,7 @@ from __future__ import annotations import shutil from enum import Enum from pathlib import Path -from typing import Dict +from typing import Dict, List from git.repo import Repo from git.repo.fun import is_git_dir @@ -200,6 +200,32 @@ class GitRepository: logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) + def get_files(self, relative_path: Path | str) -> List: + """Retrieve a list of files in the specified relative path. + + The method returns a list of file paths relative to the current FileRepository. + + :param relative_path: The relative path within the repository. + :type relative_path: Path or str + :return: A list of file paths in the specified directory. + :rtype: List[str] + """ + try: + relative_path = Path(relative_path).relative_to(self.workdir) + except ValueError: + relative_path = Path(relative_path) + + files = [] + try: + directory_path = Path(self.workdir) / relative_path + for file_path in directory_path.iterdir(): + if file_path.is_file(): + rpath = file_path.relative_to(directory_path) + files.append(str(rpath)) + except Exception as e: + logger.error(f"Error: {e}") + return files + if __name__ == "__main__": path = DEFAULT_WORKSPACE_ROOT / "git" From 526d56cf5464a441a44ff1e20c361b5abe373bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Dec 2023 10:10:30 +0800 Subject: [PATCH 382/398] feat: upgrade openai 1.x --- metagpt/llm.py | 13 +- metagpt/provider/general_api_base.py | 718 +++++++++++++++++++ metagpt/provider/general_api_requestor.py | 6 +- metagpt/provider/openai_api.py | 1 - metagpt/provider/zhipuai/async_sse_client.py | 7 +- metagpt/provider/zhipuai/zhipu_model_api.py | 4 +- metagpt/provider/zhipuai_api.py | 24 +- 7 files changed, 747 insertions(+), 26 deletions(-) create mode 100644 metagpt/provider/general_api_base.py diff --git a/metagpt/llm.py b/metagpt/llm.py index dce33b9db..2ad40cb1c 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -4,23 +4,20 @@ @Time : 2023/5/11 14:45 @Author : alexanderwu @File : llm.py -@Modified By: mashenquan, 2023-12-4. Upgrade openai to 1.x """ from metagpt.config import CONFIG from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.human_provider import HumanProvider from metagpt.provider.openai_api import OpenAIGPTAPI +from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI from metagpt.provider.spark_api import SparkAPI -# openai v1.x removed the 'api_requestor', making interfaces built on it no longer functional. -# More: https://github.com/openai/openai-python/discussions/742 -# from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI +from metagpt.provider.human_provider import HumanProvider _ = HumanProvider() # Avoid pre-commit error def LLM() -> "BaseGPTAPI": - """initialize different LLM instance according to the key field existence""" + """ initialize different LLM instance according to the key field existence""" # TODO a little trick, can use registry to initialize LLM instance further if CONFIG.openai_api_key: llm = OpenAIGPTAPI() @@ -28,8 +25,8 @@ def LLM() -> "BaseGPTAPI": llm = Claude() elif CONFIG.spark_api_key: llm = SparkAPI() - # elif CONFIG.zhipuai_api_key: # openai v1.x removed the 'api_requestor' - # llm = ZhiPuAIGPTAPI() + elif CONFIG.zhipuai_api_key: + llm = ZhiPuAIGPTAPI() else: raise RuntimeError("You should config a LLM configuration first") diff --git a/metagpt/provider/general_api_base.py b/metagpt/provider/general_api_base.py new file mode 100644 index 000000000..da16e942d --- /dev/null +++ b/metagpt/provider/general_api_base.py @@ -0,0 +1,718 @@ +import asyncio +import json +import os +import platform +import re +import sys +import threading +import time +from contextlib import asynccontextmanager +from enum import Enum +from typing import ( + AsyncGenerator, + AsyncIterator, + Callable, + Dict, + Iterator, + Optional, + Tuple, + Union, + overload, +) +from urllib.parse import urlencode, urlsplit, urlunsplit + +import aiohttp +import requests + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +import logging + +import openai +from openai import version + +logger = logging.getLogger("openai") + +TIMEOUT_SECS = 600 +MAX_SESSION_LIFETIME_SECS = 180 +MAX_CONNECTION_RETRIES = 2 + +# Has one attribute per thread, 'session'. +_thread_context = threading.local() + +OPENAI_LOG = os.environ.get("OPENAI_LOG") +OPENAI_LOG = "debug" + + +class ApiType(Enum): + AZURE = 1 + OPEN_AI = 2 + AZURE_AD = 3 + + @staticmethod + def from_str(label): + if label.lower() == "azure": + return ApiType.AZURE + elif label.lower() in ("azure_ad", "azuread"): + return ApiType.AZURE_AD + elif label.lower() in ("open_ai", "openai"): + return ApiType.OPEN_AI + else: + raise openai.OpenAIError( + "The API type provided in invalid. Please select one of the supported API types: 'azure', 'azure_ad', 'open_ai'" + ) + + +api_key_to_header = ( + lambda api, key: {"Authorization": f"Bearer {key}"} + if api in (ApiType.OPEN_AI, ApiType.AZURE_AD) + else {"api-key": f"{key}"} +) + + +def _console_log_level(): + if OPENAI_LOG in ["debug", "info"]: + return OPENAI_LOG + else: + return None + + +def log_debug(message, **params): + msg = logfmt(dict(message=message, **params)) + if _console_log_level() == "debug": + print(msg, file=sys.stderr) + logger.debug(msg) + + +def log_info(message, **params): + msg = logfmt(dict(message=message, **params)) + if _console_log_level() in ["debug", "info"]: + print(msg, file=sys.stderr) + logger.info(msg) + + +def log_warn(message, **params): + msg = logfmt(dict(message=message, **params)) + print(msg, file=sys.stderr) + logger.warn(msg) + + +def logfmt(props): + def fmt(key, val): + # Handle case where val is a bytes or bytesarray + if hasattr(val, "decode"): + val = val.decode("utf-8") + # Check if val is already a string to avoid re-encoding into ascii. + if not isinstance(val, str): + val = str(val) + if re.search(r"\s", val): + val = repr(val) + # key should already be a string + if re.search(r"\s", key): + key = repr(key) + return "{key}={val}".format(key=key, val=val) + + return " ".join([fmt(key, val) for key, val in sorted(props.items())]) + + +class OpenAIResponse: + def __init__(self, data, headers): + self._headers = headers + self.data = data + + @property + def request_id(self) -> Optional[str]: + return self._headers.get("request-id") + + @property + def retry_after(self) -> Optional[int]: + try: + return int(self._headers.get("retry-after")) + except TypeError: + return None + + @property + def operation_location(self) -> Optional[str]: + return self._headers.get("operation-location") + + @property + def organization(self) -> Optional[str]: + return self._headers.get("OpenAI-Organization") + + @property + def response_ms(self) -> Optional[int]: + h = self._headers.get("Openai-Processing-Ms") + return None if h is None else round(float(h)) + + +def _build_api_url(url, query): + scheme, netloc, path, base_query, fragment = urlsplit(url) + + if base_query: + query = "%s&%s" % (base_query, query) + + return urlunsplit((scheme, netloc, path, query, fragment)) + + +def _requests_proxies_arg(proxy) -> Optional[Dict[str, str]]: + """Returns a value suitable for the 'proxies' argument to 'requests.request.""" + if proxy is None: + return None + elif isinstance(proxy, str): + return {"http": proxy, "https": proxy} + elif isinstance(proxy, dict): + return proxy.copy() + else: + raise ValueError( + "'openai.proxy' must be specified as either a string URL or a dict with string URL under the https and/or http keys." + ) + + +def _aiohttp_proxies_arg(proxy) -> Optional[str]: + """Returns a value suitable for the 'proxies' argument to 'aiohttp.ClientSession.request.""" + if proxy is None: + return None + elif isinstance(proxy, str): + return proxy + elif isinstance(proxy, dict): + return proxy["https"] if "https" in proxy else proxy["http"] + else: + raise ValueError( + "'openai.proxy' must be specified as either a string URL or a dict with string URL under the https and/or http keys." + ) + + +def _make_session() -> requests.Session: + s = requests.Session() + s.mount( + "https://", + requests.adapters.HTTPAdapter(max_retries=MAX_CONNECTION_RETRIES), + ) + return s + + +def parse_stream_helper(line: bytes) -> Optional[str]: + if line: + if line.strip() == b"data: [DONE]": + # return here will cause GeneratorExit exception in urllib3 + # and it will close http connection with TCP Reset + return None + if line.startswith(b"data: "): + line = line[len(b"data: ") :] + return line.decode("utf-8") + else: + return None + return None + + +def parse_stream(rbody: Iterator[bytes]) -> Iterator[str]: + for line in rbody: + _line = parse_stream_helper(line) + if _line is not None: + yield _line + + +async def parse_stream_async(rbody: aiohttp.StreamReader): + async for line in rbody: + _line = parse_stream_helper(line) + if _line is not None: + yield _line + + +class APIRequestor: + def __init__( + self, + key=None, + base_url=None, + api_type=None, + api_version=None, + organization=None, + ): + self.base_url = base_url or openai.base_url + self.api_key = key or openai.api_key + self.api_type = ApiType.from_str(api_type) if api_type else ApiType.from_str("openai") + self.api_version = api_version or openai.api_version + self.organization = organization or openai.organization + + def _check_polling_response(self, response: OpenAIResponse, predicate: Callable[[OpenAIResponse], bool]): + if not predicate(response): + return + error_data = response.data["error"] + message = error_data.get("message", "Operation failed") + code = error_data.get("code") + raise openai.APIError(message=message, body=dict(code=code)) + + def _poll( + self, method, url, until, failed, params=None, headers=None, interval=None, delay=None + ) -> Tuple[Iterator[OpenAIResponse], bool, str]: + if delay: + time.sleep(delay) + + response, b, api_key = self.request(method, url, params, headers) + self._check_polling_response(response, failed) + start_time = time.time() + while not until(response): + if time.time() - start_time > TIMEOUT_SECS: + raise openai.APITimeoutError("Operation polling timed out.") + + time.sleep(interval or response.retry_after or 10) + response, b, api_key = self.request(method, url, params, headers) + self._check_polling_response(response, failed) + + response.data = response.data["result"] + return response, b, api_key + + async def _apoll( + self, method, url, until, failed, params=None, headers=None, interval=None, delay=None + ) -> Tuple[Iterator[OpenAIResponse], bool, str]: + if delay: + await asyncio.sleep(delay) + + response, b, api_key = await self.arequest(method, url, params, headers) + self._check_polling_response(response, failed) + start_time = time.time() + while not until(response): + if time.time() - start_time > TIMEOUT_SECS: + raise openai.APITimeoutError("Operation polling timed out.") + + await asyncio.sleep(interval or response.retry_after or 10) + response, b, api_key = await self.arequest(method, url, params, headers) + self._check_polling_response(response, failed) + + response.data = response.data["result"] + return response, b, api_key + + @overload + def request( + self, + method, + url, + params, + headers, + files, + stream: Literal[True], + request_id: Optional[str] = ..., + request_timeout: Optional[Union[float, Tuple[float, float]]] = ..., + ) -> Tuple[Iterator[OpenAIResponse], bool, str]: + pass + + @overload + def request( + self, + method, + url, + params=..., + headers=..., + files=..., + *, + stream: Literal[True], + request_id: Optional[str] = ..., + request_timeout: Optional[Union[float, Tuple[float, float]]] = ..., + ) -> Tuple[Iterator[OpenAIResponse], bool, str]: + pass + + @overload + def request( + self, + method, + url, + params=..., + headers=..., + files=..., + stream: Literal[False] = ..., + request_id: Optional[str] = ..., + request_timeout: Optional[Union[float, Tuple[float, float]]] = ..., + ) -> Tuple[OpenAIResponse, bool, str]: + pass + + @overload + def request( + self, + method, + url, + params=..., + headers=..., + files=..., + stream: bool = ..., + request_id: Optional[str] = ..., + request_timeout: Optional[Union[float, Tuple[float, float]]] = ..., + ) -> Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], bool, str]: + pass + + def request( + self, + method, + url, + params=None, + headers=None, + files=None, + stream: bool = False, + request_id: Optional[str] = None, + request_timeout: Optional[Union[float, Tuple[float, float]]] = None, + ) -> Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], bool, str]: + result = self.request_raw( + method.lower(), + url, + params=params, + supplied_headers=headers, + files=files, + stream=stream, + request_id=request_id, + request_timeout=request_timeout, + ) + resp, got_stream = self._interpret_response(result, stream) + return resp, got_stream, self.api_key + + @overload + async def arequest( + self, + method, + url, + params, + headers, + files, + stream: Literal[True], + request_id: Optional[str] = ..., + request_timeout: Optional[Union[float, Tuple[float, float]]] = ..., + ) -> Tuple[AsyncGenerator[OpenAIResponse, None], bool, str]: + pass + + @overload + async def arequest( + self, + method, + url, + params=..., + headers=..., + files=..., + *, + stream: Literal[True], + request_id: Optional[str] = ..., + request_timeout: Optional[Union[float, Tuple[float, float]]] = ..., + ) -> Tuple[AsyncGenerator[OpenAIResponse, None], bool, str]: + pass + + @overload + async def arequest( + self, + method, + url, + params=..., + headers=..., + files=..., + stream: Literal[False] = ..., + request_id: Optional[str] = ..., + request_timeout: Optional[Union[float, Tuple[float, float]]] = ..., + ) -> Tuple[OpenAIResponse, bool, str]: + pass + + @overload + async def arequest( + self, + method, + url, + params=..., + headers=..., + files=..., + stream: bool = ..., + request_id: Optional[str] = ..., + request_timeout: Optional[Union[float, Tuple[float, float]]] = ..., + ) -> Tuple[Union[OpenAIResponse, AsyncGenerator[OpenAIResponse, None]], bool, str]: + pass + + async def arequest( + self, + method, + url, + params=None, + headers=None, + files=None, + stream: bool = False, + request_id: Optional[str] = None, + request_timeout: Optional[Union[float, Tuple[float, float]]] = None, + ) -> Tuple[Union[OpenAIResponse, AsyncGenerator[OpenAIResponse, None]], bool, str]: + ctx = aiohttp_session() + session = await ctx.__aenter__() + try: + result = await self.arequest_raw( + method.lower(), + url, + session, + params=params, + supplied_headers=headers, + files=files, + request_id=request_id, + request_timeout=request_timeout, + ) + resp, got_stream = await self._interpret_async_response(result, stream) + except Exception: + await ctx.__aexit__(None, None, None) + raise + if got_stream: + + async def wrap_resp(): + assert isinstance(resp, AsyncGenerator) + try: + async for r in resp: + yield r + finally: + await ctx.__aexit__(None, None, None) + + return wrap_resp(), got_stream, self.api_key + else: + await ctx.__aexit__(None, None, None) + return resp, got_stream, self.api_key + + def handle_error_response(self, rbody, rcode, resp, rheaders, stream_error=False): + try: + error_data = resp["error"] + except (KeyError, TypeError): + raise openai.APIError( + "Invalid response object from API: %r (HTTP response code " "was %d)" % (rbody, rcode) + ) + + if "internal_message" in error_data: + error_data["message"] += "\n\n" + error_data["internal_message"] + + log_info( + "OpenAI API error received", + error_code=error_data.get("code"), + error_type=error_data.get("type"), + error_message=error_data.get("message"), + error_param=error_data.get("param"), + stream_error=stream_error, + ) + + # Rate limits were previously coded as 400's with code 'rate_limit' + if rcode == 429: + return openai.RateLimitError(f"{error_data.get('message')} {rbody} {rcode} {resp} {rheaders}", body=rbody) + elif rcode in [400, 404, 415]: + return openai.BadRequestError( + message=f'{error_data.get("message")}, {error_data.get("param")}, {error_data.get("code")} {rbody} {rcode} {resp} {rheaders}', + body=rbody, + ) + elif rcode == 401: + return openai.AuthenticationError( + f"{error_data.get('message')} {rbody} {rcode} {resp} {rheaders}", body=rbody + ) + elif rcode == 403: + return openai.PermissionDeniedError( + f"{error_data.get('message')} {rbody} {rcode} {resp} {rheaders}", body=rbody + ) + elif rcode == 409: + return openai.ConflictError(f"{error_data.get('message')} {rbody} {rcode} {resp} {rheaders}", body=rbody) + elif stream_error: + # TODO: we will soon attach status codes to stream errors + parts = [error_data.get("message"), "(Error occurred while streaming.)"] + message = " ".join([p for p in parts if p is not None]) + return openai.APIError(f"{message} {rbody} {rcode} {resp} {rheaders}", body=rbody) + else: + return openai.APIError( + f"{error_data.get('message')} {rbody} {rcode} {resp} {rheaders}", + body=rbody, + ) + + def request_headers(self, method: str, extra, request_id: Optional[str]) -> Dict[str, str]: + user_agent = "OpenAI/v1 PythonBindings/%s" % (version.VERSION,) + + uname_without_node = " ".join(v for k, v in platform.uname()._asdict().items() if k != "node") + ua = { + "bindings_version": version.VERSION, + "httplib": "requests", + "lang": "python", + "lang_version": platform.python_version(), + "platform": platform.platform(), + "publisher": "openai", + "uname": uname_without_node, + } + + headers = { + "X-OpenAI-Client-User-Agent": json.dumps(ua), + "User-Agent": user_agent, + } + + headers.update(api_key_to_header(self.api_type, self.api_key)) + + if self.organization: + headers["OpenAI-Organization"] = self.organization + + if self.api_version is not None and self.api_type == ApiType.OPEN_AI: + headers["OpenAI-Version"] = self.api_version + if request_id is not None: + headers["X-Request-Id"] = request_id + headers.update(extra) + + return headers + + def _validate_headers(self, supplied_headers: Optional[Dict[str, str]]) -> Dict[str, str]: + headers: Dict[str, str] = {} + if supplied_headers is None: + return headers + + if not isinstance(supplied_headers, dict): + raise TypeError("Headers must be a dictionary") + + for k, v in supplied_headers.items(): + if not isinstance(k, str): + raise TypeError("Header keys must be strings") + if not isinstance(v, str): + raise TypeError("Header values must be strings") + headers[k] = v + + # NOTE: It is possible to do more validation of the headers, but a request could always + # be made to the API manually with invalid headers, so we need to handle them server side. + + return headers + + def _prepare_request_raw( + self, + url, + supplied_headers, + method, + params, + files, + request_id: Optional[str], + ) -> Tuple[str, Dict[str, str], Optional[bytes]]: + abs_url = "%s%s" % (self.base_url, url) + headers = self._validate_headers(supplied_headers) + + data = None + if method == "get" or method == "delete": + if params: + encoded_params = urlencode([(k, v) for k, v in params.items() if v is not None]) + abs_url = _build_api_url(abs_url, encoded_params) + elif method in {"post", "put"}: + if params and files: + data = params + if params and not files: + data = json.dumps(params).encode() + headers["Content-Type"] = "application/json" + else: + raise openai.APIConnectionError( + "Unrecognized HTTP method %r. This may indicate a bug in the " + "OpenAI bindings. Please contact us through our help center at help.openai.com for " + "assistance." % (method,) + ) + + headers = self.request_headers(method, headers, request_id) + + log_debug("Request to OpenAI API", method=method, path=abs_url) + log_debug("Post details", data=data, api_version=self.api_version) + + return abs_url, headers, data + + def request_raw( + self, + method, + url, + *, + params=None, + supplied_headers: Optional[Dict[str, str]] = None, + files=None, + stream: bool = False, + request_id: Optional[str] = None, + request_timeout: Optional[Union[float, Tuple[float, float]]] = None, + ) -> requests.Response: + abs_url, headers, data = self._prepare_request_raw(url, supplied_headers, method, params, files, request_id) + + if not hasattr(_thread_context, "session"): + _thread_context.session = _make_session() + _thread_context.session_create_time = time.time() + elif time.time() - getattr(_thread_context, "session_create_time", 0) >= MAX_SESSION_LIFETIME_SECS: + _thread_context.session.close() + _thread_context.session = _make_session() + _thread_context.session_create_time = time.time() + try: + result = _thread_context.session.request( + method, + abs_url, + headers=headers, + data=data, + files=files, + stream=stream, + timeout=request_timeout if request_timeout else TIMEOUT_SECS, + proxies=_thread_context.session.proxies, + ) + except requests.exceptions.Timeout as e: + raise openai.APITimeoutError("Request timed out: {}".format(e)) from e + except requests.exceptions.RequestException as e: + raise openai.APIConnectionError("Error communicating with OpenAI: {}".format(e)) from e + log_debug( + "OpenAI API response", + path=abs_url, + response_code=result.status_code, + processing_ms=result.headers.get("OpenAI-Processing-Ms"), + request_id=result.headers.get("X-Request-Id"), + ) + return result + + async def arequest_raw( + self, + method, + url, + session, + *, + params=None, + supplied_headers: Optional[Dict[str, str]] = None, + files=None, + request_id: Optional[str] = None, + request_timeout: Optional[Union[float, Tuple[float, float]]] = None, + ) -> aiohttp.ClientResponse: + abs_url, headers, data = self._prepare_request_raw(url, supplied_headers, method, params, files, request_id) + + if isinstance(request_timeout, tuple): + timeout = aiohttp.ClientTimeout( + connect=request_timeout[0], + total=request_timeout[1], + ) + else: + timeout = aiohttp.ClientTimeout(total=request_timeout if request_timeout else TIMEOUT_SECS) + + if files: + # TODO: Use `aiohttp.MultipartWriter` to create the multipart form data here. + # For now we use the private `requests` method that is known to have worked so far. + data, content_type = requests.models.RequestEncodingMixin._encode_files(files, data) # type: ignore + headers["Content-Type"] = content_type + request_kwargs = { + "method": method, + "url": abs_url, + "headers": headers, + "data": data, + "timeout": timeout, + } + try: + result = await session.request(**request_kwargs) + log_info( + "OpenAI API response", + path=abs_url, + response_code=result.status, + processing_ms=result.headers.get("OpenAI-Processing-Ms"), + request_id=result.headers.get("X-Request-Id"), + ) + return result + except (aiohttp.ServerTimeoutError, asyncio.TimeoutError) as e: + raise openai.APITimeoutError("Request timed out") from e + except aiohttp.ClientError as e: + raise openai.APIConnectionError("Error communicating with OpenAI") from e + + def _interpret_response( + self, result: requests.Response, stream: bool + ) -> Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], bool]: + """Returns the response(s) and a bool indicating whether it is a stream.""" + + async def _interpret_async_response( + self, result: aiohttp.ClientResponse, stream: bool + ) -> Tuple[Union[OpenAIResponse, AsyncGenerator[OpenAIResponse, None]], bool]: + """Returns the response(s) and a bool indicating whether it is a stream.""" + + def _interpret_response_line(self, rbody: str, rcode: int, rheaders, stream: bool) -> OpenAIResponse: + ... + + +@asynccontextmanager +async def aiohttp_session() -> AsyncIterator[aiohttp.ClientSession]: + async with aiohttp.ClientSession() as session: + yield session diff --git a/metagpt/provider/general_api_requestor.py b/metagpt/provider/general_api_requestor.py index 875122e8b..f8321cc6b 100644 --- a/metagpt/provider/general_api_requestor.py +++ b/metagpt/provider/general_api_requestor.py @@ -6,16 +6,16 @@ import asyncio from typing import AsyncGenerator, Tuple, Union import aiohttp -from openai.api_requestor import APIRequestor from metagpt.logs import logger +from metagpt.provider.general_api_base import APIRequestor class GeneralAPIRequestor(APIRequestor): """ usage - # full_url = "{api_base}{url}" - requester = GeneralAPIRequestor(api_base=api_base) + # full_url = "{base_url}{url}" + requester = GeneralAPIRequestor(base_url=base_url) result, _, api_key = await requester.arequest( method=method, url=url, diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 45fc763be..2d4b1583a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -179,7 +179,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): "n": 1, "stop": None, "temperature": 0.3, - "timeout": 3, } if configs: kwargs.update(configs) diff --git a/metagpt/provider/zhipuai/async_sse_client.py b/metagpt/provider/zhipuai/async_sse_client.py index d7168202a..b819fdc63 100644 --- a/metagpt/provider/zhipuai/async_sse_client.py +++ b/metagpt/provider/zhipuai/async_sse_client.py @@ -3,10 +3,11 @@ # @Desc : async_sse_client to make keep the use of Event to access response # refs to `https://github.com/zhipuai/zhipuai-sdk-python/blob/main/zhipuai/utils/sse_client.py` -from zhipuai.utils.sse_client import _FIELD_SEPARATOR, Event, SSEClient +from zhipuai.utils.sse_client import SSEClient, Event, _FIELD_SEPARATOR class AsyncSSEClient(SSEClient): + async def _aread(self): data = b"" async for chunk in self._event_source: @@ -36,7 +37,9 @@ class AsyncSSEClient(SSEClient): # Ignore unknown fields. if field not in event.__dict__: - self._logger.debug("Saw invalid field %s while parsing " "Server Side Event", field) + self._logger.debug( + "Saw invalid field %s while parsing " "Server Side Event", field + ) continue if len(data) > 1: diff --git a/metagpt/provider/zhipuai/zhipu_model_api.py b/metagpt/provider/zhipuai/zhipu_model_api.py index 23dd7229d..19eb52530 100644 --- a/metagpt/provider/zhipuai/zhipu_model_api.py +++ b/metagpt/provider/zhipuai/zhipu_model_api.py @@ -41,8 +41,8 @@ class ZhiPuModelAPI(ModelAPI): # TODO to make the async request to be more generic for models in http mode. assert method in ["post", "get"] - api_base, url = cls.split_zhipu_api_url(invoke_type, kwargs) - requester = GeneralAPIRequestor(api_base=api_base) + base_url, url = cls.split_zhipu_api_url(invoke_type, kwargs) + requester = GeneralAPIRequestor(base_url=base_url) result, _, api_key = await requester.arequest( method=method, url=url, diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index edd9084e3..3161c0e88 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -2,12 +2,8 @@ # -*- coding: utf-8 -*- # @Desc : zhipuai LLM from https://open.bigmodel.cn/dev/api#sdk -import json from enum import Enum - -import openai -import zhipuai -from requests import ConnectionError +import json from tenacity import ( after_log, retry, @@ -15,6 +11,10 @@ from tenacity import ( stop_after_attempt, wait_fixed, ) +from requests import ConnectionError + +import openai +import zhipuai from metagpt.config import CONFIG from metagpt.logs import logger @@ -50,11 +50,15 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): openai.api_key = zhipuai.api_key # due to use openai sdk, set the api_key but it will't be used. def _const_kwargs(self, messages: list[dict]) -> dict: - kwargs = {"model": self.model, "prompt": messages, "temperature": 0.3} + kwargs = { + "model": self.model, + "prompt": messages, + "temperature": 0.3 + } return kwargs def _update_costs(self, usage: dict): - """update each request's token cost""" + """ update each request's token cost """ if CONFIG.calc_usage: try: prompt_tokens = int(usage.get("prompt_tokens", 0)) @@ -64,7 +68,7 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): logger.error("zhipuai updats costs failed!", e) def get_choice_text(self, resp: dict) -> str: - """get the first text of choice from llm response""" + """ get the first text of choice from llm response """ assist_msg = resp.get("data", {}).get("choices", [{"role": "error"}])[-1] assert assist_msg["role"] == "assistant" return assist_msg.get("content") @@ -125,10 +129,10 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): wait=wait_fixed(1), after=after_log(logger, logger.level("WARNING").name), retry=retry_if_exception_type(ConnectionError), - retry_error_callback=log_and_reraise, + retry_error_callback=log_and_reraise ) async def acompletion_text(self, messages: list[dict], stream=False) -> str: - """response in async with stream or non-stream mode""" + """ response in async with stream or non-stream mode """ if stream: return await self._achat_completion_stream(messages) resp = await self._achat_completion(messages) From 9cc8fd887f68c52509ff7de9c50a4bc9e8029f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Dec 2023 23:04:07 +0800 Subject: [PATCH 383/398] feat: +SummarizeCode, refactor project_name --- metagpt/actions/design_api.py | 61 ++---- metagpt/actions/prepare_documents.py | 2 +- metagpt/actions/project_management.py | 10 +- metagpt/actions/summarize_code.py | 9 +- metagpt/actions/write_code.py | 20 +- metagpt/actions/write_code_review.py | 3 +- metagpt/actions/write_prd.py | 48 ++++- metagpt/actions/write_test.py | 7 +- metagpt/const.py | 3 + metagpt/provider/base_gpt_api.py | 4 +- metagpt/roles/engineer.py | 134 ++++++++---- metagpt/roles/qa_engineer.py | 12 +- metagpt/roles/role.py | 3 +- metagpt/schema.py | 20 +- metagpt/startup.py | 5 + metagpt/utils/dependency_file.py | 5 +- metagpt/utils/file_repository.py | 33 +++ metagpt/utils/git_repository.py | 35 +++- tests/conftest.py | 16 ++ tests/metagpt/actions/mock.py | 2 +- tests/metagpt/actions/test_debug_error.py | 86 ++++---- tests/metagpt/actions/test_design_api.py | 26 +-- .../metagpt/actions/test_prepare_documents.py | 30 +++ tests/metagpt/actions/test_run_code.py | 62 +++--- tests/metagpt/actions/test_summarize_code.py | 195 ++++++++++++++++++ tests/metagpt/actions/test_write_code.py | 17 +- .../metagpt/actions/test_write_code_review.py | 12 +- tests/metagpt/actions/test_write_prd.py | 7 +- tests/metagpt/actions/test_write_test.py | 22 +- tests/metagpt/roles/mock.py | 2 +- tests/metagpt/utils/test_file_repository.py | 4 + 31 files changed, 655 insertions(+), 240 deletions(-) create mode 100644 tests/metagpt/actions/test_prepare_documents.py create mode 100644 tests/metagpt/actions/test_summarize_code.py diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index c5787ba20..eb73ed94f 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023/11/27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ import json from pathlib import Path @@ -23,7 +24,6 @@ from metagpt.const import ( ) from metagpt.logs import logger from metagpt.schema import Document, Documents -from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -43,7 +43,7 @@ Requirement: Fill in the following missing information based on the context, eac ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## Project name: Constant text. ## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -58,15 +58,15 @@ and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Implementation approach": "We will ...", - "project_name": "snake_game", + "Project name": "{project_name}", "File list": ["main.py"], "Data structures and interfaces": ' classDiagram - class Game{ + class Game{{ +int score - } + }} ... Game "1" -- "1" Food: has ', @@ -77,7 +77,7 @@ and only output the json inside this tag, nothing else G->>M: end game ', "Anything UNCLEAR": "The requirement is clear to me." -} +}} [/CONTENT] """, }, @@ -96,7 +96,7 @@ ATTENTION: Output carefully referenced "Format example" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## Project name: Constant text. ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -112,9 +112,9 @@ ATTENTION: Output carefully referenced "Format example" in format. ## Implementation approach We will ... -## project_name +## Project name ```python -"snake_game" +"{project_name}" ``` ## File list @@ -151,7 +151,7 @@ The requirement is clear to me. OUTPUT_MAPPING = { "Implementation approach": (str, ...), - "project_name": (str, ...), + "Project name": (str, ...), "File list": (List[str], ...), "Data structures and interfaces": (str, ...), "Program call flow": (str, ...), @@ -173,7 +173,7 @@ ATTENTION: Output carefully referenced "Old Design" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## Project name: Constant text "{project_name}". ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -229,50 +229,21 @@ class WriteDesign(Action): async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) + format_example = format_example.format(project_name=CONFIG.project_name) prompt = prompt_template.format(context=context, format_example=format_example) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - self._rename_project_name(system_design=system_design) - await self._rename_workspace(system_design) return system_design async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): - prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content) + prompt = MERGE_PROMPT.format( + old_design=system_design_doc.content, context=prd_doc.content, project_name=CONFIG.project_name + ) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python # package name" contain space, have to use setattr - self._rename_project_name(system_design=system_design) system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) return system_design_doc - @staticmethod - def _rename_project_name(system_design): - # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" - # contain space, have to use setattr - if CONFIG.project_name: - setattr( - system_design.instruct_content, - "project_name", - CONFIG.project_name, - ) - return - setattr( - system_design.instruct_content, - "project_name", - system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), - ) - - @staticmethod - async def _rename_workspace(system_design): - if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to - # Section 2.2.3.10 of RFC 135 - return - - if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["project_name"] - else: - ws_name = CodeParser.parse_str(block="project_name", text=system_design) - CONFIG.git_repo.rename_root(ws_name) - async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: prd = await prds_file_repo.get(filename) old_system_design_doc = await system_design_file_repo.get(filename) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index b751dc970..4a2082a07 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -3,7 +3,7 @@ """ @Time : 2023/11/20 @Author : mashenquan -@File : git_repository.py +@File : prepare_documents.py @Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. RFC 135 2.2.3.5.1. """ diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 3d59daeed..95da0d65a 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -183,6 +183,10 @@ MERGE_PROMPT = """ ## Old Tasks {old_tasks} ----- + +## Format example +{format_example} +----- Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules. Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. @@ -201,7 +205,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format, +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Format example" format, and only output the json inside this tag, nothing else """ @@ -264,7 +268,9 @@ class WriteTasks(Action): return rsp async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content) + _, format_example = get_template(templates, format) + prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content, + format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) task_doc.content = rsp.instruct_content.json(ensure_ascii=False) return task_doc diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py index 88a37536b..d10cd6c55 100644 --- a/metagpt/actions/summarize_code.py +++ b/metagpt/actions/summarize_code.py @@ -3,12 +3,15 @@ """ @Author : alexanderwu @File : summarize_code.py +@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode. """ +from pathlib import Path from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action from metagpt.config import CONFIG +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO from metagpt.logs import logger from metagpt.utils.file_repository import FileRepository @@ -95,8 +98,10 @@ class SummarizeCode(Action): return code_rsp async def run(self): - design_doc = await FileRepository.get_file(self.context.design_filename) - task_doc = await FileRepository.get_file(self.context.task_filename) + design_pathname = Path(self.context.design_filename) + design_doc = await FileRepository.get_file(filename=design_pathname.name, relative_path=SYSTEM_DESIGN_FILE_REPO) + task_pathname = Path(self.context.task_filename) + task_doc = await FileRepository.get_file(filename=task_pathname.name, relative_path=TASK_FILE_REPO) src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) code_blocks = [] for filename in self.context.codes_filenames: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 59ccb49a5..9b20843c7 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -15,13 +15,13 @@ RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. """ - from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action -from metagpt.const import TEST_OUTPUTS_FILE_REPO +from metagpt.config import CONFIG +from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger -from metagpt.schema import CodingContext, RunCodeResult +from metagpt.schema import CodingContext, Document, RunCodeResult from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository @@ -50,6 +50,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc # Debug logs ```text {logs} + +{summary_log} ``` ----- @@ -90,18 +92,26 @@ class WriteCode(Action): test_doc = await FileRepository.get_file( filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO ) + summary_doc = None + if coding_context.design_doc.filename: + summary_doc = await FileRepository.get_file( + filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO + ) logs = "" if test_doc: test_detail = RunCodeResult.loads(test_doc.content) logs = test_detail.stderr prompt = PROMPT_TEMPLATE.format( design=coding_context.design_doc.content, - tasks=coding_context.task_doc.content, - code=coding_context.code_doc.content, + tasks=coding_context.task_doc.content if coding_context.task_doc else "", + code=coding_context.code_doc.content if coding_context.code_doc else "", logs=logs, filename=self.context.filename, + summary_log=summary_doc.content if summary_doc else "", ) logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) + if not coding_context.code_doc: + coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace) coding_context.code_doc.content = code return coding_context diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 364f6af57..f7c6845d2 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -108,10 +108,11 @@ class WriteCodeReview(Action): k = CONFIG.code_review_k_times or 1 for i in range(k): format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) + task_content = self.context.task_doc.content if self.context.task_doc else "" context = "\n----------\n".join( [ "```text\n" + self.context.design_doc.content + "```\n", - "```text\n" + self.context.task_doc.content + "```\n", + "```text\n" + task_content + "```\n", "```python\n" + self.context.code_doc.content + "```\n", ] ) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 3967a0578..530a22def 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -8,6 +8,7 @@ 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. 3. Move the document storage operations related to WritePRD from the save operation of WriteDesign. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ from __future__ import annotations @@ -27,6 +28,7 @@ from metagpt.const import ( ) from metagpt.logs import logger from metagpt.schema import Document, Documents +from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -53,7 +55,7 @@ ATTENTION: Output carefully referenced "Format example" in format. {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -85,9 +87,10 @@ and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Language": "", "Original Requirements": "", + "Project Name": "{project_name}", "Search Information": "", "Requirements": "", "Product Goals": [], @@ -111,7 +114,7 @@ and only output the json inside this tag, nothing else "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], "UI Design draft": "", "Anything UNCLEAR": "", -} +}} [/CONTENT] """, }, @@ -228,6 +231,7 @@ There are no unclear points. OUTPUT_MAPPING = { "Language": (str, ...), "Original Requirements": (str, ...), + "Project Name": (str, ...), "Product Goals": (List[str], ...), "User Stories": (List[str], ...), "Competitive Analysis": (List[str], ...), @@ -270,7 +274,7 @@ ATTENTION: Output carefully referenced "Old PRD" in format. {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -320,6 +324,7 @@ class WritePRD(Action): if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"REWRITE PRD:{prd_doc.filename}") # If there is no existing PRD, generate one using 'docs/requirement.txt'. if not change_files.docs: prd_doc = await self._update_prd( @@ -327,6 +332,7 @@ class WritePRD(Action): ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"NEW PRD:{prd_doc.filename}") # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the # 'publish' message to transition the workflow to the next stage. This design allows room for global # optimization in subsequent steps. @@ -343,32 +349,36 @@ class WritePRD(Action): # logger.info(format) prompt_template, format_example = get_template(templates, format) + project_name = CONFIG.project_name if CONFIG.project_name else "" + format_example = format_example.format(project_name=project_name) # logger.info(prompt_template) # logger.info(format_example) prompt = prompt_template.format( - requirements=requirements, search_information=info, format_example=format_example + requirements=requirements, search_information=info, format_example=format_example, project_name=project_name ) # logger.info(prompt) # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) + await self._rename_workspace(prd) return prd async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: - m = json.loads(old_prd_doc.content) - if m.get("Original Requirements") == new_requirement_doc.content: - # There have been no changes in the requirements, so they are considered unrelated. - return False prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) res = await self._aask(prompt=prompt) - logger.info(f"[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") + logger.info(f"REQ-RELATIVE:[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") if "YES" in res: return True return False async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + prompt = MERGE_PROMPT.format( + requirements=new_requirement_doc.content, old_prd=prd_doc.content, project_name=CONFIG.project_name + ) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) prd_doc.content = prd.instruct_content.json(ensure_ascii=False) + await self._rename_workspace(prd) return prd_doc async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: @@ -404,3 +414,19 @@ class WritePRD(Action): @staticmethod async def _save_pdf(prd_doc): await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO) + + @staticmethod + async def _rename_workspace(prd): + if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to + # Section 2.2.3.10 of RFC 135 + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + return + + if not CONFIG.project_name: + if isinstance(prd, ActionOutput): + ws_name = prd.instruct_content.dict()["Project Name"] + else: + ws_name = CodeParser.parse_str(block="Project Name", text=prd) + CONFIG.project_name = ws_name + CONFIG.git_repo.rename_root(CONFIG.project_name) diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 7cbb42e1d..65673807f 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -9,8 +9,9 @@ """ from metagpt.actions.action import Action from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO from metagpt.logs import logger -from metagpt.schema import TestingContext +from metagpt.schema import Document, TestingContext from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -52,6 +53,10 @@ class WriteTest(Action): return code async def run(self, *args, **kwargs) -> TestingContext: + if not self.context.test_doc: + self.context.test_doc = Document( + filename="test_" + self.context.code_doc.filename, root_path=TEST_CODES_FILE_REPO + ) prompt = PROMPT_TEMPLATE.format( code_to_test=self.context.code_doc.content, test_file_name=self.context.test_doc.filename, diff --git a/metagpt/const.py b/metagpt/const.py index a646cea7a..bd735a5e1 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for common properties in the Message. @Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. +@Modified By: mashenquan, 2023/12/5. Add directories for code summarization.. """ import contextvars import os @@ -87,5 +88,7 @@ PRD_PDF_FILE_REPO = "resources/prd" TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" TEST_CODES_FILE_REPO = "tests" TEST_OUTPUTS_FILE_REPO = "test_outputs" +CODE_SUMMARIES_FILE_REPO = "docs/code_summaries" +CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries" YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 565ae94f7..6c1dc8338 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -38,7 +38,7 @@ class BaseGPTAPI(BaseChatbot): rsp = self.completion(message) return self.get_choice_text(rsp) - async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: + async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, stream=True) -> str: if system_msgs: message = ( self._system_msgs(system_msgs) + [self._user_msg(msg)] @@ -49,7 +49,7 @@ class BaseGPTAPI(BaseChatbot): message = ( [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] ) - rsp = await self.acompletion_text(message, stream=True) + rsp = await self.acompletion_text(message, stream=stream) logger.debug(message) # logger.debug(rsp) return rsp diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index d42835a1b..9f8eb6482 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -13,17 +13,25 @@ @Modified By: mashenquan, 2023-11-27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.5 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from __future__ import annotations import json +from collections import defaultdict from pathlib import Path from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG -from metagpt.const import MESSAGE_ROUTE_TO_NONE, SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.const import ( + CODE_SUMMARIES_FILE_REPO, + CODE_SUMMARIES_PDF_FILE_REPO, + SYSTEM_DESIGN_FILE_REPO, + TASK_FILE_REPO, +) from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import ( @@ -33,6 +41,16 @@ from metagpt.schema import ( Documents, Message, ) +from metagpt.utils.common import any_to_str, any_to_str_set + +IS_PASS_PROMPT = """ +{context} + +---- +Does the above log indicate anything that needs to be done? +If there are any tasks to be completed, please answer 'NO' along with the to-do list in JSON format; +otherwise, answer 'YES' in JSON format. +""" class Engineer(Role): @@ -60,7 +78,7 @@ class Engineer(Role): """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self.use_code_review = use_code_review - self._watch([WriteTasks]) + self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview]) self.code_todos = [] self.summarize_todos = [] self.n_borg = n_borg @@ -105,39 +123,88 @@ class Engineer(Role): if self._rc.todo is None: return None if isinstance(self._rc.todo, WriteCode): - changed_files = await self._act_sp_with_cr(review=self.use_code_review) - # Unit tests only. - if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files: - changed_files.add(CONFIG.REQA_FILENAME) - return Message( - content="\n".join(changed_files), - role=self.profile, - cause_by=WriteCodeReview if self.use_code_review else WriteCode, - send_to="Edward", # The name of QaEngineer - ) + return await self._act_write_code() if isinstance(self._rc.todo, SummarizeCode): - summaries = [] - for todo in self.summarize_todos: - summary = await todo.run() - summaries.append(summary.json(ensure_ascii=False)) + return await self._act_summarize() + return None + + async def _act_write_code(self): + changed_files = await self._act_sp_with_cr(review=self.use_code_review) + return Message( + content="\n".join(changed_files), + role=self.profile, + cause_by=WriteCodeReview if self.use_code_review else WriteCode, + send_to=self, + sent_from=self, + ) + + async def _act_summarize(self): + code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO) + code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO) + tasks = [] + src_relative_path = CONFIG.src_workspace.relative_to(CONFIG.git_repo.workdir) + for todo in self.summarize_todos: + summary = await todo.run() + summary_filename = Path(todo.context.design_filename).with_suffix(".md").name + dependencies = {todo.context.design_filename, todo.context.task_filename} + for filename in todo.context.codes_filenames: + rpath = src_relative_path / filename + dependencies.add(str(rpath)) + await code_summaries_pdf_file_repo.save( + filename=summary_filename, content=summary, dependencies=dependencies + ) + is_pass, reason = await self._is_pass(summary) + if not is_pass: + todo.context.reason = reason + tasks.append(todo.context.dict()) + await code_summaries_file_repo.save( + filename=Path(todo.context.design_filename).name, + content=todo.context.json(), + dependencies=dependencies, + ) + else: + await code_summaries_file_repo.delete(filename=Path(todo.context.design_filename).name) + + logger.info(f"--max-auto-summarize-code={CONFIG.max_auto_summarize_code}") + if not tasks or CONFIG.max_auto_summarize_code == 0: return Message( - content="\n".join(summaries), + content="", role=self.profile, cause_by=SummarizeCode, - send_to=MESSAGE_ROUTE_TO_NONE, + sent_from=self, + send_to="Edward", # The name of QaEngineer ) - return None + # The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. + # This parameter is used for debugging the workflow. + CONFIG.max_auto_summarize_code -= 1 if CONFIG.max_auto_summarize_code > 0 else 0 + return Message( + content=json.dumps(tasks), role=self.profile, cause_by=SummarizeCode, send_to=self, sent_from=self + ) + + async def _is_pass(self, summary) -> (str, str): + rsp = await self._llm.aask(msg=IS_PASS_PROMPT.format(context=summary), stream=False) + logger.info(rsp) + if "YES" in rsp: + return True, rsp + return False, rsp async def _think(self) -> Action | None: if not CONFIG.src_workspace: CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name - if not self.code_todos: - await self._new_code_actions() - elif not self.summarize_todos: - await self._new_summarize_actions() - else: + write_code_filters = any_to_str_set([WriteTasks, SummarizeCode]) + summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) + if not self._rc.news: return None - return self._rc.todo # For agent store + msg = self._rc.news[0] + if msg.cause_by in write_code_filters: + logger.info(f"TODO WriteCode:{msg.json()}") + await self._new_code_actions() + return self._rc.todo + if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): + logger.info(f"TODO SummarizeCode:{msg.json()}") + await self._new_summarize_actions() + return self._rc.todo + return None @staticmethod async def _new_coding_context( @@ -151,9 +218,9 @@ class Engineer(Role): design_doc = None for i in dependencies: if str(i.parent) == TASK_FILE_REPO: - task_doc = task_file_repo.get(i.filename) + task_doc = await task_file_repo.get(i.name) elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO: - design_doc = design_file_repo.get(i.filename) + design_doc = await design_file_repo.get(i.name) context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc) return context @@ -216,16 +283,13 @@ class Engineer(Role): async def _new_summarize_actions(self): src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - changed_src_files = src_file_repo.changed_files + src_files = src_file_repo.all_files # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). - summarizations = {} - for filename in changed_src_files: - dependencies = src_file_repo.get_dependency(filename=filename) + summarizations = defaultdict(list) + for filename in src_files: + dependencies = await src_file_repo.get_dependency(filename=filename) ctx = CodeSummarizeContext.loads(filenames=dependencies) - if ctx not in summarizations: - summarizations[ctx] = set() - srcs = summarizations.get(ctx) - srcs.add(filename) + summarizations[ctx].append(filename) for ctx, filenames in summarizations.items(): ctx.codes_filenames = filenames self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm)) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 41a3213dc..15a01b9e9 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -11,10 +11,13 @@ WriteTest/RunCode/DebugError object, rather than passing them in when calling the run function. 2. According to Section 2.2.3.5.7 of RFC 135, change the method of transferring files from using the Message to using file references. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest # from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( MESSAGE_ROUTE_TO_NONE, @@ -40,13 +43,16 @@ class QaEngineer(Role): self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates - self._watch([WriteCode, WriteCodeReview, WriteTest, RunCode, DebugError]) + self._watch([SummarizeCode, WriteTest, RunCode, DebugError]) self.test_round = 0 self.test_round_allowed = test_round_allowed async def _write_test(self, message: Message) -> None: - changed_files = message.content.splitlines() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + changed_files = set(src_file_repo.changed_files.keys()) + # Unit tests only. + if CONFIG.reqa_file and CONFIG.reqa_file not in changed_files: + changed_files.add(CONFIG.reqa_file) tests_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) for filename in changed_files: # write tests @@ -146,7 +152,7 @@ class QaEngineer(Role): ) return result_msg - code_filters = any_to_str_set({WriteCode, WriteCodeReview}) + code_filters = any_to_str_set({SummarizeCode}) test_filters = any_to_str_set({WriteTest, DebugError}) run_filters = any_to_str_set({RunCode}) for msg in self._rc.news: diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1e99cc1ff..2651be7eb 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -284,9 +284,10 @@ class Role: instruct_content=response.instruct_content, role=self.profile, cause_by=self._rc.todo, + sent_from=self, ) else: - msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) + msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self) self._rc.memory.add(msg) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index d1174799a..a8c1b7726 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -48,9 +48,9 @@ class Document(BaseModel): Represents a document. """ - root_path: str - filename: str - content: Optional[str] = None + root_path: str = "" + filename: str = "" + content: str = "" def get_meta(self) -> Document: """Get metadata of the document. @@ -260,8 +260,8 @@ class MessageQueue: class CodingContext(BaseModel): filename: str design_doc: Document - task_doc: Document - code_doc: Document + task_doc: Optional[Document] + code_doc: Optional[Document] @staticmethod def loads(val: str) -> CodingContext | None: @@ -275,7 +275,7 @@ class CodingContext(BaseModel): class TestingContext(BaseModel): filename: str code_doc: Document - test_doc: Document + test_doc: Optional[Document] @staticmethod def loads(val: str) -> TestingContext | None: @@ -324,10 +324,11 @@ class RunCodeResult(BaseModel): class CodeSummarizeContext(BaseModel): design_filename: str = "" task_filename: str = "" - codes_filenames: Set[str] = Field(default_factory=set) + codes_filenames: List[str] = Field(default_factory=list) + reason: str = "" @staticmethod - def loads(filenames: Set) -> CodeSummarizeContext: + def loads(filenames: List) -> CodeSummarizeContext: ctx = CodeSummarizeContext() for filename in filenames: if Path(filename).is_relative_to(SYSTEM_DESIGN_FILE_REPO): @@ -337,3 +338,6 @@ class CodeSummarizeContext(BaseModel): ctx.task_filename = str(filename) continue return ctx + + def __hash__(self): + return hash((self.design_filename, self.task_filename)) diff --git a/metagpt/startup.py b/metagpt/startup.py index 78f32d556..43c946040 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -24,6 +24,10 @@ def startup( help="Specify the directory path of the old version project to fulfill the " "incremental requirements.", ), reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."), + max_auto_summarize_code: int = typer.Option( + default=-1, + help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the workflow.", + ), ): """Run a startup. Be a boss.""" from metagpt.roles import ( @@ -40,6 +44,7 @@ def startup( CONFIG.inc = inc CONFIG.project_path = project_path CONFIG.reqa_file = reqa_file + CONFIG.max_auto_summarize_code = max_auto_summarize_code company = Team() company.hire( diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py index 653e07ef9..e8347d567 100644 --- a/metagpt/utils/dependency_file.py +++ b/metagpt/utils/dependency_file.py @@ -14,6 +14,7 @@ from typing import Set import aiofiles +from metagpt.config import CONFIG from metagpt.logs import logger @@ -81,7 +82,7 @@ class DependencyFile: if persist: await self.save() - async def get(self, filename: Path | str, persist=False): + async def get(self, filename: Path | str, persist=True): """Get dependencies for a file asynchronously. :param filename: The filename or path. @@ -91,7 +92,7 @@ class DependencyFile: if persist: await self.load() - root = self._filename.parent + root = CONFIG.git_repo.workdir try: key = Path(filename).relative_to(root) except ValueError: diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 0815bf90a..2cace7232 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -151,6 +151,17 @@ class FileRepository: relative_files[str(rf)] = ct return relative_files + @property + def all_files(self) -> List: + """Get a dictionary of all files in the repository. + + The dictionary includes file paths relative to the current FileRepository. + + :return: A dictionary where keys are file paths and values are file information. + :rtype: List + """ + return self._git_repo.get_files(relative_path=self._relative_path) + def get_change_dir_files(self, dir: Path | str) -> List: """Get the files in a directory that have changed. @@ -259,3 +270,25 @@ class FileRepository: """ file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) return await file_repo.save_doc(doc=doc, with_suffix=with_suffix, dependencies=dependencies) + + async def delete(self, filename: Path | str): + """Delete a file from the file repository. + + This method deletes a file from the file repository based on the provided filename. + + :param filename: The name or path of the file to be deleted. + :type filename: Path or str + """ + pathname = self.workdir / filename + if not pathname.exists(): + return + pathname.unlink(missing_ok=True) + + dependency_file = await self._git_repo.get_dependency() + await dependency_file.update(filename=pathname, dependencies=None) + logger.info(f"remove dependency key: {str(pathname)}") + + @staticmethod + async def delete_file(filename: Path | str, relative_path: Path | str = "."): + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + await file_repo.delete(filename=filename) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 7c9ec645f..d58f68109 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -11,7 +11,7 @@ from __future__ import annotations import shutil from enum import Enum from pathlib import Path -from typing import Dict +from typing import Dict, List from git.repo import Repo from git.repo.fun import is_git_dir @@ -200,6 +200,39 @@ class GitRepository: logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) + def get_files(self, relative_path: Path | str, root_relative_path: Path | str = None) -> List: + """Retrieve a list of files in the specified relative path. + + The method returns a list of file paths relative to the current FileRepository. + + :param relative_path: The relative path within the repository. + :type relative_path: Path or str + :param root_relative_path: The root relative path within the repository. + :type root_relative_path: Path or str + :return: A list of file paths in the specified directory. + :rtype: List[str] + """ + try: + relative_path = Path(relative_path).relative_to(self.workdir) + except ValueError: + relative_path = Path(relative_path) + + if not root_relative_path: + root_relative_path = Path(self.workdir) / relative_path + files = [] + try: + directory_path = Path(self.workdir) / relative_path + for file_path in directory_path.iterdir(): + if file_path.is_file(): + rpath = file_path.relative_to(root_relative_path) + files.append(str(rpath)) + else: + subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path) + files.extend(subfolder_files) + except Exception as e: + logger.error(f"Error: {e}") + return files + if __name__ == "__main__": path = DEFAULT_WORKSPACE_ROOT / "git" diff --git a/tests/conftest.py b/tests/conftest.py index d2ac8304f..8e4422700 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,8 +12,11 @@ from unittest.mock import Mock import pytest +from metagpt.config import CONFIG +from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI +from metagpt.utils.git_repository import GitRepository class Context: @@ -68,3 +71,16 @@ def proxy(): server = asyncio.get_event_loop().run_until_complete(asyncio.start_server(handle_client, "127.0.0.1", 0)) return "http://{}:{}".format(*server.sockets[0].getsockname()) + + +# init & dispose git repo +@pytest.fixture(scope="session", autouse=True) +def setup_and_teardown_git_repo(request): + CONFIG.git_repo = GitRepository(local_path=DEFAULT_WORKSPACE_ROOT / "unittest") + + # Destroy git repo at the end of the test session. + def fin(): + CONFIG.git_repo.delete_repository() + + # Register the function for destroying the environment. + request.addfinalizer(fin) diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index c48913755..f6602a82b 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -90,7 +90,7 @@ Python's in-built data structures like lists and dictionaries will be used exten For testing, we can use the PyTest framework. This is a mature full-featured Python testing tool that helps you write better programs. -## project_name: +## Project Name: ```python "adventure_game" ``` diff --git a/tests/metagpt/actions/test_debug_error.py b/tests/metagpt/actions/test_debug_error.py index 2393d2cc9..8289fe41b 100644 --- a/tests/metagpt/actions/test_debug_error.py +++ b/tests/metagpt/actions/test_debug_error.py @@ -4,17 +4,19 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : test_debug_error.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ +import uuid + import pytest from metagpt.actions.debug_error import DebugError +from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO +from metagpt.schema import RunCodeContext, RunCodeResult +from metagpt.utils.file_repository import FileRepository -EXAMPLE_MSG_CONTENT = ''' ---- -## Development Code File Name -player.py -## Development Code -```python +CODE_CONTENT = ''' from typing import List from deck import Deck from card import Card @@ -58,12 +60,9 @@ class Player: if self.score > 21 and any(card.rank == 'A' for card in self.hand): self.score -= 10 return self.score +''' -``` -## Test File Name -test_player.py -## Test Code -```python +TEST_CONTENT = """ import unittest from blackjack_game.player import Player from blackjack_game.deck import Deck @@ -114,42 +113,41 @@ class TestPlayer(unittest.TestCase): if __name__ == '__main__': unittest.main() -``` -## Running Command -python tests/test_player.py -## Running Output -standard output: ; -standard errors: ..F.. -====================================================================== -FAIL: test_player_calculate_score_with_multiple_aces (__main__.TestPlayer) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "tests/test_player.py", line 46, in test_player_calculate_score_with_multiple_aces - self.assertEqual(player.score, 12) -AssertionError: 22 != 12 - ----------------------------------------------------------------------- -Ran 5 tests in 0.007s - -FAILED (failures=1) -; -## instruction: -The error is in the development code, specifically in the calculate_score method of the Player class. The method is not correctly handling the case where there are multiple Aces in the player's hand. The current implementation only subtracts 10 from the score once if the score is over 21 and there's an Ace in the hand. However, in the case of multiple Aces, it should subtract 10 for each Ace until the score is 21 or less. -## File To Rewrite: -player.py -## Status: -FAIL -## Send To: -Engineer ---- -''' +""" @pytest.mark.asyncio async def test_debug_error(): - debug_error = DebugError("debug_error") + CONFIG.src_workspace = CONFIG.git_repo.workdir / uuid.uuid4().hex + ctx = RunCodeContext( + code_filename="player.py", + test_filename="test_player.py", + command=["python", "tests/test_player.py"], + output_filename="output.log", + ) - file_name, rewritten_code = await debug_error.run(context=EXAMPLE_MSG_CONTENT) + await FileRepository.save_file(filename=ctx.code_filename, content=CODE_CONTENT, relative_path=CONFIG.src_workspace) + await FileRepository.save_file(filename=ctx.test_filename, content=TEST_CONTENT, relative_path=TEST_CODES_FILE_REPO) + output_data = RunCodeResult( + stdout=";", + stderr="", + summary="======================================================================\n" + "FAIL: test_player_calculate_score_with_multiple_aces (__main__.TestPlayer)\n" + "----------------------------------------------------------------------\n" + "Traceback (most recent call last):\n" + ' File "tests/test_player.py", line 46, in test_player_calculate_score_' + "with_multiple_aces\n" + " self.assertEqual(player.score, 12)\nAssertionError: 22 != 12\n\n" + "----------------------------------------------------------------------\n" + "Ran 5 tests in 0.007s\n\nFAILED (failures=1)\n;\n", + ) + await FileRepository.save_file( + filename=ctx.output_filename, content=output_data.json(), relative_path=TEST_OUTPUTS_FILE_REPO + ) + debug_error = DebugError(context=ctx) - assert "class Player" in rewritten_code # rewrite the same class - assert "while self.score > 21" in rewritten_code # a key logic to rewrite to (original one is "if self.score > 12") + rsp = await debug_error.run() + + assert "class Player" in rsp # rewrite the same class + # a key logic to rewrite to (original one is "if self.score > 12") + assert "while self.score > 21" in rsp diff --git a/tests/metagpt/actions/test_design_api.py b/tests/metagpt/actions/test_design_api.py index 0add8fb74..e90707d1a 100644 --- a/tests/metagpt/actions/test_design_api.py +++ b/tests/metagpt/actions/test_design_api.py @@ -4,33 +4,27 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : test_design_api.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.design_api import WriteDesign +from metagpt.const import PRDS_FILE_REPO from metagpt.logs import logger from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository from tests.metagpt.actions.mock import PRD_SAMPLE @pytest.mark.asyncio async def test_design_api(): - prd = "我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。" + inputs = ["我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。", PRD_SAMPLE] + for prd in inputs: + await FileRepository.save_file("new_prd.txt", content=prd, relative_path=PRDS_FILE_REPO) - design_api = WriteDesign("design_api") + design_api = WriteDesign("design_api") - result = await design_api.run([Message(content=prd, instruct_content=None)]) - logger.info(result) + result = await design_api.run([Message(content=prd, instruct_content=None)]) + logger.info(result) - assert result - - -@pytest.mark.asyncio -async def test_design_api_calculator(): - prd = PRD_SAMPLE - - design_api = WriteDesign("design_api") - result = await design_api.run([Message(content=prd, instruct_content=None)]) - logger.info(result) - - assert result + assert result diff --git a/tests/metagpt/actions/test_prepare_documents.py b/tests/metagpt/actions/test_prepare_documents.py new file mode 100644 index 000000000..31c8bcb80 --- /dev/null +++ b/tests/metagpt/actions/test_prepare_documents.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/6 +@Author : mashenquan +@File : test_prepare_documents.py +@Desc: Unit test for prepare_documents.py +""" +import pytest + +from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME +from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository + + +@pytest.mark.asyncio +async def test_prepare_documents(): + msg = Message(content="New user requirements balabala...") + + if CONFIG.git_repo: + CONFIG.git_repo.delete_repository() + CONFIG.git_repo = None + + await PrepareDocuments().run(with_messages=[msg]) + assert CONFIG.git_repo + doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) + assert doc + assert doc.content == msg.content diff --git a/tests/metagpt/actions/test_run_code.py b/tests/metagpt/actions/test_run_code.py index 1e451cb14..888418974 100644 --- a/tests/metagpt/actions/test_run_code.py +++ b/tests/metagpt/actions/test_run_code.py @@ -4,10 +4,12 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : test_run_code.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.run_code import RunCode +from metagpt.schema import RunCodeContext @pytest.mark.asyncio @@ -35,37 +37,29 @@ async def test_run_script(): @pytest.mark.asyncio async def test_run(): - action = RunCode() - result = await action.run(mode="text", code="print('Hello, World')") - assert "PASS" in result - - result = await action.run( - mode="script", - code="echo 'Hello World'", - code_file_name="", - test_code="", - test_file_name="", - command=["echo", "Hello World"], - working_directory=".", - additional_python_paths=[], - ) - assert "PASS" in result - - -@pytest.mark.asyncio -async def test_run_failure(): - action = RunCode() - result = await action.run(mode="text", code="result = 1 / 0") - assert "FAIL" in result - - result = await action.run( - mode="script", - code='python -c "print(1/0)"', - code_file_name="", - test_code="", - test_file_name="", - command=["python", "-c", "print(1/0)"], - working_directory=".", - additional_python_paths=[], - ) - assert "FAIL" in result + inputs = [ + (RunCodeContext(mode="text", code_filename="a.txt", code="print('Hello, World')"), "PASS"), + ( + RunCodeContext( + mode="script", + code_filename="a.sh", + code="echo 'Hello World'", + command=["echo", "Hello World"], + working_directory=".", + ), + "PASS", + ), + ( + RunCodeContext( + mode="script", + code_filename="a.py", + code='python -c "print(1/0)"', + command=["python", "-c", "print(1/0)"], + working_directory=".", + ), + "FAIL", + ), + ] + for ctx, result in inputs: + rsp = await RunCode(context=ctx).run() + assert result in rsp.summary diff --git a/tests/metagpt/actions/test_summarize_code.py b/tests/metagpt/actions/test_summarize_code.py new file mode 100644 index 000000000..7ecb67afd --- /dev/null +++ b/tests/metagpt/actions/test_summarize_code.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/11 17:46 +@Author : mashenquan +@File : test_summarize_code.py +@Modifiled By: mashenquan, 2023-12-6. Unit test for summarize_code.py +""" +import pytest + +from metagpt.actions.summarize_code import SummarizeCode +from metagpt.config import CONFIG +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.logs import logger +from metagpt.schema import CodeSummarizeContext +from metagpt.utils.file_repository import FileRepository + +DESIGN_CONTENT = """ +{"Implementation approach": "To develop this snake game, we will use the Python language and choose the Pygame library. Pygame is an open-source Python module collection specifically designed for writing video games. It provides functionalities such as displaying images and playing sounds, making it suitable for creating intuitive and responsive user interfaces. We will ensure efficient game logic to prevent any delays during gameplay. The scoring system will be simple, with the snake gaining points for each food it eats. We will use Pygame's event handling system to implement pause and resume functionality, as well as high-score tracking. The difficulty will increase by speeding up the snake's movement. In the initial version, we will focus on single-player mode and consider adding multiplayer mode and customizable skins in future updates. Based on the new requirement, we will also add a moving obstacle that appears randomly. If the snake eats this obstacle, the game will end. If the snake does not eat the obstacle, it will disappear after 5 seconds. For this, we need to add mechanisms for obstacle generation, movement, and disappearance in the game logic.", "Project_name": "snake_game", "File list": ["main.py", "game.py", "snake.py", "food.py", "obstacle.py", "scoreboard.py", "constants.py", "assets/styles.css", "assets/index.html"], "Data structures and interfaces": "```mermaid\n classDiagram\n class Game{\n +int score\n +int speed\n +bool game_over\n +bool paused\n +Snake snake\n +Food food\n +Obstacle obstacle\n +Scoreboard scoreboard\n +start_game() void\n +pause_game() void\n +resume_game() void\n +end_game() void\n +increase_difficulty() void\n +update() void\n +render() void\n Game()\n }\n class Snake{\n +list body_parts\n +str direction\n +bool grow\n +move() void\n +grow() void\n +check_collision() bool\n Snake()\n }\n class Food{\n +tuple position\n +spawn() void\n Food()\n }\n class Obstacle{\n +tuple position\n +int lifetime\n +bool active\n +spawn() void\n +move() void\n +check_collision() bool\n +disappear() void\n Obstacle()\n }\n class Scoreboard{\n +int high_score\n +update_score(int) void\n +reset_score() void\n +load_high_score() void\n +save_high_score() void\n Scoreboard()\n }\n class Constants{\n }\n Game \"1\" -- \"1\" Snake: has\n Game \"1\" -- \"1\" Food: has\n Game \"1\" -- \"1\" Obstacle: has\n Game \"1\" -- \"1\" Scoreboard: has\n ```", "Program call flow": "```sequenceDiagram\n participant M as Main\n participant G as Game\n participant S as Snake\n participant F as Food\n participant O as Obstacle\n participant SB as Scoreboard\n M->>G: start_game()\n loop game loop\n G->>S: move()\n G->>S: check_collision()\n G->>F: spawn()\n G->>O: spawn()\n G->>O: move()\n G->>O: check_collision()\n G->>O: disappear()\n G->>SB: update_score(score)\n G->>G: update()\n G->>G: render()\n alt if paused\n M->>G: pause_game()\n M->>G: resume_game()\n end\n alt if game_over\n G->>M: end_game()\n end\n end\n```", "Anything UNCLEAR": "There is no need for further clarification as the requirements are already clear."} +""" + +TASK_CONTENT = """ +{"Required Python third-party packages": ["pygame==2.0.1"], "Required Other language third-party packages": ["No third-party packages required for other languages."], "Full API spec": "\n openapi: 3.0.0\n info:\n title: Snake Game API\n version: \"1.0.0\"\n paths:\n /start:\n get:\n summary: Start the game\n responses:\n '200':\n description: Game started successfully\n /pause:\n get:\n summary: Pause the game\n responses:\n '200':\n description: Game paused successfully\n /resume:\n get:\n summary: Resume the game\n responses:\n '200':\n description: Game resumed successfully\n /end:\n get:\n summary: End the game\n responses:\n '200':\n description: Game ended successfully\n /score:\n get:\n summary: Get the current score\n responses:\n '200':\n description: Current score retrieved successfully\n /highscore:\n get:\n summary: Get the high score\n responses:\n '200':\n description: High score retrieved successfully\n components: {}\n ", "Logic Analysis": [["constants.py", "Contains all the constant values like screen size, colors, game speeds, etc. This should be implemented first as it provides the base values for other components."], ["snake.py", "Contains the Snake class with methods for movement, growth, and collision detection. It is dependent on constants.py for configuration values."], ["food.py", "Contains the Food class responsible for spawning food items on the screen. It is dependent on constants.py for configuration values."], ["obstacle.py", "Contains the Obstacle class with methods for spawning, moving, and disappearing of obstacles, as well as collision detection with the snake. It is dependent on constants.py for configuration values."], ["scoreboard.py", "Contains the Scoreboard class for updating, resetting, loading, and saving high scores. It may use constants.py for configuration values and depends on the game's scoring logic."], ["game.py", "Contains the main Game class which includes the game loop and methods for starting, pausing, resuming, and ending the game. It is dependent on snake.py, food.py, obstacle.py, and scoreboard.py."], ["main.py", "The entry point of the game that initializes the game and starts the game loop. It is dependent on game.py."]], "Task list": ["constants.py", "snake.py", "food.py", "obstacle.py", "scoreboard.py", "game.py", "main.py"], "Shared Knowledge": "\n 'constants.py' should contain all the necessary configurations for the game, such as screen dimensions, color definitions, and speed settings. These constants will be used across multiple files, ensuring consistency and ease of updates. Ensure that the Pygame library is initialized correctly in 'main.py' before starting the game loop. Also, make sure that the game's state is managed properly when pausing and resuming the game.\n ", "Anything UNCLEAR": "The interaction between the 'obstacle.py' and the game loop needs to be clearly defined to ensure obstacles appear and disappear correctly. The lifetime of the obstacle and its random movement should be implemented in a way that does not interfere with the game's performance."} +""" + +FOOD_PY = """ +## food.py +import random + +class Food: + def __init__(self): + self.position = (0, 0) + + def generate(self): + x = random.randint(0, 9) + y = random.randint(0, 9) + self.position = (x, y) + + def get_position(self): + return self.position + +""" + +GAME_PY = """ +## game.py +import pygame +from snake import Snake +from food import Food + +class Game: + def __init__(self): + self.score = 0 + self.level = 1 + self.snake = Snake() + self.food = Food() + + def start_game(self): + pygame.init() + self.initialize_game() + self.game_loop() + + def initialize_game(self): + self.score = 0 + self.level = 1 + self.snake.reset() + self.food.generate() + + def game_loop(self): + game_over = False + + while not game_over: + self.update() + self.draw() + self.handle_events() + self.check_collision() + self.increase_score() + self.increase_level() + + if self.snake.is_collision(): + game_over = True + self.game_over() + + def update(self): + self.snake.move() + + def draw(self): + self.snake.draw() + self.food.draw() + + def handle_events(self): + for event in pygame.event.get(): + if event.type == pygame.QUIT: + pygame.quit() + quit() + elif event.type == pygame.KEYDOWN: + if event.key == pygame.K_UP: + self.snake.change_direction("UP") + elif event.key == pygame.K_DOWN: + self.snake.change_direction("DOWN") + elif event.key == pygame.K_LEFT: + self.snake.change_direction("LEFT") + elif event.key == pygame.K_RIGHT: + self.snake.change_direction("RIGHT") + + def check_collision(self): + if self.snake.get_head() == self.food.get_position(): + self.snake.grow() + self.food.generate() + + def increase_score(self): + self.score += 1 + + def increase_level(self): + if self.score % 10 == 0: + self.level += 1 + + def game_over(self): + print("Game Over") + self.initialize_game() + +""" + +MAIN_PY = """ +## main.py +import pygame +from game import Game + +def main(): + pygame.init() + game = Game() + game.start_game() + +if __name__ == "__main__": + main() + +""" + +SNAKE_PY = """ +## snake.py +import pygame + +class Snake: + def __init__(self): + self.body = [(0, 0)] + self.direction = (1, 0) + + def move(self): + head = self.body[0] + dx, dy = self.direction + new_head = (head[0] + dx, head[1] + dy) + self.body.insert(0, new_head) + self.body.pop() + + def change_direction(self, direction): + if direction == "UP": + self.direction = (0, -1) + elif direction == "DOWN": + self.direction = (0, 1) + elif direction == "LEFT": + self.direction = (-1, 0) + elif direction == "RIGHT": + self.direction = (1, 0) + + def grow(self): + tail = self.body[-1] + dx, dy = self.direction + new_tail = (tail[0] - dx, tail[1] - dy) + self.body.append(new_tail) + + def get_head(self): + return self.body[0] + + def get_body(self): + return self.body[1:] + +""" + + +@pytest.mark.asyncio +async def test_summarize_code(): + CONFIG.src_workspace = CONFIG.git_repo.workdir / "src" + await FileRepository.save_file(filename="1.json", relative_path=SYSTEM_DESIGN_FILE_REPO, content=DESIGN_CONTENT) + await FileRepository.save_file(filename="1.json", relative_path=TASK_FILE_REPO, content=TASK_CONTENT) + await FileRepository.save_file(filename="food.py", relative_path=CONFIG.src_workspace, content=FOOD_PY) + await FileRepository.save_file(filename="game.py", relative_path=CONFIG.src_workspace, content=GAME_PY) + await FileRepository.save_file(filename="main.py", relative_path=CONFIG.src_workspace, content=MAIN_PY) + await FileRepository.save_file(filename="snake.py", relative_path=CONFIG.src_workspace, content=SNAKE_PY) + + src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) + all_files = src_file_repo.all_files + ctx = CodeSummarizeContext(design_filename="1.json", task_filename="1.json", codes_filenames=all_files) + action = SummarizeCode(context=ctx) + rsp = await action.run() + assert rsp + logger.info(rsp) diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index eb5e3de91..54229089c 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -4,26 +4,31 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_code.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.write_code import WriteCode from metagpt.llm import LLM from metagpt.logs import logger +from metagpt.schema import CodingContext, Document from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE @pytest.mark.asyncio async def test_write_code(): - api_design = "设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。" - write_code = WriteCode("write_code") + context = CodingContext( + filename="task_filename.py", design_doc=Document(content="设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。") + ) + doc = Document(content=context.json()) + write_code = WriteCode(context=doc) - code = await write_code.run(api_design) - logger.info(code) + code = await write_code.run() + logger.info(code.json()) # 我们不能精确地预测生成的代码，但我们可以检查某些关键字 - assert "def add" in code - assert "return" in code + assert "def add" in code.code_doc.content + assert "return" in code.code_doc.content @pytest.mark.asyncio diff --git a/tests/metagpt/actions/test_write_code_review.py b/tests/metagpt/actions/test_write_code_review.py index 21bc563ec..e16eb7348 100644 --- a/tests/metagpt/actions/test_write_code_review.py +++ b/tests/metagpt/actions/test_write_code_review.py @@ -8,6 +8,8 @@ import pytest from metagpt.actions.write_code_review import WriteCodeReview +from metagpt.document import Document +from metagpt.schema import CodingContext @pytest.mark.asyncio @@ -16,13 +18,15 @@ async def test_write_code_review(capfd): def add(a, b): return a + """ - # write_code_review = WriteCodeReview("write_code_review") + context = CodingContext( + filename="math.py", design_doc=Document(content="编写一个从a加b的函数，返回a+b"), code_doc=Document(content=code) + ) - code = await WriteCodeReview().run(context="编写一个从a加b的函数，返回a+b", code=code, filename="math.py") + context = await WriteCodeReview(context=context).run() # 我们不能精确地预测生成的代码评审，但我们可以检查返回的是否为字符串 - assert isinstance(code, str) - assert len(code) > 0 + assert isinstance(context.code_doc.content, str) + assert len(context.code_doc.content) > 0 captured = capfd.readouterr() print(f"输出内容: {captured.out}") diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 8f8ef84f5..08be3cf75 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -9,19 +9,24 @@ import pytest from metagpt.actions import UserRequirement +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, PRDS_FILE_REPO, REQUIREMENT_FILENAME from metagpt.logs import logger from metagpt.roles.product_manager import ProductManager from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository @pytest.mark.asyncio async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" + await FileRepository.save_file(filename=REQUIREMENT_FILENAME, content=requirements, relative_path=DOCS_FILE_REPO) prd = await product_manager.run(Message(content=requirements, cause_by=UserRequirement)) logger.info(requirements) logger.info(prd) # Assert the prd is not None or empty assert prd is not None - assert prd != "" + assert prd.content != "" + assert CONFIG.git_repo.new_file_repository(relative_path=PRDS_FILE_REPO).changed_files diff --git a/tests/metagpt/actions/test_write_test.py b/tests/metagpt/actions/test_write_test.py index e5acdff44..a3190fb0e 100644 --- a/tests/metagpt/actions/test_write_test.py +++ b/tests/metagpt/actions/test_write_test.py @@ -9,6 +9,7 @@ import pytest from metagpt.actions.write_test import WriteTest from metagpt.logs import logger +from metagpt.schema import Document, TestingContext @pytest.mark.asyncio @@ -24,22 +25,17 @@ async def test_write_test(): def generate(self, max_y: int, max_x: int): self.position = (random.randint(1, max_y - 1), random.randint(1, max_x - 1)) """ + context = TestingContext(filename="food.py", code_doc=Document(filename="food.py", content=code)) + write_test = WriteTest(context=context) - write_test = WriteTest() - - test_code = await write_test.run( - code_to_test=code, - test_file_name="test_food.py", - source_file_path="/some/dummy/path/cli_snake_game/cli_snake_game/food.py", - workspace="/some/dummy/path/cli_snake_game", - ) - logger.info(test_code) + context = await write_test.run() + logger.info(context.json()) # We cannot exactly predict the generated test cases, but we can check if it is a string and if it is not empty - assert isinstance(test_code, str) - assert "from cli_snake_game.food import Food" in test_code - assert "class TestFood(unittest.TestCase)" in test_code - assert "def test_generate" in test_code + assert isinstance(context.test_doc.content, str) + assert "from food import Food" in context.test_doc.content + assert "class TestFood(unittest.TestCase)" in context.test_doc.content + assert "def test_generate" in context.test_doc.content @pytest.mark.asyncio diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 5500b69f7..75f6b3b43 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -71,7 +71,7 @@ PRD = '''## 原始需求 ``` ''' -SYSTEM_DESIGN = """## project_name +SYSTEM_DESIGN = """## Project name ```python "smart_search_engine" ``` diff --git a/tests/metagpt/utils/test_file_repository.py b/tests/metagpt/utils/test_file_repository.py index a830b58aa..92e5204c5 100644 --- a/tests/metagpt/utils/test_file_repository.py +++ b/tests/metagpt/utils/test_file_repository.py @@ -43,6 +43,10 @@ async def test_file_repo(): assert {"a.txt"} == await file_repo.get_changed_dependency("b.txt") await file_repo.save("d/e.txt", "EEE") assert ["d/e.txt"] == file_repo.get_change_dir_files("d") + assert set(file_repo.all_files) == {"a.txt", "b.txt", "d/e.txt"} + await file_repo.delete("d/e.txt") + await file_repo.delete("d/e.txt") # delete twice + assert set(file_repo.all_files) == {"a.txt", "b.txt"} git_repo.delete_repository() From 5394da6d37399a480c39ae6d5ebedac7169efa25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Dec 2023 15:51:12 +0800 Subject: [PATCH 384/398] fixbug: azure call function --- metagpt/actions/design_api.py | 4 ++-- metagpt/actions/prepare_documents.py | 5 ++++- metagpt/provider/openai_api.py | 12 ++++++++---- metagpt/utils/git_repository.py | 5 ++++- requirements.txt | 2 +- tests/metagpt/test_gpt.py | 27 +++++++++++++++++---------- 6 files changed, 36 insertions(+), 19 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index eb73ed94f..557ebcbbd 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -267,10 +267,10 @@ class WriteDesign(Action): @staticmethod async def _save_data_api_design(design_doc): m = json.loads(design_doc.content) - data_api_design = m.get("Data structures and interface definitions") + data_api_design = m.get("Data structures and interfaces") if not data_api_design: return - pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc.filename).with_suffix("") + pathname = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("") await WriteDesign._save_mermaid_file(data_api_design, pathname) logger.info(f"Save class view to {str(pathname)}") diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 4a2082a07..05255dcc5 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -26,7 +26,10 @@ class PrepareDocuments(Action): if not CONFIG.git_repo: # Create and initialize the workspace folder, initialize the Git environment. project_name = CONFIG.project_name or FileRepository.new_filename() - workdir = Path(CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name) + workdir = CONFIG.project_path + if not workdir and CONFIG.workspace: + workdir = Path(CONFIG.workspace) / project_name + workdir = Path(workdir or DEFAULT_WORKSPACE_ROOT / project_name) if not CONFIG.inc and workdir.exists(): shutil.rmtree(workdir) CONFIG.git_repo = GitRepository() diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 2d4b1583a..97bc67069 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -12,6 +12,7 @@ import asyncio import time from typing import NamedTuple, Union +import openai from openai import APIConnectionError, AsyncAzureOpenAI, AsyncOpenAI, RateLimitError from openai.types import CompletionUsage from tenacity import ( @@ -188,7 +189,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): else: kwargs["model"] = self.model kwargs["timeout"] = max(CONFIG.TIMEOUT, timeout) if CONFIG.TIMEOUT is not None else timeout - + return kwargs async def _achat_completion(self, messages: list[dict], timeout=3) -> dict: @@ -312,8 +313,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): >>> rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} """ messages = self._process_message(messages) - rsp = await self._achat_completion_function(messages, **kwargs) - return self.get_choice_function_arguments(rsp) + try: + rsp = await self._achat_completion_function(messages, **kwargs) + return self.get_choice_function_arguments(rsp) + except openai.NotFoundError as e: + logger.error(f"API TYPE:{CONFIG.openai_api_type}, err:{e}") + raise e def _calc_usage(self, messages: list[dict], rsp: str) -> CompletionUsage: if CONFIG.calc_usage: @@ -406,4 +411,3 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return loop else: raise e - diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 9a9ed0fce..5aec4509c 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -197,7 +197,10 @@ class GitRepository: if new_path.exists(): logger.info(f"Delete directory {str(new_path)}") shutil.rmtree(new_path) - os.rename(src=str(self.workdir), dst=str(new_path)) # self.workdir.rename(new_path) + try: + shutil.move(src=str(self.workdir), dst=str(new_path)) + except Exception as e: + logger.warning(f"Move {str(self.workdir)} to {str(new_path)} error: {e}") logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) diff --git a/requirements.txt b/requirements.txt index bcd2db243..de80b0949 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,4 +52,4 @@ websocket-client==1.6.2 aiofiles==23.2.1 gitpython==3.1.40 zhipuai==1.0.7 - +socksio~=1.0.0 diff --git a/tests/metagpt/test_gpt.py b/tests/metagpt/test_gpt.py index 431858d4c..291531122 100644 --- a/tests/metagpt/test_gpt.py +++ b/tests/metagpt/test_gpt.py @@ -5,9 +5,10 @@ @Author : alexanderwu @File : test_gpt.py """ - +import openai import pytest +from metagpt.config import CONFIG from metagpt.logs import logger @@ -18,14 +19,17 @@ class TestGPT: logger.info(answer) assert len(answer) > 0 - # def test_gptapi_ask_batch(self, llm_api): - # answer = llm_api.ask_batch(['请扮演一个Google Python专家工程师，如果理解，回复明白', '写一个hello world']) - # assert len(answer) > 0 + def test_gptapi_ask_batch(self, llm_api): + answer = llm_api.ask_batch(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) + assert len(answer) > 0 def test_llm_api_ask_code(self, llm_api): - answer = llm_api.ask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) - logger.info(answer) - assert len(answer) > 0 + try: + answer = llm_api.ask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) + logger.info(answer) + assert len(answer) > 0 + except openai.NotFoundError: + assert CONFIG.openai_api_type == "azure" @pytest.mark.asyncio async def test_llm_api_aask(self, llm_api): @@ -35,9 +39,12 @@ class TestGPT: @pytest.mark.asyncio async def test_llm_api_aask_code(self, llm_api): - answer = await llm_api.aask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) - logger.info(answer) - assert len(answer) > 0 + try: + answer = await llm_api.aask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) + logger.info(answer) + assert len(answer) > 0 + except openai.NotFoundError: + assert CONFIG.openai_api_type == "azure" @pytest.mark.asyncio async def test_llm_api_costs(self, llm_api): From 376897e7309d3939833755f9ca1db9423bb29d8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 9 Dec 2023 16:26:25 +0800 Subject: [PATCH 385/398] feat: rebase geekan:env_refactor --- tests/metagpt/test_gpt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/metagpt/test_gpt.py b/tests/metagpt/test_gpt.py index 291531122..dda5e6252 100644 --- a/tests/metagpt/test_gpt.py +++ b/tests/metagpt/test_gpt.py @@ -20,7 +20,7 @@ class TestGPT: assert len(answer) > 0 def test_gptapi_ask_batch(self, llm_api): - answer = llm_api.ask_batch(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) + answer = llm_api.ask_batch(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"], timeout=60) assert len(answer) > 0 def test_llm_api_ask_code(self, llm_api): From 379b7b58206f4fe4c2c2a6e8d039e0c28e58cbfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 13 Dec 2023 19:18:38 +0800 Subject: [PATCH 386/398] feat: merge huggingface --- config/config.yaml | 6 ++- metagpt/actions/prepare_documents.py | 4 +- metagpt/config.py | 15 +++++++- metagpt/environment.py | 2 +- metagpt/roles/engineer.py | 9 ++++- metagpt/roles/product_manager.py | 8 ++++ metagpt/roles/role.py | 56 ++++++++++------------------ metagpt/team.py | 7 ++-- metagpt/utils/common.py | 11 ++++++ 9 files changed, 73 insertions(+), 45 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 9acdbe8a1..b841ee477 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -94,4 +94,8 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### browser path for pyppeteer engine, support Chrome, Chromium,MS Edge #PYPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable" -PROMPT_FORMAT: json #json or markdown \ No newline at end of file +PROMPT_FORMAT: json #json or markdown + +### Agent configurations +# RAISE_NOT_CONFIG_ERROR: true # "true" if the LLM key is not configured, throw a NotConfiguredException, else "false". +# WORKSPACE_PATH_WITH_UID: false # "true" if using `{workspace}/{uid}` as the workspace path; "false" use `{workspace}`. \ No newline at end of file diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 05255dcc5..8d3445ae4 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -27,8 +27,8 @@ class PrepareDocuments(Action): # Create and initialize the workspace folder, initialize the Git environment. project_name = CONFIG.project_name or FileRepository.new_filename() workdir = CONFIG.project_path - if not workdir and CONFIG.workspace: - workdir = Path(CONFIG.workspace) / project_name + if not workdir and CONFIG.workspace_path: + workdir = Path(CONFIG.workspace_path) / project_name workdir = Path(workdir or DEFAULT_WORKSPACE_ROOT / project_name) if not CONFIG.inc and workdir.exists(): shutil.rmtree(workdir) diff --git a/metagpt/config.py b/metagpt/config.py index d04ae7291..aabd54c4b 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -6,10 +6,12 @@ Provide configuration, singleton 1. According to Section 2.2.3.11 of RFC 135, add git repository support. 2. Add the parameter `src_workspace` for the old version project path. """ +import datetime import os from copy import deepcopy from pathlib import Path from typing import Any +from uuid import uuid4 import yaml @@ -60,7 +62,11 @@ class Config(metaclass=Singleton): and (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key) and (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key) ): - raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") + val = self._get("RAISE_NOT_CONFIG_ERROR") + if val is None or val.lower() == "true": + raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") + else: # for agent + logger.warning("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy self.openai_api_type = self._get("OPENAI_API_TYPE") @@ -103,8 +109,15 @@ class Config(metaclass=Singleton): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.prompt_format = self._get("PROMPT_FORMAT", "markdown") + workspace_uid = ( + self._get("WORKSPACE_UID") or f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[-8:]}" + ) self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) + val = self._get("WORKSPACE_PATH_WITH_UID") + if val and val.lower() == "true": # for agent + self.workspace_path = self.workspace_path / workspace_uid self._ensure_workspace_exists() + self.max_auto_summarize_code = self.max_auto_summarize_code or self._get("MAX_AUTO_SUMMARIZE_CODE", 1) def _ensure_workspace_exists(self): self.workspace_path.mkdir(parents=True, exist_ok=True) diff --git a/metagpt/environment.py b/metagpt/environment.py index 02eb3d340..88beb5f25 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -49,7 +49,7 @@ class Environment(BaseModel): for role in roles: self.add_role(role) - def publish_message(self, message: Message) -> bool: + def publish_message(self, message: Message, peekable: bool = True) -> bool: """ Distribute the message to the recipients. In accordance with the Message routing structure design in Chapter 2.2.1 of RFC 116, as already planned diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index cedd2101f..4f7f0b796 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -42,7 +42,7 @@ from metagpt.schema import ( Documents, Message, ) -from metagpt.utils.common import any_to_str, any_to_str_set +from metagpt.utils.common import any_to_name, any_to_str, any_to_str_set IS_PASS_PROMPT = """ {context} @@ -83,6 +83,7 @@ class Engineer(Role): self.code_todos = [] self.summarize_todos = [] self.n_borg = n_borg + self._next_todo = any_to_name(WriteCode) @staticmethod def _parse_tasks(task_msg: Document) -> list[str]: @@ -124,8 +125,10 @@ class Engineer(Role): if self._rc.todo is None: return None if isinstance(self._rc.todo, WriteCode): + self._next_todo = any_to_name(SummarizeCode) return await self._act_write_code() if isinstance(self._rc.todo, SummarizeCode): + self._next_todo = any_to_name(WriteCode) return await self._act_summarize() return None @@ -296,3 +299,7 @@ class Engineer(Role): self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm)) if self.summarize_todos: self._rc.todo = self.summarize_todos[0] + + @property + def todo(self) -> str: + return self._next_todo diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 017feade7..284fcca96 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -11,6 +11,7 @@ from metagpt.actions import UserRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.config import CONFIG from metagpt.roles import Role +from metagpt.utils.common import any_to_name class ProductManager(Role): @@ -55,3 +56,10 @@ class ProductManager(Role): async def _observe(self, ignore_memory=False) -> int: return await super(ProductManager, self)._observe(ignore_memory=True) + + @property + def todo(self) -> str: + if self._rc.state == 0: + return any_to_name(WritePRD) + else: + return any_to_name(PrepareDocuments) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 52ac3cf28..e34daa307 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -30,10 +30,8 @@ from metagpt.config import CONFIG from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger from metagpt.memory import Memory - -# from metagpt.memory import LongTermMemory from metagpt.schema import Message, MessageQueue -from metagpt.utils.common import any_to_str +from metagpt.utils.common import any_to_name, any_to_str PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -191,6 +189,9 @@ class Role: # check RoleContext after adding watch actions self._rc.check(self._role_id) + def is_watch(self, caused_by: str): + return caused_by in self._rc.watch + def subscribe(self, tags: Set[str]): """Used to receive Messages with certain tags from the environment. Message will be put into personal message buffer to be further processed in _observe. By default, a Role subscribes Messages with a tag of its own name @@ -213,22 +214,6 @@ class Role: if env: env.set_subscription(self, self._subscription) - # # Replaced by FileRepository.set_file - # def set_doc(self, content: str, filename: str): - # return self._rc.env.set_doc(content, filename) - # - # # Replaced by FileRepository.get_file - # def get_doc(self, filename: str): - # return self._rc.env.get_doc(filename) - # - # # Replaced by CONFIG.xx - # def set(self, k, v): - # return self._rc.env.set(k, v) - # - # # Replaced by CONFIG.xx - # def get(self, k): - # return self._rc.env.get(k) - @property def profile(self): """Get the role description (position)""" @@ -368,23 +353,6 @@ class Role: self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None return rsp - # # Replaced by run() - # def recv(self, message: Message) -> None: - # """add message to history.""" - # # self._history += f"\n{message}" - # # self._context = self._history - # if message in self._rc.memory.get(): - # return - # self._rc.memory.add(message) - - # # Replaced by run() - # async def handle(self, message: Message) -> Message: - # """Receive information and reply with actions""" - # # logger.debug(f"{self.name=}, {self.profile=}, {message.role=}") - # self.recv(message) - # - # return await self._react() - def get_memories(self, k=0) -> list[Message]: """A wrapper to return the most recent k memories of this role, return all when k=0""" return self._rc.memory.get(k=k) @@ -418,3 +386,19 @@ class Role: def is_idle(self) -> bool: """If true, all actions have been executed.""" return not self._rc.news and not self._rc.todo and self._rc.msg_buffer.empty() + + async def think(self) -> Action: + """The exported `think` function""" + await self._think() + return self._rc.todo + + async def act(self) -> ActionOutput: + """The exported `act` function""" + msg = await self._act() + return ActionOutput(content=msg.content, instruct_content=msg.instruct_content) + + @property + def todo(self) -> str: + if self._actions: + return any_to_name(self._actions[0]) + return "" diff --git a/metagpt/team.py b/metagpt/team.py index 92f379c97..152ad24f0 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -52,13 +52,14 @@ class Team(BaseModel): # Human requirement. self.env.publish_message( - Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to or MESSAGE_ROUTE_TO_ALL) + Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to or MESSAGE_ROUTE_TO_ALL), + peekable=False, ) def _save(self): logger.info(self.json(ensure_ascii=False)) - async def run(self, n_round=3): + async def run(self, n_round=3, auto_archive=True): """Run company until target round or no money""" while n_round > 0: # self._save() @@ -66,6 +67,6 @@ class Team(BaseModel): logger.debug(f"{n_round=}") self._check_balance() await self.env.run() - if CONFIG.git_repo: + if auto_archive and CONFIG.git_repo: CONFIG.git_repo.archive() return self.env.history diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index f08519f8e..8d4d8eaf9 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -358,3 +358,14 @@ def is_subscribed(message, tags): if t in message.send_to: return True return False + + +def any_to_name(val): + """ + Convert a value to its name by extracting the last part of the dotted path. + + :param val: The value to convert. + + :return: The name of the value. + """ + return any_to_str(val).split(".")[-1] From ea21217a697abf7f2bc2e0b014478544ec8bb61f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 14 Dec 2023 22:59:41 +0800 Subject: [PATCH 387/398] feat: merge send18 --- config/config.yaml | 18 +- examples/search_kb.py | 6 - examples/search_with_specific_engine.py | 4 +- examples/write_teaching_plan.py | 15 +- metagpt/__init__.py | 15 -- metagpt/actions/action.py | 10 +- metagpt/actions/action_output.py | 4 +- metagpt/actions/design_api.py | 36 ---- metagpt/actions/project_management.py | 43 +---- metagpt/actions/write_code.py | 33 +--- metagpt/actions/write_prd.py | 64 ++----- metagpt/actions/write_teaching_plan.py | 139 ++++++++------- metagpt/config.py | 2 + metagpt/const.py | 5 +- metagpt/document_store/faiss_store.py | 23 +-- metagpt/llm.py | 29 +--- metagpt/management/skill_manager.py | 6 - metagpt/provider/__init__.py | 22 ++- metagpt/provider/human_provider.py | 7 +- metagpt/provider/metagpt_llm_api.py | 79 ++++----- metagpt/provider/openai_api.py | 11 +- metagpt/provider/zhipuai/async_sse_client.py | 7 +- metagpt/provider/zhipuai_api.py | 27 ++- metagpt/roles/engineer.py | 164 ------------------ metagpt/roles/qa_engineer.py | 38 +--- metagpt/roles/researcher.py | 11 +- metagpt/roles/role.py | 146 +--------------- metagpt/schema.py | 19 +- metagpt/team.py | 5 +- metagpt/tools/__init__.py | 4 +- metagpt/tools/hello.py | 2 +- metagpt/tools/metagpt_text_to_image.py | 11 +- metagpt/tools/openai_text_to_embedding.py | 14 +- metagpt/tools/sd_engine.py | 21 +-- metagpt/tools/web_browser_engine.py | 10 +- metagpt/tools/web_browser_engine_selenium.py | 14 +- metagpt/utils/common.py | 25 ++- metagpt/utils/cost_manager.py | 9 +- metagpt/utils/git_repository.py | 9 +- metagpt/utils/mermaid.py | 50 +----- tests/conftest.py | 4 +- tests/metagpt/actions/test_ui_design.py | 1 - tests/metagpt/actions/test_write_code.py | 3 +- .../actions/test_write_teaching_plan.py | 21 +-- tests/metagpt/learn/test_text_to_embedding.py | 6 +- tests/metagpt/learn/test_text_to_image.py | 8 +- tests/metagpt/learn/test_text_to_speech.py | 10 +- tests/metagpt/memory/test_brain_memory.py | 14 +- tests/metagpt/roles/test_teacher.py | 34 ++-- tests/metagpt/test_environment.py | 22 ++- tests/metagpt/test_llm.py | 7 +- tests/metagpt/tools/test_sd_tool.py | 1 - .../test_web_browser_engine_playwright.py | 5 +- tests/metagpt/utils/test_config.py | 3 +- 54 files changed, 366 insertions(+), 930 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index ff1ae769d..3aeabf251 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -105,15 +105,15 @@ PROMPT_FORMAT: json #json or markdown #METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL ### S3 config -S3_ACCESS_KEY: "YOUR_S3_ACCESS_KEY" -S3_SECRET_KEY: "YOUR_S3_SECRET_KEY" -S3_ENDPOINT_URL: "YOUR_S3_ENDPOINT_URL" -S3_SECURE: true # true/false -S3_BUCKET: "YOUR_S3_BUCKET" +#S3_ACCESS_KEY: "YOUR_S3_ACCESS_KEY" +#S3_SECRET_KEY: "YOUR_S3_SECRET_KEY" +#S3_ENDPOINT_URL: "YOUR_S3_ENDPOINT_URL" +#S3_SECURE: true # true/false +#S3_BUCKET: "YOUR_S3_BUCKET" ### Redis config -REDIS_HOST: "YOUR_REDIS_HOST" -REDIS_PORT: "YOUR_REDIS_PORT" -REDIS_PASSWORD: "YOUR_REDIS_PASSWORD" -REDIS_DB: "YOUR_REDIS_DB_INDEX, str, 0-based" +#REDIS_HOST: "YOUR_REDIS_HOST" +#REDIS_PORT: "YOUR_REDIS_PORT" +#REDIS_PASSWORD: "YOUR_REDIS_PASSWORD" +#REDIS_DB: "YOUR_REDIS_DB_INDEX, str, 0-based" diff --git a/examples/search_kb.py b/examples/search_kb.py index c2ded1769..85d99854e 100644 --- a/examples/search_kb.py +++ b/examples/search_kb.py @@ -5,14 +5,8 @@ @Modified By: mashenquan, 2023-8-9, fix-bug: cannot find metagpt module. """ import asyncio -<<<<<<< HEAD from metagpt.actions import Action -======= -from pathlib import Path -import sys -sys.path.append(str(Path(__file__).resolve().parent.parent)) ->>>>>>> send18/dev from metagpt.const import DATA_PATH from metagpt.document_store import FaissStore from metagpt.logs import logger diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index c7c455b7e..97db1624a 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -4,9 +4,7 @@ @Modified By: mashenquan, 2023-8-9, fix-bug: cannot find metagpt module. """ import asyncio -from pathlib import Path -import sys -sys.path.append(str(Path(__file__).resolve().parent.parent)) + from metagpt.roles import Searcher from metagpt.tools import SearchEngineType diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index c3a647b94..01181dc2b 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -15,14 +15,15 @@ import asyncio from pathlib import Path -from metagpt.config import CONFIG - import aiofiles import fire -from metagpt.logs import logger + from metagpt.actions.write_teaching_plan import TeachingPlanRequirement +from metagpt.config import CONFIG +from metagpt.logs import logger from metagpt.roles.teacher import Teacher -from metagpt.software_company import SoftwareCompany +from metagpt.schema import Message +from metagpt.team import Team async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): @@ -82,10 +83,10 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * logger.info("No course content provided, using the demo course.") lesson = demo_lesson - company = SoftwareCompany() + company = Team() company.hire([Teacher(*args, **kwargs)]) company.invest(investment) - company.start_project(lesson, cause_by=TeachingPlanRequirement, role="Teacher", **kwargs) + company.env.publish_message(Message(content=lesson, cause_by=TeachingPlanRequirement)) await company.run(n_round=1) @@ -102,7 +103,7 @@ def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) -if __name__ == '__main__': +if __name__ == "__main__": """ Formats: ``` diff --git a/metagpt/__init__.py b/metagpt/__init__.py index aa1965e31..71ddd1aff 100644 --- a/metagpt/__init__.py +++ b/metagpt/__init__.py @@ -1,22 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -<<<<<<< HEAD # @Time : 2023/4/24 22:26 # @Author : alexanderwu # @File : __init__.py from metagpt import _compat as _ # noqa: F401 -======= -""" -@Time : 2023/4/24 22:26 -@Author : alexanderwu -@File : __init__.py -@Desc : mashenquan, 2023/8/22. Add `Message` for importing by external projects. -""" - -from metagpt.schema import Message - -__all__ = [ - "Message", -] ->>>>>>> send18/dev diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 442004e09..2b4317736 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -8,19 +8,21 @@ @Modified By: mashenquan, 2023/9/8. Replace LLM with LLMFactory """ -import re from __future__ import annotations + +import re from abc import ABC from typing import Optional + from tenacity import retry, stop_after_attempt, wait_random_exponential + from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM from metagpt.logs import logger -from metagpt.utils.common import OutputParser -from metagpt.utils.custom_decoder import CustomDecoder -from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.common import OutputParser +from metagpt.utils.custom_decoder import CustomDecoder + class Action(ABC): def __init__(self, name: str = "", context=None, llm: BaseGPTAPI = None): diff --git a/metagpt/actions/action_output.py b/metagpt/actions/action_output.py index 49c7dea2e..87d1c31ff 100644 --- a/metagpt/actions/action_output.py +++ b/metagpt/actions/action_output.py @@ -7,7 +7,7 @@ @Modified By: mashenquan, 2023/8/20. Allow 'instruct_content' to be blank. """ -from typing import Dict, Type, Optional +from typing import Dict, Optional, Type from pydantic import BaseModel, create_model, root_validator, validator @@ -16,7 +16,7 @@ class ActionOutput: content: str instruct_content: Optional[BaseModel] = None - def __init__(self, content: str, instruct_content: BaseModel=None): + def __init__(self, content: str, instruct_content: BaseModel = None): self.content = content self.instruct_content = instruct_content diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index bccbc1261..557ebcbbd 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : design_api.py -<<<<<<< HEAD @Modified By: mashenquan, 2023/11/27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality. @@ -23,16 +22,6 @@ from metagpt.const import ( SYSTEM_DESIGN_FILE_REPO, SYSTEM_DESIGN_PDF_FILE_REPO, ) -======= -@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. -""" -from typing import List - -import aiofiles - -from metagpt.actions import Action -from metagpt.config import CONFIG ->>>>>>> send18/dev from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository @@ -208,7 +197,6 @@ class WriteDesign(Action): "clearly and in detail." ) -<<<<<<< HEAD async def run(self, with_messages, format=CONFIG.prompt_format): # Use `git diff` to identify which PRD documents have been modified in the `docs/prds` directory. prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) @@ -244,30 +232,6 @@ class WriteDesign(Action): format_example = format_example.format(project_name=CONFIG.project_name) prompt = prompt_template.format(context=context, format_example=format_example) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) -======= - async def _save_system_design(self, docs_path, resources_path, content): - data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content) - seq_flow = CodeParser.parse_code(block="Program call flow", text=content) - await mermaid_to_file(data_api_design, resources_path / "data_api_design") - await mermaid_to_file(seq_flow, resources_path / "seq_flow") - system_design_file = docs_path / "system_design.md" - logger.info(f"Saving System Designs to {system_design_file}") - async with aiofiles.open(system_design_file, "w") as f: - await f.write(content) - - async def _save(self, system_design: str): - workspace = CONFIG.workspace - docs_path = workspace / "docs" - resources_path = workspace / "resources" - docs_path.mkdir(parents=True, exist_ok=True) - resources_path.mkdir(parents=True, exist_ok=True) - await self._save_system_design(docs_path, resources_path, system_design) - - async def run(self, context, **kwargs): - prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) - system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) - await self._save(system_design.content) ->>>>>>> send18/dev return system_design async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 53ef872e2..40965ab5c 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -4,19 +4,14 @@ @Time : 2023/5/11 19:12 @Author : alexanderwu @File : project_management.py -<<<<<<< HEAD @Modified By: mashenquan, 2023/11/27. 1. Divide the context into three components: legacy code, unit test code, and console log. 2. Move the document storage operations related to WritePRD from the save operation of WriteDesign. 3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality. -======= -@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. ->>>>>>> send18/dev """ import json from typing import List -<<<<<<< HEAD from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.config import CONFIG @@ -91,14 +86,6 @@ and only output the json inside this tag, nothing else }, "markdown": { "PROMPT_TEMPLATE": """ -======= -import aiofiles - -from metagpt.actions.action import Action -from metagpt.config import CONFIG - -PROMPT_TEMPLATE = """ ->>>>>>> send18/dev # Context {context} @@ -121,11 +108,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -<<<<<<< HEAD ## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. -======= -""" ->>>>>>> send18/dev """, "FORMAT_EXAMPLE": ''' @@ -197,7 +180,6 @@ MERGE_PROMPT = """ # Context {context} -<<<<<<< HEAD ## Old Tasks {old_tasks} ----- @@ -228,13 +210,10 @@ and only output the json inside this tag, nothing else """ -======= ->>>>>>> send18/dev class WriteTasks(Action): def __init__(self, name="CreateTasks", context=None, llm=None): super().__init__(name, context, llm) -<<<<<<< HEAD async def run(self, with_messages, format=CONFIG.prompt_format): system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) changed_system_designs = system_design_file_repo.changed_files @@ -286,29 +265,13 @@ class WriteTasks(Action): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) -======= - async def _save(self, rsp): - file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md" - async with aiofiles.open(file_path, "w") as f: - await f.write(rsp.content) - - # Write requirements.txt - requirements_path = CONFIG.workspace / "requirements.txt" - - async with aiofiles.open(requirements_path, "w") as f: - await f.write(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) - - async def run(self, context, **kwargs): - prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) - rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) - await self._save(rsp) ->>>>>>> send18/dev return rsp async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: _, format_example = get_template(templates, format) - prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content, - format_example=format_example) + prompt = MERGE_PROMPT.format( + context=system_design_doc.content, old_tasks=task_doc.content, format_example=format_example + ) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) task_doc.content = rsp.instruct_content.json(ensure_ascii=False) return task_doc diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index b61e3886c..a2501db2a 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -14,27 +14,23 @@ 3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. """ -<<<<<<< HEAD import json from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO, TASK_FILE_REPO, BUGFIX_FILENAME, \ - DOCS_FILE_REPO -======= -from tenacity import retry, stop_after_attempt, wait_fixed - -from metagpt.actions.action import Action ->>>>>>> send18/dev +from metagpt.const import ( + BUGFIX_FILENAME, + CODE_SUMMARIES_FILE_REPO, + DOCS_FILE_REPO, + TASK_FILE_REPO, + TEST_OUTPUTS_FILE_REPO, +) from metagpt.logs import logger from metagpt.schema import CodingContext, Document, RunCodeResult from metagpt.utils.common import CodeParser -<<<<<<< HEAD from metagpt.utils.file_repository import FileRepository -======= ->>>>>>> send18/dev PROMPT_TEMPLATE = """ NOTICE @@ -98,21 +94,12 @@ class WriteCode(Action): def __init__(self, name="WriteCode", context=None, llm=None): super().__init__(name, context, llm) -<<<<<<< HEAD @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) async def write_code(self, prompt) -> str: -======= - def _is_invalid(self, filename): - return any(i in filename for i in ["mp3", "wav"]) - - @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def write_code(self, prompt): ->>>>>>> send18/dev code_rsp = await self._aask(prompt) code = CodeParser.parse_code(block="", text=code_rsp) return code -<<<<<<< HEAD async def run(self, *args, **kwargs) -> CodingContext: bug_feedback = await FileRepository.get_file(filename=BUGFIX_FILENAME, relative_path=DOCS_FILE_REPO) coding_context = CodingContext.loads(self.context.content) @@ -139,11 +126,6 @@ class WriteCode(Action): summary_log=summary_doc.content if summary_doc else "", ) logger.info(f"Writing {coding_context.filename}..") -======= - async def run(self, context, filename): - prompt = PROMPT_TEMPLATE.format(context=context, filename=filename) - logger.info(f"Writing {filename}..") ->>>>>>> send18/dev code = await self.write_code(prompt) if not coding_context.code_doc: coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace) @@ -166,4 +148,3 @@ class WriteCode(Action): continue codes.append(doc.content) return "\n----------\n".join(codes) - diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index d8042b3ed..9aacb0751 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -16,22 +16,20 @@ import json from pathlib import Path from typing import List -import aiofiles - from metagpt.actions import Action, ActionOutput from metagpt.actions.fix_bug import FixBug from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG -<<<<<<< HEAD from metagpt.const import ( + BUGFIX_FILENAME, COMPETITIVE_ANALYSIS_FILE_REPO, DOCS_FILE_REPO, PRD_PDF_FILE_REPO, PRDS_FILE_REPO, - REQUIREMENT_FILENAME, BUGFIX_FILENAME, + REQUIREMENT_FILENAME, ) from metagpt.logs import logger -from metagpt.schema import Document, Documents, Message, BugFixContext +from metagpt.schema import BugFixContext, Document, Documents, Message from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template @@ -55,11 +53,6 @@ Requirements: According to the context, fill in the following missing informatio ATTENTION: Output carefully referenced "Format example" in format. ## YOU NEED TO FULFILL THE BELOW JSON DOC -======= -from metagpt.logs import logger -from metagpt.utils.common import CodeParser -from metagpt.utils.mermaid import mermaid_to_file ->>>>>>> send18/dev {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. @@ -245,11 +238,7 @@ OUTPUT_MAPPING = { "Competitive Analysis": (List[str], ...), "Competitive Quadrant Chart": (str, ...), "Requirement Analysis": (str, ...), -<<<<<<< HEAD "Requirement Pool": (List[List[str]], ...), -======= - "Requirement Pool": (List[Tuple[str, str]], ...), ->>>>>>> send18/dev "UI Design draft": (str, ...), "Anything UNCLEAR": (str, ...), } @@ -346,12 +335,14 @@ class WritePRD(Action): await docs_file_repo.save(filename=BUGFIX_FILENAME, content=requirement_doc.content) await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="") bug_fix = BugFixContext(filename=BUGFIX_FILENAME) - return Message(content=bug_fix.json(), instruct_content=bug_fix, - role=self.profile, - cause_by=FixBug, - sent_from=self, - send_to="Alex", # the name of Engineer - ) + return Message( + content=bug_fix.json(), + instruct_content=bug_fix, + role=self.profile, + cause_by=FixBug, + sent_from=self, + send_to="Alex", # the name of Engineer + ) else: await docs_file_repo.delete(filename=BUGFIX_FILENAME) @@ -388,7 +379,6 @@ class WritePRD(Action): logger.info(sas.result) logger.info(rsp) -<<<<<<< HEAD # logger.info(format) prompt_template, format_example = get_template(templates, format) project_name = CONFIG.project_name if CONFIG.project_name else "" @@ -447,7 +437,7 @@ class WritePRD(Action): if not quadrant_chart: return pathname = ( - CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") + CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") ) if not pathname.parent.exists(): pathname.parent.mkdir(parents=True, exist_ok=True) @@ -480,33 +470,3 @@ class WritePRD(Action): if "YES" in res: return True return False -======= - prompt = PROMPT_TEMPLATE.format( - requirements=requirements, search_information=info, format_example=FORMAT_EXAMPLE - ) - logger.debug(prompt) - prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) - - await self._save(prd.content) - return prd - - async def _save_prd(self, docs_path, resources_path, prd): - prd_file = docs_path / "prd.md" - quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) - await mermaid_to_file( - mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / "competitive_analysis" - ) - async with aiofiles.open(prd_file, "w") as f: - await f.write(prd) - logger.info(f"Saving PRD to {prd_file}") - - async def _save(self, prd): - workspace = CONFIG.workspace - workspace.mkdir(parents=True, exist_ok=True) - - docs_path = workspace / "docs" - resources_path = workspace / "resources" - docs_path.mkdir(parents=True, exist_ok=True) - resources_path.mkdir(parents=True, exist_ok=True) - await self._save_prd(docs_path, resources_path, prd) ->>>>>>> send18/dev diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 7c959ce85..529c563db 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -5,9 +5,10 @@ @Author : mashenquan @File : write_teaching_plan.py """ -from metagpt.logs import logger from metagpt.actions import Action +from metagpt.logs import logger from metagpt.schema import Message +from metagpt.utils.common import format_value class TeachingPlanRequirement(Action): @@ -40,17 +41,18 @@ class WriteTeachingPlanPart(Action): statement_patterns = self.TOPIC_STATEMENTS.get(self.topic, []) statements = [] - from metagpt.roles import Role for p in statement_patterns: - s = Role.format_value(p) + s = format_value(p) statements.append(s) formatter = self.PROMPT_TITLE_TEMPLATE if self.topic == self.COURSE_TITLE else self.PROMPT_TEMPLATE - prompt = formatter.format(formation=self.FORMATION, - role=self.prefix, - statements="\n".join(statements), - lesson=messages[0].content, - topic=self.topic, - language=self.language) + prompt = formatter.format( + formation=self.FORMATION, + role=self.prefix, + statements="\n".join(statements), + lesson=messages[0].content, + topic=self.topic, + language=self.language, + ) logger.debug(prompt) rsp = await self._aask(prompt=prompt) @@ -61,14 +63,14 @@ class WriteTeachingPlanPart(Action): def _set_result(self, rsp): if self.DATA_BEGIN_TAG in rsp: ix = rsp.index(self.DATA_BEGIN_TAG) - rsp = rsp[ix + len(self.DATA_BEGIN_TAG):] + rsp = rsp[ix + len(self.DATA_BEGIN_TAG) :] if self.DATA_END_TAG in rsp: ix = rsp.index(self.DATA_END_TAG) rsp = rsp[0:ix] self.rsp = rsp.strip() if self.topic != self.COURSE_TITLE: return - if '#' not in self.rsp or self.rsp.index('#') != 0: + if "#" not in self.rsp or self.rsp.index("#") != 0: self.rsp = "# " + self.rsp def __str__(self): @@ -79,81 +81,102 @@ class WriteTeachingPlanPart(Action): """Show `topic` value when debug""" return self.topic - FORMATION = "\"Capacity and role\" defines the role you are currently playing;\n" \ - "\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n" \ - "\t\"Statement\" defines the work detail you need to complete at this stage;\n" \ - "\t\"Answer options\" defines the format requirements for your responses;\n" \ - "\t\"Constraint\" defines the conditions that your responses must comply with." + FORMATION = ( + '"Capacity and role" defines the role you are currently playing;\n' + '\t"[LESSON_BEGIN]" and "[LESSON_END]" tags enclose the content of textbook;\n' + '\t"Statement" defines the work detail you need to complete at this stage;\n' + '\t"Answer options" defines the format requirements for your responses;\n' + '\t"Constraint" defines the conditions that your responses must comply with.' + ) COURSE_TITLE = "Title" TOPICS = [ - COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", - "Teaching Methods and Strategies", "Learning Activities", - "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", - "Vocabulary Cloze", "Choice Questions", "Grammar Questions", "Translation Questions" + COURSE_TITLE, + "Teaching Hours", + "Teaching Objectives", + "Teaching Content", + "Teaching Methods and Strategies", + "Learning Activities", + "Teaching Time Allocation", + "Assessment and Feedback", + "Teaching Summary and Improvement", + "Vocabulary Cloze", + "Choice Questions", + "Grammar Questions", + "Translation Questions", ] TOPIC_STATEMENTS = { - COURSE_TITLE: ["Statement: Find and return the title of the lesson only in markdown first-level header format, " - "without anything else."], + COURSE_TITLE: [ + "Statement: Find and return the title of the lesson only in markdown first-level header format, " + "without anything else." + ], "Teaching Content": [ - "Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar " + 'Statement: "Teaching Content" must include vocabulary, analysis, and examples of various grammar ' "structures that appear in the textbook, as well as the listening materials and key points.", - "Statement: \"Teaching Content\" must include more examples."], + 'Statement: "Teaching Content" must include more examples.', + ], "Teaching Time Allocation": [ - "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each " - "part of the textbook content."], + 'Statement: "Teaching Time Allocation" must include how much time is allocated to each ' + "part of the textbook content." + ], "Teaching Methods and Strategies": [ - "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, " + 'Statement: "Teaching Methods and Strategies" must include teaching focus, difficulties, materials, ' "procedures, in detail." ], "Vocabulary Cloze": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + 'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", ' "create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} " "answers, and it should also include 10 {teaching_language} questions with {language} answers. " "The key-related vocabulary and phrases in the textbook content must all be included in the exercises.", ], "Grammar Questions": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create grammar questions. 10 questions."], + 'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", ' + "create grammar questions. 10 questions." + ], "Choice Questions": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create choice questions. 10 questions."], + 'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", ' + "create choice questions. 10 questions." + ], "Translation Questions": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + 'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", ' "create translation questions. The translation should include 10 {language} questions with " "{teaching_language} answers, and it should also include 10 {teaching_language} questions with " "{language} answers." - ] + ], } # Teaching plan title - PROMPT_TITLE_TEMPLATE = "Do not refer to the context of the previous conversation records, " \ - "start the conversation anew.\n\n" \ - "Formation: {formation}\n\n" \ - "{statements}\n" \ - "Constraint: Writing in {language}.\n" \ - "Answer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" " \ - "and \"[TEACHING_PLAN_END]\" tags.\n" \ - "[LESSON_BEGIN]\n" \ - "{lesson}\n" \ - "[LESSON_END]" + PROMPT_TITLE_TEMPLATE = ( + "Do not refer to the context of the previous conversation records, " + "start the conversation anew.\n\n" + "Formation: {formation}\n\n" + "{statements}\n" + "Constraint: Writing in {language}.\n" + 'Answer options: Encloses the lesson title with "[TEACHING_PLAN_BEGIN]" ' + 'and "[TEACHING_PLAN_END]" tags.\n' + "[LESSON_BEGIN]\n" + "{lesson}\n" + "[LESSON_END]" + ) # Teaching plan parts: - PROMPT_TEMPLATE = "Do not refer to the context of the previous conversation records, " \ - "start the conversation anew.\n\n" \ - "Formation: {formation}\n\n" \ - "Capacity and role: {role}\n" \ - "Statement: Write the \"{topic}\" part of teaching plan, " \ - "WITHOUT ANY content unrelated to \"{topic}\"!!\n" \ - "{statements}\n" \ - "Answer options: Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" " \ - "and \"[TEACHING_PLAN_END]\" tags.\n" \ - "Answer options: Using proper markdown format from second-level header format.\n" \ - "Constraint: Writing in {language}.\n" \ - "[LESSON_BEGIN]\n" \ - "{lesson}\n" \ - "[LESSON_END]" + PROMPT_TEMPLATE = ( + "Do not refer to the context of the previous conversation records, " + "start the conversation anew.\n\n" + "Formation: {formation}\n\n" + "Capacity and role: {role}\n" + 'Statement: Write the "{topic}" part of teaching plan, ' + 'WITHOUT ANY content unrelated to "{topic}"!!\n' + "{statements}\n" + 'Answer options: Enclose the teaching plan content with "[TEACHING_PLAN_BEGIN]" ' + 'and "[TEACHING_PLAN_END]" tags.\n' + "Answer options: Using proper markdown format from second-level header format.\n" + "Constraint: Writing in {language}.\n" + "[LESSON_BEGIN]\n" + "{lesson}\n" + "[LESSON_END]" + ) DATA_BEGIN_TAG = "[TEACHING_PLAN_BEGIN]" DATA_END_TAG = "[TEACHING_PLAN_END]" diff --git a/metagpt/config.py b/metagpt/config.py index d3123b1f7..92980ec4e 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -13,7 +13,9 @@ from copy import deepcopy from pathlib import Path from typing import Any from uuid import uuid4 + import yaml + from metagpt.const import DEFAULT_WORKSPACE_ROOT, METAGPT_ROOT, OPTIONS from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType diff --git a/metagpt/const.py b/metagpt/const.py index c2b6c308d..03f3d8fe3 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -12,7 +12,9 @@ import contextvars import os from pathlib import Path + from loguru import logger + import metagpt OPTIONS = contextvars.ContextVar("OPTIONS") @@ -89,6 +91,8 @@ TEST_CODES_FILE_REPO = "tests" TEST_OUTPUTS_FILE_REPO = "test_outputs" CODE_SUMMARIES_FILE_REPO = "docs/code_summaries" CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries" +RESOURCES_FILE_REPO = "resources" +SD_OUTPUT_FILE_REPO = "resources/SD_Output" YAPI_URL = "http://yapi.deepwisdomai.com/" @@ -105,4 +109,3 @@ BASE64_FORMAT = "base64" # REDIS REDIS_KEY = "REDIS_KEY" - diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 65685dffa..7acaa194d 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -21,18 +21,13 @@ from metagpt.logs import logger class FaissStore(LocalStore): -<<<<<<< HEAD - def __init__(self, raw_data_path: Path, cache_dir=None, meta_col="source", content_col="output"): - self.meta_col = meta_col - self.content_col = content_col - super().__init__(raw_data_path, cache_dir) -======= - def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output", embedding_conf=None): + def __init__( + self, raw_data_path: Path, cache_dir=None, meta_col="source", content_col="output", embedding_conf=None + ): self.meta_col = meta_col self.content_col = content_col self.embedding_conf = embedding_conf or {} - super().__init__(raw_data, cache_dir) ->>>>>>> send18/dev + super().__init__(raw_data_path, cache_dir) def _load(self) -> Optional["FaissStore"]: index_file, store_file = self._get_index_and_store_fname() @@ -46,7 +41,9 @@ class FaissStore(LocalStore): return store def _write(self, docs, metadatas): - store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07", **self.embedding_conf), metadatas=metadatas) + store = FAISS.from_texts( + docs, OpenAIEmbeddings(openai_api_version="2020-11-07", **self.embedding_conf), metadatas=metadatas + ) return store def persist(self): @@ -92,12 +89,6 @@ class FaissStore(LocalStore): if __name__ == "__main__": faiss_store = FaissStore(DATA_PATH / "qcs/qcs_4w.json") -<<<<<<< HEAD logger.info(faiss_store.search("Oily Skin Facial Cleanser")) faiss_store.add([f"Oily Skin Facial Cleanser-{i}" for i in range(3)]) logger.info(faiss_store.search("Oily Skin Facial Cleanser")) -======= - logger.info(faiss_store.search("油皮洗面奶")) - faiss_store.add([f"油皮洗面奶-{i}" for i in range(3)]) - logger.info(faiss_store.search("油皮洗面奶")) ->>>>>>> send18/dev diff --git a/metagpt/llm.py b/metagpt/llm.py index 525d2a65e..7701ebec2 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -6,36 +6,19 @@ @File : llm.py @Modified By: mashenquan, 2023 """ -from enum import Enum + from metagpt.config import CONFIG +from metagpt.provider import LLMType from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.openai_api import OpenAIGPTAPI -from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI -from metagpt.provider.spark_api import SparkAPI from metagpt.provider.human_provider import HumanProvider from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI +from metagpt.provider.openai_api import OpenAIGPTAPI +from metagpt.provider.spark_api import SparkAPI +from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI _ = HumanProvider() # Avoid pre-commit error -class LLMType(Enum): - OPENAI = "OpenAI" - METAGPT = "MetaGPT" - CLAUDE = "Claude" - UNKNOWN = "UNKNOWN" - - @classmethod - def get(cls, value): - for member in cls: - if member.value == value: - return member - return cls.UNKNOWN - - @classmethod - def __missing__(cls, value): - return cls.UNKNOWN - - # Used in agents class LLMFactory: @staticmethod @@ -62,5 +45,5 @@ class LLMFactory: # Used in metagpt def LLM() -> "BaseGPTAPI": - """ initialize different LLM instance according to the key field existence""" + """initialize different LLM instance according to the key field existence""" return LLMFactory.new_llm() diff --git a/metagpt/management/skill_manager.py b/metagpt/management/skill_manager.py index 33f283680..e4892e3d9 100644 --- a/metagpt/management/skill_manager.py +++ b/metagpt/management/skill_manager.py @@ -18,14 +18,8 @@ class SkillManager: """Used to manage all skills""" def __init__(self): -<<<<<<< HEAD - self._llm = LLM() self._store = ChromaStore("skill_manager") self._skills: dict[str:Skill] = {} -======= - self._store = ChromaStore('skill_manager') - self._skills: dict[str: Skill] = {} ->>>>>>> send18/dev def add_skill(self, skill: Skill): """ diff --git a/metagpt/provider/__init__.py b/metagpt/provider/__init__.py index 9895aa7fc..3517e1376 100644 --- a/metagpt/provider/__init__.py +++ b/metagpt/provider/__init__.py @@ -4,11 +4,23 @@ @Time : 2023/5/5 22:59 @Author : alexanderwu @File : __init__.py -@Modified By: mashenquan, 2023/9/8. Add `MetaGPTLLMAPI` +@Modified By: mashenquan, 2023-12-15. Add LLMType """ - -from metagpt.provider.openai_api import OpenAIGPTAPI -from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI +from enum import Enum -__all__ = ["OpenAIGPTAPI", "MetaGPTLLMAPI"] +class LLMType(Enum): + OPENAI = "OpenAI" + METAGPT = "MetaGPT" + UNKNOWN = "UNKNOWN" + + @classmethod + def get(cls, value): + for member in cls: + if member.value == value: + return member + return cls.UNKNOWN + + @classmethod + def __missing__(cls, value): + return cls.UNKNOWN diff --git a/metagpt/provider/human_provider.py b/metagpt/provider/human_provider.py index ba9c93c88..5850dd8dc 100644 --- a/metagpt/provider/human_provider.py +++ b/metagpt/provider/human_provider.py @@ -21,11 +21,14 @@ class HumanProvider(BaseGPTAPI): exit() return rsp - async def aask(self, msg: str, + async def aask( + self, + msg: str, system_msgs: Optional[list[str]] = None, format_msgs: Optional[list[dict[str, str]]] = None, generator: bool = False, - timeout=3,) -> str: + timeout=3, + ) -> str: return self.ask(msg, timeout=timeout) def completion(self, messages: list[dict], timeout=3): diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 925ac6623..994fc39ff 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -7,13 +7,14 @@ """ from metagpt.provider.openai_api import OpenAIGPTAPI + # from metagpt.provider.base_gpt_api import BaseGPTAPI # from metagpt.provider.openai_api import RateLimiter class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" - + def __init__(self): super(MetaGPTLLMAPI, self).__init__() @@ -24,7 +25,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # self.auto_max_tokens = False # self._cost_manager = CostManager() # RateLimiter.__init__(self, rpm=self.rpm) - # + # # def __init_openai(self, config): # openai.api_key = config.openai_api_key # if config.openai_api_base: @@ -33,10 +34,10 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # openai.api_type = config.openai_api_type # openai.api_version = config.openai_api_version # self.rpm = int(config.get("RPM", 10)) - # + # # async def _achat_completion_stream(self, messages: list[dict]) -> str: # response = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages), stream=True) - # + # # # create variables to collect the stream of chunks # collected_chunks = [] # collected_messages = [] @@ -50,12 +51,12 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # if "content" in chunk_message: # print(chunk_message["content"], end="") # print() - # + # # full_reply_content = "".join([m.get("content", "") for m in collected_messages]) # usage = self._calc_usage(messages, full_reply_content) # self._update_costs(usage) # return full_reply_content - # + # # def _cons_kwargs(self, messages: list[dict], **configs) -> dict: # kwargs = { # "messages": messages, @@ -67,7 +68,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # } # if configs: # kwargs.update(configs) - # + # # if CONFIG.openai_api_type == "azure": # if CONFIG.deployment_name and CONFIG.deployment_id: # raise ValueError("You can only use one of the `deployment_id` or `deployment_name` model") @@ -82,27 +83,27 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # kwargs_mode = {"model": self.model} # kwargs.update(kwargs_mode) # return kwargs - # + # # async def _achat_completion(self, messages: list[dict]) -> dict: # rsp = await self.llm.ChatCompletion.acreate(**self._cons_kwargs(messages)) # self._update_costs(rsp.get("usage")) # return rsp - # + # # def _chat_completion(self, messages: list[dict]) -> dict: # rsp = self.llm.ChatCompletion.create(**self._cons_kwargs(messages)) # self._update_costs(rsp) # return rsp - # + # # def completion(self, messages: list[dict]) -> dict: # # if isinstance(messages[0], Message): # # messages = self.messages_to_dict(messages) # return self._chat_completion(messages) - # + # # async def acompletion(self, messages: list[dict]) -> dict: # # if isinstance(messages[0], Message): # # messages = self.messages_to_dict(messages) # return await self._achat_completion(messages) - # + # # @retry( # wait=wait_random_exponential(min=1, max=60), # stop=stop_after_attempt(6), @@ -116,7 +117,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # return await self._achat_completion_stream(messages) # rsp = await self._achat_completion(messages) # return self.get_choice_text(rsp) - # + # # def _func_configs(self, messages: list[dict], **kwargs) -> dict: # """ # Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create @@ -127,25 +128,25 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # "tool_choice": GENERAL_TOOL_CHOICE, # } # kwargs.update(configs) - # + # # return self._cons_kwargs(messages, **kwargs) - # + # # def _chat_completion_function(self, messages: list[dict], **kwargs) -> dict: # rsp = self.llm.ChatCompletion.create(**self._func_configs(messages, **kwargs)) # self._update_costs(rsp.get("usage")) # return rsp - # + # # async def _achat_completion_function(self, messages: list[dict], **chat_configs) -> dict: # rsp = await self.llm.ChatCompletion.acreate(**self._func_configs(messages, **chat_configs)) # self._update_costs(rsp.get("usage")) # return rsp - # + # # def _process_message(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: # """convert messages to list[dict].""" # if isinstance(messages, list): # messages = [Message(msg) if isinstance(msg, str) else msg for msg in messages] # return [msg if isinstance(msg, dict) else msg.to_dict() for msg in messages] - # + # # if isinstance(messages, Message): # messages = [messages.to_dict()] # elif isinstance(messages, str): @@ -155,14 +156,14 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # f"Only support messages type are: str, Message, list[dict], but got {type(messages).__name__}!" # ) # return messages - # + # # def ask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: # """Use function of tools to ask a code. - # + # # Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create - # + # # Examples: - # + # # >>> llm = OpenAIGPTAPI() # >>> llm.ask_code("Write a python hello world code.") # {'language': 'python', 'code': "print('Hello, World!')"} @@ -173,14 +174,14 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # messages = self._process_message(messages) # rsp = self._chat_completion_function(messages, **kwargs) # return self.get_choice_function_arguments(rsp) - # + # # async def aask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: # """Use function of tools to ask a code. - # + # # Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create - # + # # Examples: - # + # # >>> llm = OpenAIGPTAPI() # >>> rsp = await llm.ask_code("Write a python hello world code.") # >>> rsp @@ -191,7 +192,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # messages = self._process_message(messages) # rsp = await self._achat_completion_function(messages, **kwargs) # return self.get_choice_function_arguments(rsp) - # + # # def _calc_usage(self, messages: list[dict], rsp: str) -> dict: # usage = {} # if CONFIG.calc_usage: @@ -205,23 +206,23 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # logger.error("usage calculation failed!", e) # else: # return usage - # + # # async def acompletion_batch(self, batch: list[list[dict]]) -> list[dict]: # """Return full JSON""" # split_batches = self.split_batches(batch) # all_results = [] - # + # # for small_batch in split_batches: # logger.info(small_batch) # await self.wait_if_needed(len(small_batch)) - # + # # future = [self.acompletion(prompt) for prompt in small_batch] # results = await asyncio.gather(*future) # logger.info(results) # all_results.extend(results) - # + # # return all_results - # + # # async def acompletion_batch_text(self, batch: list[list[dict]]) -> list[str]: # """Only return plain text""" # raw_results = await self.acompletion_batch(batch) @@ -231,7 +232,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # results.append(result) # logger.info(f"Result of task {idx}: {result}") # return results - # + # # def _update_costs(self, usage: dict): # if CONFIG.calc_usage: # try: @@ -240,15 +241,15 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) # except Exception as e: # logger.error("updating costs failed!", e) - # + # # def get_costs(self) -> Costs: # return self._cost_manager.get_costs() - # + # # def get_max_tokens(self, messages: list[dict]): # if not self.auto_max_tokens: # return CONFIG.max_tokens_rsp # return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - # + # # def moderation(self, content: Union[str, list[str]]): # try: # if not content: @@ -258,11 +259,11 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # return rsp # except Exception as e: # logger.error(f"moderating failed:{e}") - # + # # def _moderation(self, content: Union[str, list[str]]): # rsp = self.llm.Moderation.create(input=content) # return rsp - # + # # async def amoderation(self, content: Union[str, list[str]]): # try: # if not content: @@ -272,7 +273,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): # return rsp # except Exception as e: # logger.error(f"moderating failed:{e}") - # + # # async def _amoderation(self, content: Union[str, list[str]]): # rsp = await self.llm.Moderation.acreate(input=content) # return rsp diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 58d04cf84..206be29d0 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -9,12 +9,13 @@ @Modified By: mashenquan, 2023/12/1. Fix bug: Unclosed connection caused by openai 0.x. """ -from typing import Union -from openai import APIConnectionError, AsyncAzureOpenAI, AsyncOpenAI, RateLimitError -from openai.types import CompletionUsage import asyncio import time +from typing import Union + import openai +from openai import APIConnectionError, AsyncAzureOpenAI, AsyncOpenAI, RateLimitError +from openai.types import CompletionUsage from tenacity import ( after_log, retry, @@ -22,9 +23,10 @@ from tenacity import ( stop_after_attempt, wait_random_exponential, ) + from metagpt.config import CONFIG -from metagpt.llm import LLMType from metagpt.logs import logger +from metagpt.provider import LLMType from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE from metagpt.schema import Message @@ -348,4 +350,3 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text, cacheable=False) return await memory.summarize(llm=self, max_words=max_words, keep_language=keep_language) - diff --git a/metagpt/provider/zhipuai/async_sse_client.py b/metagpt/provider/zhipuai/async_sse_client.py index b819fdc63..d7168202a 100644 --- a/metagpt/provider/zhipuai/async_sse_client.py +++ b/metagpt/provider/zhipuai/async_sse_client.py @@ -3,11 +3,10 @@ # @Desc : async_sse_client to make keep the use of Event to access response # refs to `https://github.com/zhipuai/zhipuai-sdk-python/blob/main/zhipuai/utils/sse_client.py` -from zhipuai.utils.sse_client import SSEClient, Event, _FIELD_SEPARATOR +from zhipuai.utils.sse_client import _FIELD_SEPARATOR, Event, SSEClient class AsyncSSEClient(SSEClient): - async def _aread(self): data = b"" async for chunk in self._event_source: @@ -37,9 +36,7 @@ class AsyncSSEClient(SSEClient): # Ignore unknown fields. if field not in event.__dict__: - self._logger.debug( - "Saw invalid field %s while parsing " "Server Side Event", field - ) + self._logger.debug("Saw invalid field %s while parsing " "Server Side Event", field) continue if len(data) > 1: diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 206f0dab9..82513f83c 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -2,8 +2,12 @@ # -*- coding: utf-8 -*- # @Desc : zhipuai LLM from https://open.bigmodel.cn/dev/api#sdk -from enum import Enum import json +from enum import Enum + +import openai +import zhipuai +from requests import ConnectionError from tenacity import ( after_log, retry, @@ -11,16 +15,13 @@ from tenacity import ( stop_after_attempt, wait_random_exponential, ) -from requests import ConnectionError - -import openai -import zhipuai from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.provider.openai_api import CostManager, log_and_reraise +from metagpt.provider.openai_api import log_and_reraise from metagpt.provider.zhipuai.zhipu_model_api import ZhiPuModelAPI +from metagpt.utils.cost_manager import CostManager class ZhiPuEvent(Enum): @@ -50,15 +51,11 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): openai.api_key = zhipuai.api_key # due to use openai sdk, set the api_key but it will't be used. def _const_kwargs(self, messages: list[dict]) -> dict: - kwargs = { - "model": self.model, - "prompt": messages, - "temperature": 0.3 - } + kwargs = {"model": self.model, "prompt": messages, "temperature": 0.3} return kwargs def _update_costs(self, usage: dict): - """ update each request's token cost """ + """update each request's token cost""" if CONFIG.calc_usage: try: prompt_tokens = int(usage.get("prompt_tokens", 0)) @@ -68,7 +65,7 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): logger.error("zhipuai updats costs failed!", e) def get_choice_text(self, resp: dict) -> str: - """ get the first text of choice from llm response """ + """get the first text of choice from llm response""" assist_msg = resp.get("data", {}).get("choices", [{"role": "error"}])[-1] assert assist_msg["role"] == "assistant" return assist_msg.get("content") @@ -129,10 +126,10 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): wait=wait_random_exponential(min=1, max=60), after=after_log(logger, logger.level("WARNING").name), retry=retry_if_exception_type(ConnectionError), - retry_error_callback=log_and_reraise + retry_error_callback=log_and_reraise, ) async def acompletion_text(self, messages: list[dict], stream=False) -> str: - """ response in async with stream or non-stream mode """ + """response in async with stream or non-stream mode""" if stream: return await self._achat_completion_stream(messages) resp = await self._achat_completion(messages) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index e1ab3b06b..4f7f0b796 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -16,19 +16,13 @@ @Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results of SummarizeCode. """ -<<<<<<< HEAD from __future__ import annotations import json from collections import defaultdict -======= -import asyncio -from collections import OrderedDict ->>>>>>> send18/dev from pathlib import Path from typing import Set -<<<<<<< HEAD from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.fix_bug import FixBug from metagpt.actions.summarize_code import SummarizeCode @@ -49,18 +43,6 @@ from metagpt.schema import ( Message, ) from metagpt.utils.common import any_to_name, any_to_str, any_to_str_set -======= -import aiofiles - -from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks -from metagpt.config import CONFIG -from metagpt.logs import logger -from metagpt.roles import Role -from metagpt.schema import Message -from metagpt.utils.common import CodeParser -from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP - ->>>>>>> send18/dev IS_PASS_PROMPT = """ {context} @@ -85,7 +67,6 @@ class Engineer(Role): use_code_review (bool): Whether to use code review. """ -<<<<<<< HEAD def __init__( self, name: str = "Alex", @@ -96,18 +77,6 @@ class Engineer(Role): use_code_review: bool = False, ) -> None: """Initializes the Engineer role with given attributes.""" -======= -class Engineer(Role): - def __init__( - self, - name="Alex", - profile="Engineer", - goal="Write elegant, readable, extensible, efficient code", - constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", - n_borg=1, - use_code_review=False, - ): ->>>>>>> send18/dev super().__init__(name, profile, goal, constraints) self.use_code_review = use_code_review self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview, FixBug]) @@ -121,7 +90,6 @@ class Engineer(Role): m = json.loads(task_msg.content) return m.get("Task list") -<<<<<<< HEAD async def _act_sp_with_cr(self, review=False) -> Set[str]: changed_files = set() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) @@ -145,83 +113,8 @@ class Engineer(Role): msg = Message( content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode ) -======= - @classmethod - def parse_tasks(self, task_msg: Message) -> list[str]: - if task_msg.instruct_content: - return task_msg.instruct_content.dict().get("Task list") - return CodeParser.parse_file_list(block="Task list", text=task_msg.content) - - @classmethod - def parse_code(self, code_text: str) -> str: - return CodeParser.parse_code(block="", text=code_text) - - @classmethod - def parse_workspace(cls, system_design_msg: Message) -> str: - if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"') - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - - def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] - if not msg: - return CONFIG.workspace / "src" - workspace = self.parse_workspace(msg) - # Codes are written in workspace/{package_name}/{package_name} - return CONFIG.workspace / workspace - - async def write_file(self, filename: str, code: str): - workspace = self.get_workspace() - filename = filename.replace('"', "").replace("\n", "") - file = workspace / filename - file.parent.mkdir(parents=True, exist_ok=True) - async with aiofiles.open(file, "w") as f: - await f.write(code) - return file - - def recv(self, message: Message) -> None: - self._rc.memory.add(message) - if message in self._rc.important_memory: - self.todos = self.parse_tasks(message) - - async def _act_mp(self) -> Message: - # self.recreate_workspace() - todo_coros = [] - for todo in self.todos: - todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo - ) - todo_coros.append(todo_coro) - - rsps = await gather_ordered_k(todo_coros, self.n_borg) - for todo, code_rsp in zip(self.todos, rsps): - _ = self.parse_code(code_rsp) - logger.info(todo) - logger.info(code_rsp) - # self.write_file(todo, code) - msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) - del self.todos[0] - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) - return msg - - async def _act_sp(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later - instruct_content = {} - for todo in self.todos: - code = await WriteCode().run(context=self._rc.history, filename=todo) - # logger.info(todo) - # logger.info(code_rsp) - # code = self.parse_code(code_rsp) - file_path = await self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo)) ->>>>>>> send18/dev - self._rc.memory.add(msg) - instruct_content[todo] = code - -<<<<<<< HEAD changed_files.add(coding_context.code_doc.filename) if not changed_files: logger.info("Nothing has changed.") @@ -247,22 +140,8 @@ class Engineer(Role): cause_by=WriteCodeReview if self.use_code_review else WriteCode, send_to=self, sent_from=self, -======= - # code_msg = todo + FILENAME_CODE_SEP + str(file_path) - code_msg = (todo, file_path) - code_msg_all.append(code_msg) - - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message( - content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all), - instruct_content=instruct_content, - role=self.profile, - cause_by=type(self._rc.todo), - send_to="QaEngineer", ->>>>>>> send18/dev ) -<<<<<<< HEAD async def _act_summarize(self): code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO) code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO) @@ -353,49 +232,6 @@ class Engineer(Role): async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_repo, dependency): context = await Engineer._new_coding_context( filename, src_file_repo, task_file_repo, design_file_repo, dependency -======= - async def _act_sp_precision(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later - instruct_content = {} - for todo in self.todos: - """ - # 从历史信息中挑选必须的信息，以减少prompt长度（人工经验总结） - 1. Architect全部 - 2. ProjectManager全部 - 3. 是否需要其他代码（暂时需要）？ - TODO:目标是不需要。在任务拆分清楚后，根据设计思路，不需要其他代码也能够写清楚单个文件，如果不能则表示还需要在定义的更清晰，这个是代码能够写长的关键 - """ - context = [] - msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) - for m in msg: - context.append(m.content) - context_str = "\n".join(context) - # 编写code - code = await WriteCode().run(context=context_str, filename=todo) - # code review - if self.use_code_review: - try: - rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) - code = rewrite_code - except Exception as e: - logger.error("code review failed!", e) - pass - file_path = await self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode) - self._rc.memory.add(msg) - instruct_content[todo] = code - - code_msg = (todo, file_path) - code_msg_all.append(code_msg) - - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message( - content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all), - instruct_content=instruct_content, - role=self.profile, - cause_by=type(self._rc.todo), - send_to="QaEngineer", ->>>>>>> send18/dev ) coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json()) return coding_doc diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index c8bca8c42..c1573e63b 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -14,10 +14,7 @@ @Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results of SummarizeCode. """ -<<<<<<< HEAD -from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest - -# from metagpt.const import WORKSPACE_ROOT +from metagpt.actions import DebugError, RunCode, WriteTest from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( @@ -25,13 +22,6 @@ from metagpt.const import ( TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO, ) -======= -import os -from pathlib import Path - -from metagpt.actions import DebugError, RunCode, WriteCode, WriteDesign, WriteTest -from metagpt.config import CONFIG ->>>>>>> send18/dev from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext @@ -55,32 +45,6 @@ class QaEngineer(Role): self.test_round = 0 self.test_round_allowed = test_round_allowed -<<<<<<< HEAD -======= - @classmethod - def parse_workspace(cls, system_design_msg: Message) -> str: - if not system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name") - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - - def get_workspace(self, return_proj_dir=True) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] - if not msg: - return CONFIG.workspace / "src" - workspace = self.parse_workspace(msg) - # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. - if return_proj_dir: - return CONFIG.workspace / workspace - # development codes directory: workspace/{package_name}/{package_name} - return CONFIG.workspace / workspace / workspace - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() / "tests" - file = workspace / filename - file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) - ->>>>>>> send18/dev async def _write_test(self, message: Message) -> None: src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) changed_files = set(src_file_repo.changed_files.keys()) diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 576e57969..d13d43495 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -1,16 +1,10 @@ #!/usr/bin/env python """ -<<<<<<< HEAD +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. @Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ -======= -@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. - -""" ->>>>>>> send18/dev - import asyncio from pydantic import BaseModel @@ -47,8 +41,6 @@ class Researcher(Role): if language not in ("en-us", "zh-cn"): logger.warning(f"The language `{language}` has not been tested, it may not work.") -<<<<<<< HEAD -======= async def _think(self) -> bool: if self._rc.todo is None: self._set_state(0) @@ -60,7 +52,6 @@ class Researcher(Role): self._rc.todo = None return False ->>>>>>> send18/dev async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") todo = self._rc.todo diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 9f2cb7753..1f28e3c57 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,7 +4,7 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py -<<<<<<< HEAD +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. @Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: 1. Merge the `recv` functionality into the `_observe` function. Future message reading operations will be consolidated within the `_observe` function. @@ -18,10 +18,6 @@ only. In the normal workflow, you should use `publish_message` or `put_message` to transmit messages. @Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing functionality is to be consolidated into the `Environment` class. -======= -@Modified By: mashenquan, 2023-8-7, Support template-style variables, such as '{teaching_language} Teacher'. -@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. ->>>>>>> send18/dev """ from __future__ import annotations @@ -31,20 +27,11 @@ from typing import Iterable, Set, Type from pydantic import BaseModel, Field from metagpt.actions import Action, ActionOutput -from metagpt.config import CONFIG -<<<<<<< HEAD from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger from metagpt.memory import Memory from metagpt.schema import Message, MessageQueue from metagpt.utils.common import any_to_name, any_to_str -======= -from metagpt.const import OPTIONS -from metagpt.llm import LLMFactory -from metagpt.logs import logger -from metagpt.memory import LongTermMemory, Memory -from metagpt.schema import Message, MessageTag ->>>>>>> send18/dev PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -87,11 +74,7 @@ class RoleReactMode(str, Enum): class RoleSetting(BaseModel): -<<<<<<< HEAD - """Role Settings""" -======= """Role properties""" ->>>>>>> send18/dev name: str profile: str @@ -108,16 +91,10 @@ class RoleSetting(BaseModel): class RoleContext(BaseModel): -<<<<<<< HEAD """Role Runtime Context""" env: "Environment" = Field(default=None) msg_buffer: MessageQueue = Field(default_factory=MessageQueue) # Message Buffer with Asynchronous Updates -======= - """Runtime role context""" - - env: "Environment" = Field(default=None) ->>>>>>> send18/dev memory: Memory = Field(default_factory=Memory) # long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None @@ -133,34 +110,22 @@ class RoleContext(BaseModel): arbitrary_types_allowed = True def check(self, role_id: str): - if CONFIG.long_term_memory: - self.long_term_memory.recover_memory(role_id, self) - self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation + # if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: + # self.long_term_memory.recover_memory(role_id, self) + # self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation + pass @property def important_memory(self) -> list[Message]: -<<<<<<< HEAD - """Get the information corresponding to the watched actions""" -======= """Retrieve information corresponding to the attention action.""" ->>>>>>> send18/dev return self.memory.get_by_actions(self.watch) @property def history(self) -> list[Message]: return self.memory.get() - @property - def prerequisite(self): - """Retrieve information with `prerequisite` tag""" - if self.memory and hasattr(self.memory, "get_by_tags"): - vv = self.memory.get_by_tags([MessageTag.Prerequisite.value]) - return vv[-1:] if len(vv) > 1 else vv - return [] - class Role: -<<<<<<< HEAD """Role/Agent""" def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False): @@ -168,20 +133,6 @@ class Role: self._setting = RoleSetting( name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, is_human=is_human ) -======= - """Role/Proxy""" - - def __init__(self, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): - # Replace template-style variables, such as '{teaching_language} Teacher'. - name = Role.format_value(name) - profile = Role.format_value(profile) - goal = Role.format_value(goal) - constraints = Role.format_value(constraints) - desc = Role.format_value(desc) - - self._llm = LLMFactory.new_llm() - self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) ->>>>>>> send18/dev self._states = [] self._actions = [] self._role_id = str(self._setting) @@ -258,12 +209,8 @@ class Role: self._rc.todo = self._actions[self._rc.state] if state >= 0 else None def set_env(self, env: "Environment"): -<<<<<<< HEAD """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" -======= - """设置角色工作所处的环境，角色可以向环境说话，也可以通过观察接受环境消息""" ->>>>>>> send18/dev self._rc.env = env if env: env.set_subscription(self, self._subscription) @@ -275,7 +222,6 @@ class Role: @property def name(self): -<<<<<<< HEAD """Get virtual user name""" return self._setting.name @@ -283,9 +229,6 @@ class Role: def subscription(self) -> Set: """The labels for messages to be consumed by the Role object.""" return self._subscription -======= - """Return role `name`, read only""" - return self._setting.name @property def desc(self): @@ -306,7 +249,6 @@ class Role: def action_count(self): """Return number of action""" return len(self._actions) ->>>>>>> send18/dev def _get_prefix(self): """Get the role prefix""" @@ -314,20 +256,14 @@ class Role: return self._setting.desc return PREFIX_TEMPLATE.format(**self._setting.dict()) -<<<<<<< HEAD - async def _think(self) -> None: - """Think about what to do and decide on the next action""" -======= async def _think(self) -> bool: """Consider what to do and decide on the next course of action. Return false if nothing can be done.""" ->>>>>>> send18/dev if len(self._actions) == 1: # If there is only one action, then only this one can be performed self._set_state(0) return True prompt = self._get_prefix() prompt += STATE_TEMPLATE.format( -<<<<<<< HEAD history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1, @@ -344,49 +280,27 @@ class Role: if next_state == -1: logger.info(f"End actions with {next_state=}") self._set_state(next_state) -======= - history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1 - ) - next_state = await self._llm.aask(prompt) - logger.debug(f"{prompt=}") - if not next_state.isdigit() or int(next_state) not in range(len(self._states)): - logger.warning(f"Invalid answer of state, {next_state=}") - next_state = "0" - self._set_state(int(next_state)) return True ->>>>>>> send18/dev async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") -<<<<<<< HEAD response = await self._rc.todo.run(self._rc.important_memory) -======= - requirement = self._rc.important_memory or self._rc.prerequisite - response = await self._rc.todo.run(requirement) - # logger.info(response) ->>>>>>> send18/dev if isinstance(response, ActionOutput): msg = Message( content=response.content, instruct_content=response.instruct_content, role=self.profile, -<<<<<<< HEAD cause_by=self._rc.todo, sent_from=self, ) elif isinstance(response, Message): msg = response -======= - cause_by=type(self._rc.todo), - ) ->>>>>>> send18/dev else: msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self) self._rc.memory.add(msg) return msg -<<<<<<< HEAD async def _observe(self, ignore_memory=False) -> int: """Prepare new messages for processing from the message buffer and other sources.""" # Read unprocessed messages from the msg buffer. @@ -400,21 +314,6 @@ class Role: # Design Rules: # If you need to further categorize Message objects, you can do so using the Message.set_meta function. # msg_buffer is a receiving buffer, avoid adding message data and operations to msg_buffer. -======= - async def _observe(self) -> int: - """从环境中观察，获得重要信息，并加入记忆""" - if not self._rc.env: - return 0 - env_msgs = self._rc.env.memory.get() - - observed = self._rc.env.memory.get_by_actions(self._rc.watch) - - self._rc.news = self._rc.memory.remember(observed) # remember recent exact or similar memories - - for i in env_msgs: - self.recv(i) - ->>>>>>> send18/dev news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news] if news_text: logger.debug(f"{self._setting} observed: {news_text}") @@ -505,36 +404,10 @@ class Role: self.publish_message(rsp) return rsp -<<<<<<< HEAD @property def is_idle(self) -> bool: """If true, all actions have been executed.""" return not self._rc.news and not self._rc.todo and self._rc.msg_buffer.empty() -======= - @staticmethod - def format_value(value): - """Fill parameters inside `value` with `options`.""" - if not isinstance(value, str): - return value - if "{" not in value: - return value - - merged_opts = OPTIONS.get() or {} - try: - return value.format(**merged_opts) - except KeyError as e: - logger.warning(f"Parameter is missing:{e}") - - for k, v in merged_opts.items(): - value = value.replace("{" + f"{k}" + "}", str(v)) - return value - - def add_action(self, act): - self._actions.append(act) - - def add_to_do(self, act): - self._rc.todo = act ->>>>>>> send18/dev async def think(self) -> Action: """The exported `think` function""" @@ -547,16 +420,7 @@ class Role: return ActionOutput(content=msg.content, instruct_content=msg.instruct_content) @property -<<<<<<< HEAD def todo(self) -> str: if self._actions: return any_to_name(self._actions[0]) return "" -======= - def todo_description(self): - if not self._rc or not self._rc.todo: - return "" - if self._rc.todo.desc: - return self._rc.todo.desc - return f"{type(self._rc.todo).__name__}" ->>>>>>> send18/dev diff --git a/metagpt/schema.py b/metagpt/schema.py index 70e84ff15..baed5582b 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -22,7 +22,9 @@ from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError from pathlib import Path from typing import Dict, List, Optional, Set, TypedDict + from pydantic import BaseModel, Field + from metagpt.config import CONFIG from metagpt.const import ( MESSAGE_ROUTE_CAUSE_BY, @@ -95,14 +97,14 @@ class Message(BaseModel): send_to: Set = Field(default_factory={MESSAGE_ROUTE_TO_ALL}) def __init__( - self, - content, - instruct_content=None, - role="user", - cause_by="", - sent_from="", - send_to=MESSAGE_ROUTE_TO_ALL, - **kwargs, + self, + content, + instruct_content=None, + role="user", + cause_by="", + sent_from="", + send_to=MESSAGE_ROUTE_TO_ALL, + **kwargs, ): """ Parameters not listed below will be stored as meta info, including custom parameters. @@ -343,4 +345,3 @@ class CodeSummarizeContext(BaseModel): class BugFixContext(BaseModel): filename: str = "" - diff --git a/metagpt/team.py b/metagpt/team.py index 91587655f..6a3fae0d9 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -45,8 +45,9 @@ class Team(BaseModel): @staticmethod def _check_balance(): if CONFIG.cost_manager.total_cost > CONFIG.cost_manager.max_budget: - raise NoMoneyException(CONFIG.cost_manager.total_cost, - f'Insufficient funds: {CONFIG.cost_manager.max_budget}') + raise NoMoneyException( + CONFIG.cost_manager.total_cost, f"Insufficient funds: {CONFIG.cost_manager.max_budget}" + ) def run_project(self, idea, send_to: str = ""): """Start a project from publishing user requirement.""" diff --git a/metagpt/tools/__init__.py b/metagpt/tools/__init__.py index a148bb744..aab8c990c 100644 --- a/metagpt/tools/__init__.py +++ b/metagpt/tools/__init__.py @@ -24,6 +24,6 @@ class WebBrowserEngineType(Enum): CUSTOM = "custom" @classmethod - def _missing_(cls, key): - """缺省类型转换""" + def __missing__(cls, key): + """Default type conversion""" return cls.CUSTOM diff --git a/metagpt/tools/hello.py b/metagpt/tools/hello.py index 2eb4c31f0..8a21e1b4e 100644 --- a/metagpt/tools/hello.py +++ b/metagpt/tools/hello.py @@ -22,6 +22,6 @@ async def post_greeting(name: str) -> str: if __name__ == "__main__": - app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') + app = connexion.AioHttpApp(__name__, specification_dir="../../.well-known/") app.add_api("openapi.yaml", arguments={"title": "Hello World Example"}) app.run(port=8080) diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index c5a0b872f..50c0edcba 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -8,18 +8,13 @@ """ import asyncio import base64 -import os -import sys -from pathlib import Path -from typing import List, Dict +from typing import Dict, List import aiohttp import requests from pydantic import BaseModel from metagpt.config import CONFIG, Config - -sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger @@ -38,9 +33,7 @@ class MetaGPTText2Image: :return: The image data is returned in Base64 encoding. """ - headers = { - "Content-Type": "application/json" - } + headers = {"Content-Type": "application/json"} dims = size_type.split("x") data = { "prompt": text, diff --git a/metagpt/tools/openai_text_to_embedding.py b/metagpt/tools/openai_text_to_embedding.py index 86b58d71f..fb6fbc653 100644 --- a/metagpt/tools/openai_text_to_embedding.py +++ b/metagpt/tools/openai_text_to_embedding.py @@ -8,26 +8,23 @@ For more details, checkout: `https://platform.openai.com/docs/api-reference/embeddings/object` """ import asyncio -import os -from pathlib import Path from typing import List import aiohttp import requests from pydantic import BaseModel -import sys from metagpt.config import CONFIG, Config - -sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger class Embedding(BaseModel): """Represents an embedding vector returned by embedding endpoint.""" + object: str # The object type, which is always "embedding". embedding: List[ - float] # The embedding vector, which is a list of floats. The length of vector depends on the model as listed in the embedding guide. + float + ] # The embedding vector, which is a list of floats. The length of vector depends on the model as listed in the embedding guide. index: int # The index of the embedding in the list of embeddings. @@ -58,10 +55,7 @@ class OpenAIText2Embedding: :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. """ - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.openai_api_key}" - } + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.openai_api_key}"} data = {"input": text, "model": model} try: async with aiohttp.ClientSession() as session: diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index 479f83c63..c4d9d2df4 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -6,21 +6,14 @@ import asyncio import base64 import io import json -import os from os.path import join from typing import List from aiohttp import ClientSession from PIL import Image, PngImagePlugin -<<<<<<< HEAD from metagpt.config import CONFIG -======= -from metagpt.config import Config -from metagpt.logs import logger ->>>>>>> send18/dev - -# from metagpt.const import WORKSPACE_ROOT +from metagpt.const import SD_OUTPUT_FILE_REPO from metagpt.logs import logger payload = { @@ -84,14 +77,10 @@ class SDEngine: return self.payload def _save(self, imgs, save_name=""): -<<<<<<< HEAD - save_dir = CONFIG.workspace_path / "resources" / "SD_Output" -======= - save_dir = CONFIG.get_workspace() / "resources" / "SD_Output" ->>>>>>> send18/dev - if not os.path.exists(save_dir): - os.makedirs(save_dir, exist_ok=True) - batch_decode_base64_to_image(imgs, save_dir, save_name=save_name) + save_dir = CONFIG.workspace_path / SD_OUTPUT_FILE_REPO + if not save_dir.exists(): + save_dir.mkdir(parents=True, exist_ok=True) + batch_decode_base64_to_image(imgs, str(save_dir), save_name=save_name) async def run_t2i(self, prompts: List): # Asynchronously run the SD API for multiple prompts diff --git a/metagpt/tools/web_browser_engine.py b/metagpt/tools/web_browser_engine.py index 1f1a5ec67..cda137cbd 100644 --- a/metagpt/tools/web_browser_engine.py +++ b/metagpt/tools/web_browser_engine.py @@ -20,16 +20,16 @@ class WebBrowserEngine: engine: WebBrowserEngineType | None = None, run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None, ): - engine = engine or options.get("web_browser_engine") + engine = engine or CONFIG.web_browser_engine if engine is None: raise NotImplementedError if WebBrowserEngineType(engine) is WebBrowserEngineType.PLAYWRIGHT: module = "metagpt.tools.web_browser_engine_playwright" - run_func = importlib.import_module(module).PlaywrightWrapper(options=options).run + run_func = importlib.import_module(module).PlaywrightWrapper().run elif WebBrowserEngineType(engine) is WebBrowserEngineType.SELENIUM: module = "metagpt.tools.web_browser_engine_selenium" - run_func = importlib.import_module(module).SeleniumWrapper(options=options).run + run_func = importlib.import_module(module).SeleniumWrapper().run elif WebBrowserEngineType(engine) is WebBrowserEngineType.CUSTOM: run_func = run_func else: @@ -53,8 +53,6 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, engine_type: Literal["playwright", "selenium"] = "playwright", **kwargs): - return await WebBrowserEngine(options=CONFIG.options, engine=WebBrowserEngineType(engine_type), **kwargs).run( - url, *urls - ) + return await WebBrowserEngine(engine=WebBrowserEngineType(engine_type), **kwargs).run(url, *urls) fire.Fire(main) diff --git a/metagpt/tools/web_browser_engine_selenium.py b/metagpt/tools/web_browser_engine_selenium.py index b0fcb3fe1..51d26e551 100644 --- a/metagpt/tools/web_browser_engine_selenium.py +++ b/metagpt/tools/web_browser_engine_selenium.py @@ -9,13 +9,13 @@ import asyncio import importlib from concurrent import futures from copy import deepcopy -from typing import Literal, Dict +from typing import Dict, Literal from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait -from metagpt.config import Config +from metagpt.config import CONFIG from metagpt.utils.parse_html import WebPage @@ -41,11 +41,11 @@ class SeleniumWrapper: executor: futures.Executor | None = None, ) -> None: if browser_type is None: - browser_type = options.get("selenium_browser_type") + browser_type = CONFIG.selenium_browser_type self.browser_type = browser_type launch_kwargs = launch_kwargs or {} - if options.get("global_proxy") and "proxy-server" not in launch_kwargs: - launch_kwargs["proxy-server"] = options.get("global_proxy") + if CONFIG.global_proxy and "proxy-server" not in launch_kwargs: + launch_kwargs["proxy-server"] = CONFIG.global_proxy self.executable_path = launch_kwargs.pop("executable_path", None) self.launch_args = [f"--{k}={v}" for k, v in launch_kwargs.items()] @@ -123,8 +123,6 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, browser_type: str = "chrome", **kwargs): - return await SeleniumWrapper(options=Config().runtime_options, - browser_type=browser_type, - **kwargs).run(url, *urls) + return await SeleniumWrapper(browser_type=browser_type, **kwargs).run(url, *urls) fire.Fire(main) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index b627316cd..57aba463c 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -18,10 +18,9 @@ import os import platform import re from typing import List, Tuple, Union + +from metagpt.config import CONFIG from metagpt.const import MESSAGE_ROUTE_TO_ALL -from pathlib import Path -from typing import List, Tuple -import yaml from metagpt.logs import logger @@ -186,7 +185,7 @@ class OutputParser: if start_index != -1 and end_index != -1: # Extract the structure part - structure_text = text[start_index: end_index + 1] + structure_text = text[start_index : end_index + 1] try: # Attempt to convert the text to a Python data type using ast.literal_eval @@ -371,3 +370,21 @@ def any_to_name(val): :return: The name of the value. """ return any_to_str(val).split(".")[-1] + + +def format_value(value): + """Fill parameters inside `value` with `options`.""" + if not isinstance(value, str): + return value + if "{" not in value: + return value + + merged_opts = CONFIG.options or {} + try: + return value.format(**merged_opts) + except KeyError as e: + logger.warning(f"Parameter is missing:{e}") + + for k, v in merged_opts.items(): + value = value.replace("{" + f"{k}" + "}", str(v)) + return value diff --git a/metagpt/utils/cost_manager.py b/metagpt/utils/cost_manager.py index f0fea44ce..ce53f2285 100644 --- a/metagpt/utils/cost_manager.py +++ b/metagpt/utils/cost_manager.py @@ -6,10 +6,12 @@ @Desc : mashenquan, 2023/8/28. Separate the `CostManager` class to support user-level cost accounting. """ +from typing import NamedTuple + from pydantic import BaseModel + from metagpt.logs import logger from metagpt.utils.token_counter import TOKEN_COSTS -from typing import NamedTuple class Costs(NamedTuple): @@ -39,8 +41,9 @@ class CostManager(BaseModel): """ self.total_prompt_tokens += prompt_tokens self.total_completion_tokens += completion_tokens - cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model][ - "completion"]) / 1000 + cost = ( + prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model]["completion"] + ) / 1000 self.total_cost += cost logger.info( f"Total running cost: ${self.total_cost:.3f} | Max budget: ${self.max_budget:.3f} | " diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 9827b8252..1340b1768 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -8,13 +8,15 @@ """ from __future__ import annotations -from gitignore_parser import parse_gitignore, rule_from_pattern, handle_negation import shutil from enum import Enum from pathlib import Path from typing import Dict, List + from git.repo import Repo from git.repo.fun import is_git_dir +from gitignore_parser import parse_gitignore + from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.dependency_file import DependencyFile @@ -236,8 +238,9 @@ class GitRepository: rpath = file_path.relative_to(root_relative_path) files.append(str(rpath)) else: - subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path, - filter_ignored=False) + subfolder_files = self.get_files( + relative_path=file_path, root_relative_path=root_relative_path, filter_ignored=False + ) files.extend(subfolder_files) except Exception as e: logger.error(f"Error: {e}") diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index bf7e6c4a7..3fa7ab79a 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -7,22 +7,15 @@ @Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import asyncio -<<<<<<< HEAD import os from pathlib import Path -from metagpt.config import CONFIG -from metagpt.const import METAGPT_ROOT -======= -from pathlib import Path - -# from metagpt.utils.common import check_cmd_exists import aiofiles -from metagpt.config import CONFIG, Config -from metagpt.const import PROJECT_ROOT ->>>>>>> send18/dev +from metagpt.config import CONFIG +from metagpt.const import METAGPT_ROOT from metagpt.logs import logger +from metagpt.utils.common import check_cmd_exists async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: @@ -43,7 +36,6 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, await f.write(mermaid_code) # tmp.write_text(mermaid_code, encoding="utf-8") -<<<<<<< HEAD engine = CONFIG.mermaid_engine.lower() if engine == "nodejs": if check_cmd_exists(CONFIG.mmdc) != 0: @@ -100,25 +92,6 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, logger.warning(f"Unsupported mermaid engine: {engine}") return 0 -======= - # if check_cmd_exists("mmdc") != 0: - # logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") - # return -1 - - # for suffix in ["pdf", "svg", "png"]: - for suffix in ["png"]: - output_file = f"{output_file_without_suffix}.{suffix}" - # Call the `mmdc` command to convert the Mermaid code to a PNG - logger.info(f"Generating {output_file}..") - cmds = [CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)] - - if CONFIG.puppeteer_config: - cmds.extend(["-p", CONFIG.puppeteer_config]) - process = await asyncio.create_subprocess_exec(*cmds) - await process.wait() - return process.returncode ->>>>>>> send18/dev - if __name__ == "__main__": MMC1 = """classDiagram @@ -171,22 +144,7 @@ if __name__ == "__main__": S-->>SE: return summary SE-->>M: return summary""" -<<<<<<< HEAD -if __name__ == "__main__": loop = asyncio.new_event_loop() result = loop.run_until_complete(mermaid_to_file(MMC1, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) - result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/2")) loop.close() -======= - conf = Config() - asyncio.run( - mermaid_to_file( - options=conf.runtime_options, mermaid_code=MMC1, output_file_without_suffix=PROJECT_ROOT / "tmp/1.png" - ) - ) - asyncio.run( - mermaid_to_file( - options=conf.runtime_options, mermaid_code=MMC2, output_file_without_suffix=PROJECT_ROOT / "tmp/2.png" - ) - ) ->>>>>>> send18/dev diff --git a/tests/conftest.py b/tests/conftest.py index 2709b38ae..375b9ff7f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,9 @@ import asyncio import logging import re from unittest.mock import Mock + import pytest + from metagpt.config import CONFIG from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger @@ -95,7 +97,7 @@ def setup_and_teardown_git_repo(request): # Register the function for destroying the environment. request.addfinalizer(fin) + @pytest.fixture(scope="session", autouse=True) def init_config(): Config() - diff --git a/tests/metagpt/actions/test_ui_design.py b/tests/metagpt/actions/test_ui_design.py index b9c91d21f..83590ec7d 100644 --- a/tests/metagpt/actions/test_ui_design.py +++ b/tests/metagpt/actions/test_ui_design.py @@ -101,7 +101,6 @@ body { """ - def test_ui_design_parse_css(): ui_design_work = UIDesign(name="UI design action") diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 0bd6633cd..73f3a6dcf 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -7,9 +7,10 @@ @Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM + from metagpt.actions.write_code import WriteCode from metagpt.logs import logger +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM from metagpt.schema import CodingContext, Document from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE diff --git a/tests/metagpt/actions/test_write_teaching_plan.py b/tests/metagpt/actions/test_write_teaching_plan.py index 6754fe88c..3f25b2167 100644 --- a/tests/metagpt/actions/test_write_teaching_plan.py +++ b/tests/metagpt/actions/test_write_teaching_plan.py @@ -8,8 +8,9 @@ import asyncio from typing import Optional -from pydantic import BaseModel + from langchain.llms.base import LLM +from pydantic import BaseModel from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart from metagpt.config import Config @@ -17,7 +18,7 @@ from metagpt.schema import Message class MockWriteTeachingPlanPart(WriteTeachingPlanPart): - def __init__(self, options, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): + def __init__(self, options, name: str = "", context=None, llm: LLM = None, topic="", language="Chinese"): super().__init__(options, name, context, llm, topic, language) async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> str: @@ -32,18 +33,8 @@ async def mock_write_teaching_plan_part(): language: str inputs = [ - { - "input": "AABBCC", - "name": "A", - "topic": WriteTeachingPlanPart.COURSE_TITLE, - "language": "C" - }, - { - "input": "DDEEFFF", - "name": "A1", - "topic": "B1", - "language": "C1" - } + {"input": "AABBCC", "name": "A", "topic": WriteTeachingPlanPart.COURSE_TITLE, "language": "C"}, + {"input": "DDEEFFF", "name": "A1", "topic": "B1", "language": "C1"}, ] for i in inputs: @@ -63,5 +54,5 @@ def test_suite(): loop.run_until_complete(task) -if __name__ == '__main__': +if __name__ == "__main__": test_suite() diff --git a/tests/metagpt/learn/test_text_to_embedding.py b/tests/metagpt/learn/test_text_to_embedding.py index d81a8ac1c..e3d20a759 100644 --- a/tests/metagpt/learn/test_text_to_embedding.py +++ b/tests/metagpt/learn/test_text_to_embedding.py @@ -19,9 +19,7 @@ async def mock_text_to_embedding(): class Input(BaseModel): input: str - inputs = [ - {"input": "Panda emoji"} - ] + inputs = [{"input": "Panda emoji"}] for i in inputs: seed = Input(**i) @@ -36,5 +34,5 @@ def test_suite(): loop.run_until_complete(task) -if __name__ == '__main__': +if __name__ == "__main__": test_suite() diff --git a/tests/metagpt/learn/test_text_to_image.py b/tests/metagpt/learn/test_text_to_image.py index c359797de..982a39b13 100644 --- a/tests/metagpt/learn/test_text_to_image.py +++ b/tests/metagpt/learn/test_text_to_image.py @@ -19,9 +19,7 @@ async def mock_text_to_image(): input: str size_type: str - inputs = [ - {"input": "Panda emoji", "size_type": "512x512"} - ] + inputs = [{"input": "Panda emoji", "size_type": "512x512"}] for i in inputs: seed = Input(**i) @@ -31,7 +29,7 @@ async def mock_text_to_image(): flags = ";base64," assert flags in base64_data ix = base64_data.find(flags) + len(flags) - declaration = base64_data[0: ix] + declaration = base64_data[0:ix] assert declaration data = base64_data[ix:] assert data @@ -44,5 +42,5 @@ def test_suite(): loop.run_until_complete(task) -if __name__ == '__main__': +if __name__ == "__main__": test_suite() diff --git a/tests/metagpt/learn/test_text_to_speech.py b/tests/metagpt/learn/test_text_to_speech.py index 68de5a3b2..42b6839fa 100644 --- a/tests/metagpt/learn/test_text_to_speech.py +++ b/tests/metagpt/learn/test_text_to_speech.py @@ -18,9 +18,7 @@ async def mock_text_to_speech(): class Input(BaseModel): input: str - inputs = [ - {"input": "Panda emoji"} - ] + inputs = [{"input": "Panda emoji"}] for i in inputs: seed = Input(**i) @@ -30,7 +28,7 @@ async def mock_text_to_speech(): flags = ";base64," assert flags in base64_data ix = base64_data.find(flags) + len(flags) - declaration = base64_data[0: ix] + declaration = base64_data[0:ix] assert declaration data = base64_data[ix:] assert data @@ -43,5 +41,5 @@ def test_suite(): loop.run_until_complete(task) -if __name__ == '__main__': - test_suite() \ No newline at end of file +if __name__ == "__main__": + test_suite() diff --git a/tests/metagpt/memory/test_brain_memory.py b/tests/metagpt/memory/test_brain_memory.py index b5fc942ca..2f2a984d8 100644 --- a/tests/metagpt/memory/test_brain_memory.py +++ b/tests/metagpt/memory/test_brain_memory.py @@ -21,14 +21,7 @@ def test_json(): knowledge: List[str] stack: List[str] - inputs = [ - { - "history": ["a", "b"], - "solution": ["c"], - "knowledge": ["d", "e"], - "stack": ["f"] - } - ] + inputs = [{"history": ["a", "b"], "solution": ["c"], "knowledge": ["d", "e"], "stack": ["f"]}] for i in inputs: v = Input(**i) @@ -53,5 +46,6 @@ def test_json(): msg = Message(**v) assert msg -if __name__ == '__main__': - test_json() \ No newline at end of file + +if __name__ == "__main__": + test_json() diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 8f673d6e0..82d6c7052 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -7,10 +7,9 @@ """ from typing import Dict, Optional + from pydantic import BaseModel -from metagpt.config import Config -from metagpt.provider.openai_api import CostManager from metagpt.roles.teacher import Teacher @@ -40,7 +39,7 @@ def test_init(): "expect_constraints": "Do in HaHa, CN", "kwargs": {"language": "CN", "key1": "HaHa", "something_big": "sleep", "teaching_language": "EN"}, "desc": "aaa{language}", - "expect_desc": "aaaCN" + "expect_desc": "aaaCN", }, { "name": "Lily{language}", @@ -53,17 +52,20 @@ def test_init(): "expect_constraints": "Do in {key1}, {language}", "kwargs": {}, "desc": "aaa{language}", - "expect_desc": "aaa{language}" + "expect_desc": "aaa{language}", }, ] for i in inputs: seed = Inputs(**i) - options = Config().runtime_options - cost_manager = CostManager(**options) - teacher = Teacher(options=options, cost_manager=cost_manager, name=seed.name, profile=seed.profile, - goal=seed.goal, constraints=seed.constraints, - desc=seed.desc, **seed.kwargs) + teacher = Teacher( + name=seed.name, + profile=seed.profile, + goal=seed.goal, + constraints=seed.constraints, + desc=seed.desc, + **seed.kwargs + ) assert teacher.name == seed.expect_name assert teacher.desc == seed.expect_desc assert teacher.profile == seed.expect_profile @@ -79,16 +81,8 @@ def test_new_file_name(): expect: str inputs = [ - { - "lesson_title": "# @344\n12", - "ext": ".md", - "expect": "_344_12.md" - }, - { - "lesson_title": "1#@$%!*&\\/:*?\"<>|\n\t \'1", - "ext": ".cc", - "expect": "1_1.cc" - } + {"lesson_title": "# @344\n12", "ext": ".md", "expect": "_344_12.md"}, + {"lesson_title": "1#@$%!*&\\/:*?\"<>|\n\t '1", "ext": ".cc", "expect": "1_1.cc"}, ] for i in inputs: seed = Inputs(**i) @@ -96,6 +90,6 @@ def test_new_file_name(): assert result == seed.expect -if __name__ == '__main__': +if __name__ == "__main__": test_init() test_new_file_name() diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index 29ca38f5a..933d74b97 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -9,6 +9,7 @@ """ import pytest + from metagpt.actions import UserRequirement from metagpt.environment import Environment from metagpt.logs import logger @@ -22,19 +23,16 @@ def env(): def test_add_role(env: Environment): - role = ProductManager(name="Alice", - profile="product manager", - goal="create a new product", - constraints="limited resources") + role = ProductManager( + name="Alice", profile="product manager", goal="create a new product", constraints="limited resources" + ) env.add_role(role) assert env.get_role(role.profile) == role def test_get_roles(env: Environment): - role1 = Role(name="Alice", profile="product manager", - goal="create a new product", constraints="limited resources") - role2 = Role(name="Bob", profile="engineer", - goal="develop the new product", constraints="short deadline") + role1 = Role(name="Alice", profile="product manager", goal="create a new product", constraints="limited resources") + role2 = Role(name="Bob", profile="engineer", goal="develop the new product", constraints="short deadline") env.add_role(role1) env.add_role(role2) roles = env.get_roles() @@ -43,10 +41,10 @@ def test_get_roles(env: Environment): @pytest.mark.asyncio async def test_publish_and_process_message(env: Environment): - product_manager = ProductManager(name="Alice", profile="Product Manager", - goal="做AI Native产品", constraints="资源有限") - architect = Architect(name="Bob", profile="Architect", goal="设计一个可用、高效、较低成本的系统，包括数据结构与接口", - constraints="资源有限，需要节省成本") + product_manager = ProductManager(name="Alice", profile="Product Manager", goal="做AI Native产品", constraints="资源有限") + architect = Architect( + name="Bob", profile="Architect", goal="设计一个可用、高效、较低成本的系统，包括数据结构与接口", constraints="资源有限，需要节省成本" + ) env.add_roles([product_manager, architect]) env.publish_message(Message(role="User", content="需要一个基于LLM做总结的搜索引擎", cause_by=UserRequirement)) diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 23be82268..f2d4371d5 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -9,14 +9,12 @@ import pytest -from metagpt.config import Config -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM @pytest.fixture() def llm(): - options = Config().runtime_options - return LLM(options=options, cost_manager=CostManager(**options)) + return LLM() @pytest.mark.asyncio @@ -36,5 +34,6 @@ async def test_llm_acompletion(llm): assert len(await llm.acompletion_batch([hello_msg])) > 0 assert len(await llm.acompletion_batch_text([hello_msg])) > 0 + # if __name__ == "__main__": # pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index 9003dbe9c..e457101a9 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -24,4 +24,3 @@ async def test_sd_engine_run_t2i(): await sd_engine.run_t2i(prompts=["test"]) img_path = CONFIG.workspace_path / "resources" / "SD_Output" / "output_0.png" assert os.path.exists(img_path) - diff --git a/tests/metagpt/tools/test_web_browser_engine_playwright.py b/tests/metagpt/tools/test_web_browser_engine_playwright.py index 5ebd7394e..cc6c09925 100644 --- a/tests/metagpt/tools/test_web_browser_engine_playwright.py +++ b/tests/metagpt/tools/test_web_browser_engine_playwright.py @@ -24,8 +24,9 @@ async def test_scrape_web_page(browser_type, use_proxy, kwagrs, url, urls, proxy try: if use_proxy: conf.global_proxy = proxy - browser = web_browser_engine_playwright.PlaywrightWrapper(options=conf.runtime_options, - browser_type=browser_type, **kwagrs) + browser = web_browser_engine_playwright.PlaywrightWrapper( + options=conf.runtime_options, browser_type=browser_type, **kwagrs + ) result = await browser.run(url) result = result.inner_text assert isinstance(result, str) diff --git a/tests/metagpt/utils/test_config.py b/tests/metagpt/utils/test_config.py index f38cddb0d..bd89f0ed3 100644 --- a/tests/metagpt/utils/test_config.py +++ b/tests/metagpt/utils/test_config.py @@ -33,6 +33,5 @@ def test_options(): assert config.options -if __name__ == '__main__': +if __name__ == "__main__": test_options() - From 1a36361691e2ff12739f1d446a9fdef8a173705a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 15 Dec 2023 17:13:56 +0800 Subject: [PATCH 388/398] feat: merge geekan:env_refactor --- metagpt/provider/fireworks_api.py | 3 +- metagpt/provider/open_llm_api.py | 3 +- metagpt/provider/openai_api.py | 6 +- metagpt/roles/product_manager.py | 7 +-- metagpt/utils/mermaid.py | 100 +++++++++++++++--------------- 5 files changed, 62 insertions(+), 57 deletions(-) diff --git a/metagpt/provider/fireworks_api.py b/metagpt/provider/fireworks_api.py index 47ac9cf61..5dc68ad35 100644 --- a/metagpt/provider/fireworks_api.py +++ b/metagpt/provider/fireworks_api.py @@ -5,7 +5,8 @@ import openai from metagpt.config import CONFIG -from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter +from metagpt.provider.openai_api import OpenAIGPTAPI, RateLimiter +from metagpt.utils.cost_manager import CostManager class FireWorksGPTAPI(OpenAIGPTAPI): diff --git a/metagpt/provider/open_llm_api.py b/metagpt/provider/open_llm_api.py index f421e30c8..97e4c9f67 100644 --- a/metagpt/provider/open_llm_api.py +++ b/metagpt/provider/open_llm_api.py @@ -6,7 +6,8 @@ import openai from metagpt.config import CONFIG from metagpt.logs import logger -from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter +from metagpt.provider.openai_api import OpenAIGPTAPI, RateLimiter +from metagpt.utils.cost_manager import CostManager class OpenLLMCostManager(CostManager): diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 206be29d0..493f88153 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -118,7 +118,11 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): kwargs["model"] = CONFIG.deployment_id else: kwargs["model"] = self.model - kwargs["timeout"] = max(CONFIG.TIMEOUT, timeout) if CONFIG.TIMEOUT is not None else timeout + try: + default_timeout = int(CONFIG.TIMEOUT) if CONFIG.TIMEOUT else 0 + except ValueError: + default_timeout = 0 + kwargs["timeout"] = max(default_timeout, timeout) return kwargs diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index b37a2f777..f022237f5 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -45,6 +45,7 @@ class ProductManager(Role): self._init_actions([PrepareDocuments, WritePRD]) self._watch([UserRequirement, PrepareDocuments]) + self._todo = any_to_name(PrepareDocuments) async def _think(self) -> None: """Decide what to do""" @@ -52,6 +53,7 @@ class ProductManager(Role): self._set_state(1) else: self._set_state(0) + self._todo = any_to_name(WritePRD) return self._rc.todo async def _observe(self, ignore_memory=False) -> int: @@ -59,7 +61,4 @@ class ProductManager(Role): @property def todo(self) -> str: - if self._rc.state == 0: - return any_to_name(WritePRD) - else: - return any_to_name(PrepareDocuments) + return self._todo diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 3fa7ab79a..a1a6d462b 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -93,57 +93,57 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, return 0 +MMC1 = """classDiagram +class Main { + -SearchEngine search_engine + +main() str +} +class SearchEngine { + -Index index + -Ranking ranking + -Summary summary + +search(query: str) str +} +class Index { + -KnowledgeBase knowledge_base + +create_index(data: dict) + +query_index(query: str) list +} +class Ranking { + +rank_results(results: list) list +} +class Summary { + +summarize_results(results: list) str +} +class KnowledgeBase { + +update(data: dict) + +fetch_data(query: str) dict +} +Main --> SearchEngine +SearchEngine --> Index +SearchEngine --> Ranking +SearchEngine --> Summary +Index --> KnowledgeBase""" + +MMC2 = """sequenceDiagram +participant M as Main +participant SE as SearchEngine +participant I as Index +participant R as Ranking +participant S as Summary +participant KB as KnowledgeBase +M->>SE: search(query) +SE->>I: query_index(query) +I->>KB: fetch_data(query) +KB-->>I: return data +I-->>SE: return results +SE->>R: rank_results(results) +R-->>SE: return ranked_results +SE->>S: summarize_results(ranked_results) +S-->>SE: return summary +SE-->>M: return summary""" + if __name__ == "__main__": - MMC1 = """classDiagram - class Main { - -SearchEngine search_engine - +main() str - } - class SearchEngine { - -Index index - -Ranking ranking - -Summary summary - +search(query: str) str - } - class Index { - -KnowledgeBase knowledge_base - +create_index(data: dict) - +query_index(query: str) list - } - class Ranking { - +rank_results(results: list) list - } - class Summary { - +summarize_results(results: list) str - } - class KnowledgeBase { - +update(data: dict) - +fetch_data(query: str) dict - } - Main --> SearchEngine - SearchEngine --> Index - SearchEngine --> Ranking - SearchEngine --> Summary - Index --> KnowledgeBase""" - - MMC2 = """sequenceDiagram - participant M as Main - participant SE as SearchEngine - participant I as Index - participant R as Ranking - participant S as Summary - participant KB as KnowledgeBase - M->>SE: search(query) - SE->>I: query_index(query) - I->>KB: fetch_data(query) - KB-->>I: return data - I-->>SE: return results - SE->>R: rank_results(results) - R-->>SE: return ranked_results - SE->>S: summarize_results(ranked_results) - S-->>SE: return summary - SE-->>M: return summary""" - loop = asyncio.new_event_loop() result = loop.run_until_complete(mermaid_to_file(MMC1, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/2")) From 39892f47ff3a4e270863f4db420c5ca3e2a6d67d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 15 Dec 2023 19:29:26 +0800 Subject: [PATCH 389/398] merge geekan:v0.5.0 --- config/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.yaml b/config/config.yaml index 9acdbe8a1..c436b026a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -11,6 +11,7 @@ OPENAI_API_BASE: "https://api.openai.com/v1" OPENAI_API_MODEL: "gpt-4-1106-preview" MAX_TOKENS: 4096 RPM: 10 +#LLM_TYPE: OpenAI # Except for these three major models – OpenAI, MetaGPT LLM, and Azure – other large models can be distinguished based on the validity of the key. #### if Spark #SPARK_APPID : "YOUR_APPID" From 8636026c557dbb6bbad98e1c290416a052fcbed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 15 Dec 2023 20:00:17 +0800 Subject: [PATCH 390/398] feat: merge fixbug/rfc135_merge_geekan_cli_etc_1445 --- metagpt/memory/brain_memory.py | 7 +-- metagpt/utils/mermaid.py | 100 +++++++++++++++++---------------- 2 files changed, 55 insertions(+), 52 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index be3736100..decbb6a8b 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -15,12 +15,11 @@ from typing import Dict, List, Optional import openai import pydantic -from metagpt import Message from metagpt.config import CONFIG from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS from metagpt.llm import LLMType from metagpt.logs import logger -from metagpt.schema import RawMessage +from metagpt.schema import Message, RawMessage from metagpt.utils.redis import Redis @@ -45,12 +44,12 @@ class BrainMemory(pydantic.BaseModel): cacheable: bool = True def add_talk(self, msg: Message): - msg.add_tag(MessageType.Talk.value) + msg.role = "user" self.add_history(msg) self.is_dirty = True def add_answer(self, msg: Message): - msg.add_tag(MessageType.Answer.value) + msg.role = "assistant" self.add_history(msg) self.is_dirty = True diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index a1a6d462b..9aefeb5aa 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -93,55 +93,59 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, return 0 -MMC1 = """classDiagram -class Main { - -SearchEngine search_engine - +main() str -} -class SearchEngine { - -Index index - -Ranking ranking - -Summary summary - +search(query: str) str -} -class Index { - -KnowledgeBase knowledge_base - +create_index(data: dict) - +query_index(query: str) list -} -class Ranking { - +rank_results(results: list) list -} -class Summary { - +summarize_results(results: list) str -} -class KnowledgeBase { - +update(data: dict) - +fetch_data(query: str) dict -} -Main --> SearchEngine -SearchEngine --> Index -SearchEngine --> Ranking -SearchEngine --> Summary -Index --> KnowledgeBase""" +MMC1 = """ +classDiagram + class Main { + -SearchEngine search_engine + +main() str + } + class SearchEngine { + -Index index + -Ranking ranking + -Summary summary + +search(query: str) str + } + class Index { + -KnowledgeBase knowledge_base + +create_index(data: dict) + +query_index(query: str) list + } + class Ranking { + +rank_results(results: list) list + } + class Summary { + +summarize_results(results: list) str + } + class KnowledgeBase { + +update(data: dict) + +fetch_data(query: str) dict + } + Main --> SearchEngine + SearchEngine --> Index + SearchEngine --> Ranking + SearchEngine --> Summary + Index --> KnowledgeBase +""" -MMC2 = """sequenceDiagram -participant M as Main -participant SE as SearchEngine -participant I as Index -participant R as Ranking -participant S as Summary -participant KB as KnowledgeBase -M->>SE: search(query) -SE->>I: query_index(query) -I->>KB: fetch_data(query) -KB-->>I: return data -I-->>SE: return results -SE->>R: rank_results(results) -R-->>SE: return ranked_results -SE->>S: summarize_results(ranked_results) -S-->>SE: return summary -SE-->>M: return summary""" +MMC2 = """ +sequenceDiagram + participant M as Main + participant SE as SearchEngine + participant I as Index + participant R as Ranking + participant S as Summary + participant KB as KnowledgeBase + M->>SE: search(query) + SE->>I: query_index(query) + I->>KB: fetch_data(query) + KB-->>I: return data + I-->>SE: return results + SE->>R: rank_results(results) + R-->>SE: return ranked_results + SE->>S: summarize_results(ranked_results) + S-->>SE: return summary + SE-->>M: return summary +""" if __name__ == "__main__": loop = asyncio.new_event_loop() From a9479843f65719824576c8a3e14fbc558356124a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 15 Dec 2023 20:32:16 +0800 Subject: [PATCH 391/398] feat: merge fixbug/rfc135_merge_geekan_cli_etc_1445 --- config/config.yaml | 2 +- metagpt/learn/text_to_image.py | 5 ++--- metagpt/learn/text_to_speech.py | 4 ++-- metagpt/utils/s3.py | 16 ++++++++++++++++ requirements.txt | 1 + tests/conftest.py | 2 +- tests/metagpt/learn/test_text_to_image.py | 14 +++++--------- tests/metagpt/test_environment.py | 5 +++++ 8 files changed, 33 insertions(+), 16 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 87637f0b5..496167e13 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -11,7 +11,7 @@ OPENAI_API_BASE: "https://api.openai.com/v1" OPENAI_API_MODEL: "gpt-4-1106-preview" MAX_TOKENS: 4096 RPM: 10 -#LLM_TYPE: OpenAI # Except for these three major models – OpenAI, MetaGPT LLM, and Azure – other large models can be distinguished based on the validity of the key. +LLM_TYPE: OpenAI # Except for these three major models – OpenAI, MetaGPT LLM, and Azure – other large models can be distinguished based on the validity of the key. #### if Spark #SPARK_APPID : "YOUR_APPID" diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 23c2bddad..24669312c 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -6,7 +6,6 @@ @File : text_to_image.py @Desc : Text-to-Image skill, which provides text-to-image functionality. """ -import openai.error from metagpt.config import CONFIG from metagpt.const import BASE64_FORMAT @@ -30,10 +29,10 @@ async def text_to_image(text, size_type: str = "512x512", openai_api_key="", mod elif CONFIG.OPENAI_API_KEY or openai_api_key: base64_data = await oas3_openai_text_to_image(text, size_type, openai_api_key) else: - raise openai.error.InvalidRequestError("缺少必要的参数") + raise ValueError("Missing necessary parameters.") s3 = S3() - url = await s3.cache(data=base64_data, file_ext=".png", format=BASE64_FORMAT) + url = await s3.cache(data=base64_data, file_ext=".png", format=BASE64_FORMAT) if s3.is_valid else "" if url: return f"![{text}]({url})" return image_declaration + base64_data if base64_data else "" diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 7c085c02f..972515599 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -49,7 +49,7 @@ async def text_to_speech( audio_declaration = "data:audio/wav;base64," base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) s3 = S3() - url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) + url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) if s3.is_valid else "" if url: return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data @@ -61,7 +61,7 @@ async def text_to_speech( text=text, app_id=iflytek_app_id, api_key=iflytek_api_key, api_secret=iflytek_api_secret ) s3 = S3() - url = await s3.cache(data=base64_data, file_ext=".mp3", format=BASE64_FORMAT) + url = await s3.cache(data=base64_data, file_ext=".mp3", format=BASE64_FORMAT) if s3.is_valid else "" if url: return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 4c3533d5b..9accfcade 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -152,3 +152,19 @@ class S3: logger.exception(f"{e}, stack:{traceback.format_exc()}") pathname.unlink(missing_ok=True) return None + + @property + def is_valid(self): + is_invalid = ( + not CONFIG.S3_ACCESS_KEY + or CONFIG.S3_ACCESS_KEY == "YOUR_S3_ACCESS_KEY" + or not CONFIG.S3_SECRET_KEY + or CONFIG.S3_SECRET_KEY == "YOUR_S3_SECRET_KEY" + or not CONFIG.S3_ENDPOINT_URL + or CONFIG.S3_ENDPOINT_URL == "YOUR_S3_ENDPOINT_URL" + or not CONFIG.S3_BUCKET + or CONFIG.S3_BUCKET == "YOUR_S3_BUCKET" + ) + if is_invalid: + logger.info("S3 is invalid") + return not is_invalid diff --git a/requirements.txt b/requirements.txt index cf7d8d519..d2a4e5bb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,3 +56,4 @@ zhipuai==1.0.7 socksio~=1.0.0 gitignore-parser==0.1.9 connexion[swagger-ui] +websockets~=12.0 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 375b9ff7f..47e05e20e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ from unittest.mock import Mock import pytest -from metagpt.config import CONFIG +from metagpt.config import CONFIG, Config from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI diff --git a/tests/metagpt/learn/test_text_to_image.py b/tests/metagpt/learn/test_text_to_image.py index 982a39b13..a6cbc45bf 100644 --- a/tests/metagpt/learn/test_text_to_image.py +++ b/tests/metagpt/learn/test_text_to_image.py @@ -6,15 +6,17 @@ @File : test_text_to_image.py @Desc : Unit tests. """ -import asyncio + import base64 +import pytest from pydantic import BaseModel from metagpt.learn.text_to_image import text_to_image -async def mock_text_to_image(): +@pytest.mark.asyncio +async def test(): class Input(BaseModel): input: str size_type: str @@ -36,11 +38,5 @@ async def mock_text_to_image(): assert base64.b64decode(data, validate=True) -def test_suite(): - loop = asyncio.get_event_loop() - task = loop.create_task(mock_text_to_image()) - loop.run_until_complete(task) - - if __name__ == "__main__": - test_suite() + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index 933d74b97..fd731cf9e 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -11,6 +11,7 @@ import pytest from metagpt.actions import UserRequirement +from metagpt.config import CONFIG from metagpt.environment import Environment from metagpt.logs import logger from metagpt.roles import Architect, ProductManager, Role @@ -41,6 +42,10 @@ def test_get_roles(env: Environment): @pytest.mark.asyncio async def test_publish_and_process_message(env: Environment): + if CONFIG.git_repo: + CONFIG.git_repo.delete_repository() + CONFIG.git_repo = None + product_manager = ProductManager(name="Alice", profile="Product Manager", goal="做AI Native产品", constraints="资源有限") architect = Architect( name="Bob", profile="Architect", goal="设计一个可用、高效、较低成本的系统，包括数据结构与接口", constraints="资源有限，需要节省成本" From 41361915a12b236d82980299310e555021d56a7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 18 Dec 2023 11:31:08 +0800 Subject: [PATCH 392/398] feat: upgrade openai 1.x --- metagpt/learn/text_to_speech.py | 2 +- metagpt/memory/brain_memory.py | 2 +- metagpt/provider/fireworks_api.py | 6 ++++-- metagpt/provider/open_llm_api.py | 9 ++++----- metagpt/provider/zhipuai_api.py | 4 ++-- metagpt/tools/openai_text_to_image.py | 18 +++++++----------- tests/metagpt/test_environment.py | 4 ++++ tests/metagpt/test_gpt.py | 4 ++-- tests/metagpt/test_llm.py | 4 ++-- tests/metagpt/test_startup.py | 4 ++++ tests/metagpt/test_subscription.py | 4 ++++ 11 files changed, 35 insertions(+), 26 deletions(-) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 972515599..72958b8c7 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -66,7 +66,7 @@ async def text_to_speech( return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data - raise openai.error.InvalidRequestError( + raise openai.InvalidRequestError( message="AZURE_TTS_SUBSCRIPTION_KEY, AZURE_TTS_REGION, IFLYTEK_APP_ID, IFLYTEK_API_KEY, IFLYTEK_API_SECRET error", param={}, ) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index decbb6a8b..034bcfa56 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -171,7 +171,7 @@ class BrainMemory(pydantic.BaseModel): if summary: await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS) return summary - raise openai.error.InvalidRequestError(message="text too long", param=None) + raise openai.InvalidRequestError(message="text too long", param=None) async def _metagpt_summarize(self, max_words=200, **kwargs): if not self.history: diff --git a/metagpt/provider/fireworks_api.py b/metagpt/provider/fireworks_api.py index 5dc68ad35..6625cda97 100644 --- a/metagpt/provider/fireworks_api.py +++ b/metagpt/provider/fireworks_api.py @@ -19,6 +19,8 @@ class FireWorksGPTAPI(OpenAIGPTAPI): RateLimiter.__init__(self, rpm=self.rpm) def __init_fireworks(self, config: "Config"): - openai.api_key = config.fireworks_api_key - openai.api_base = config.fireworks_api_base + # TODO: The 'openai.api_base' option isn't read in the client API. You will need to pass it when you + # instantiate the client, e.g. 'OpenAI(api_base=config.fireworks_api_base)' + # openai.api_key = config.fireworks_api_key + # openai.api_base = config.fireworks_api_base self.rpm = int(config.get("RPM", 10)) diff --git a/metagpt/provider/open_llm_api.py b/metagpt/provider/open_llm_api.py index 97e4c9f67..cd30c4a58 100644 --- a/metagpt/provider/open_llm_api.py +++ b/metagpt/provider/open_llm_api.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- # @Desc : self-host open llm model with openai-compatible interface -import openai - from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.openai_api import OpenAIGPTAPI, RateLimiter @@ -35,13 +33,14 @@ class OpenLLMCostManager(CostManager): class OpenLLMGPTAPI(OpenAIGPTAPI): def __init__(self): self.__init_openllm(CONFIG) - self.llm = openai self.model = CONFIG.open_llm_api_model self.auto_max_tokens = False self._cost_manager = OpenLLMCostManager() RateLimiter.__init__(self, rpm=self.rpm) def __init_openllm(self, config: "Config"): - openai.api_key = "sk-xx" # self-host api doesn't need api-key, use the default value - openai.api_base = config.open_llm_api_base + # TODO: The 'openai.api_base' option isn't read in the client API. You will need to pass it when you + # instantiate the client, e.g. 'OpenAI(api_base=config.open_llm_api_base)' + # openai.api_key = "sk-xx" # self-host api doesn't need api-key, use the default value + # openai.api_base = config.open_llm_api_base self.rpm = int(config.get("RPM", 10)) diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 82513f83c..ff8e5531e 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -5,7 +5,6 @@ import json from enum import Enum -import openai import zhipuai from requests import ConnectionError from tenacity import ( @@ -48,7 +47,8 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): def __init_zhipuai(self, config: CONFIG): assert config.zhipuai_api_key zhipuai.api_key = config.zhipuai_api_key - openai.api_key = zhipuai.api_key # due to use openai sdk, set the api_key but it will't be used. + # due to use openai sdk, set the api_key but it will't be used. + # openai.api_key = zhipuai.api_key # due to use openai sdk, set the api_key but it will't be used. def _const_kwargs(self, messages: list[dict]) -> dict: kwargs = {"model": self.model, "prompt": messages, "temperature": 0.3} diff --git a/metagpt/tools/openai_text_to_image.py b/metagpt/tools/openai_text_to_image.py index 6025f04ba..80de04e45 100644 --- a/metagpt/tools/openai_text_to_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -10,8 +10,8 @@ import asyncio import base64 import aiohttp -import openai import requests +from openai import AsyncOpenAI from metagpt.config import CONFIG, Config from metagpt.logs import logger @@ -23,6 +23,11 @@ class OpenAIText2Image: :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` """ self.openai_api_key = openai_api_key if openai_api_key else CONFIG.OPENAI_API_KEY + self._client = AsyncOpenAI(api_key=self.openai_api_key, base_url=CONFIG.openai_api_base) + + def __del__(self): + if self._client: + self._client.close() async def text_2_image(self, text, size_type="1024x1024"): """Text to image @@ -32,16 +37,7 @@ class OpenAIText2Image: :return: The image data is returned in Base64 encoding. """ try: - result = await openai.Image.acreate( - api_key=CONFIG.OPENAI_API_KEY, - api_base=CONFIG.OPENAI_API_BASE, - api_type=None, - api_version=None, - organization=None, - prompt=text, - n=1, - size=size_type, - ) + result = await self._client.images.generate(prompt=text, n=1, size=size_type) except Exception as e: logger.error(f"An error occurred:{e}") return "" diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index fd731cf9e..bc88eb742 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -56,3 +56,7 @@ async def test_publish_and_process_message(env: Environment): await env.run(k=2) logger.info(f"{env.history=}") assert len(env.history) > 10 + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_gpt.py b/tests/metagpt/test_gpt.py index dda5e6252..daafeb708 100644 --- a/tests/metagpt/test_gpt.py +++ b/tests/metagpt/test_gpt.py @@ -54,5 +54,5 @@ class TestGPT: assert costs.total_cost > 0 -# if __name__ == "__main__": -# pytest.main([__file__, "-s"]) +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index f2d4371d5..d972e55c0 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -35,5 +35,5 @@ async def test_llm_acompletion(llm): assert len(await llm.acompletion_batch_text([hello_msg])) > 0 -# if __name__ == "__main__": -# pytest.main([__file__, "-s"]) +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_startup.py b/tests/metagpt/test_startup.py index c34fd2c31..c8d4d5d29 100644 --- a/tests/metagpt/test_startup.py +++ b/tests/metagpt/test_startup.py @@ -26,3 +26,7 @@ async def test_team(): # def test_startup(): # args = ["Make a 2048 game"] # result = runner.invoke(app, args) + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_subscription.py b/tests/metagpt/test_subscription.py index 2e898424d..1399df7fe 100644 --- a/tests/metagpt/test_subscription.py +++ b/tests/metagpt/test_subscription.py @@ -100,3 +100,7 @@ async def test_subscription_run_error(loguru_caplog): logs = "".join(loguru_caplog.messages) assert "run error" in logs assert "has completed" in logs + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 9c405dfa77c81a629f86b82c7721a2389db93472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 18 Dec 2023 16:13:21 +0800 Subject: [PATCH 393/398] fixbug: recursive user requirement dead loop --- metagpt/actions/role_run.py | 16 ++++++++++++++++ metagpt/roles/role.py | 21 +++++++-------------- metagpt/schema.py | 4 ---- 3 files changed, 23 insertions(+), 18 deletions(-) create mode 100644 metagpt/actions/role_run.py diff --git a/metagpt/actions/role_run.py b/metagpt/actions/role_run.py new file mode 100644 index 000000000..9f0c626b8 --- /dev/null +++ b/metagpt/actions/role_run.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/18 +@Author : mashenquan +@File : role_run.py +@Desc : Message type caused by `Role.run()` invocation. +""" +from metagpt.actions import Action + + +class RoleRun(Action): + """Message type caused by `Role.run` invocation""" + + async def run(self, *args, **kwargs): + raise NotImplementedError diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1e7ebf711..413595c6b 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -27,7 +27,7 @@ from pydantic import BaseModel, Field from metagpt.actions import Action, ActionOutput from metagpt.actions.action_node import ActionNode -from metagpt.actions.add_requirement import UserRequirement +from metagpt.actions.role_run import RoleRun from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger from metagpt.memory import Memory @@ -127,17 +127,7 @@ class RoleContext(BaseModel): return self.memory.get() -class _RoleInjector(type): - def __call__(cls, *args, **kwargs): - instance = super().__call__(*args, **kwargs) - - if not instance._rc.watch: - instance._watch([UserRequirement]) - - return instance - - -class Role(metaclass=_RoleInjector): +class Role: """Role/Agent""" def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False): @@ -152,7 +142,6 @@ class Role(metaclass=_RoleInjector): self._rc = RoleContext() self._subscription = {any_to_str(self), name} if name else {any_to_str(self)} - def _reset(self): self._states = [] self._actions = [] @@ -304,7 +293,9 @@ class Role(metaclass=_RoleInjector): old_messages = [] if ignore_memory else self._rc.memory.get() self._rc.memory.add_batch(news) # Filter out messages of interest. - self._rc.news = [n for n in news if n.cause_by in self._rc.watch and n not in old_messages] + watch = self._rc.watch or set() + watch.add(any_to_str(RoleRun)) + self._rc.news = [n for n in news if n.cause_by in watch and n not in old_messages] # Design Rules: # If you need to further categorize Message objects, you can do so using the Message.set_meta function. @@ -401,6 +392,8 @@ class Role(metaclass=_RoleInjector): msg = with_message elif isinstance(with_message, list): msg = Message("\n".join(with_message)) + if not msg.cause_by: + msg.cause_by = RoleRun self.put_message(msg) if not await self._observe(): diff --git a/metagpt/schema.py b/metagpt/schema.py index 5aec378e4..758149efa 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -121,10 +121,6 @@ class Message(BaseModel): :param send_to: Specifies the target recipient or consumer for message delivery in the environment. :param role: Message meta info tells who sent this message. """ - if not cause_by: - from metagpt.actions import UserRequirement - cause_by = UserRequirement - super().__init__( id=uuid.uuid4().hex, content=content, From e42b1969cca62a1c5b209278e2f2678d518342e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 19 Dec 2023 10:44:06 +0800 Subject: [PATCH 394/398] fixbug: Message id, token counter --- metagpt/schema.py | 10 ++++++++-- tests/metagpt/test_role.py | 6 +++--- tests/metagpt/test_schema.py | 3 --- tests/metagpt/utils/test_token_counter.py | 6 +++++- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 758149efa..9916bffff 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -163,8 +163,14 @@ class Message(BaseModel): def load(val): """Convert the json string to object.""" try: - d = json.loads(val) - return Message(**d) + m = json.loads(val) + id = m.get("id") + if "id" in m: + del m["id"] + msg = Message(**m) + if id: + msg.id = id + return msg except JSONDecodeError as err: logger.error(f"parse json failed: {val}, error:{err}") return None diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 8fac2503c..cf09d6f0a 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -88,13 +88,13 @@ async def test_react(): @pytest.mark.asyncio async def test_msg_to(): m = Message(content="a", send_to=["a", MockRole, Message]) - assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) + assert m.send_to == {"a", get_class_name(MockRole), get_class_name(Message)} m = Message(content="a", cause_by=MockAction, send_to={"a", MockRole, Message}) - assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) + assert m.send_to == {"a", get_class_name(MockRole), get_class_name(Message)} m = Message(content="a", send_to=("a", MockRole, Message)) - assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) + assert m.send_to == {"a", get_class_name(MockRole), get_class_name(Message)} if __name__ == "__main__": diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 51ebd5baa..40b18e0f4 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -16,7 +16,6 @@ from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage from metagpt.utils.common import get_class_name -@pytest.mark.asyncio def test_messages(): test_content = "test_message" msgs = [ @@ -30,7 +29,6 @@ def test_messages(): assert all([i in text for i in roles]) -@pytest.mark.asyncio def test_message(): m = Message("a", role="v1") v = m.dump() @@ -61,7 +59,6 @@ def test_message(): assert m.content == "b" -@pytest.mark.asyncio def test_routes(): m = Message("a", role="b", cause_by="c", x="d", send_to="c") m.send_to = "b" diff --git a/tests/metagpt/utils/test_token_counter.py b/tests/metagpt/utils/test_token_counter.py index 479ccc22d..acb99d717 100644 --- a/tests/metagpt/utils/test_token_counter.py +++ b/tests/metagpt/utils/test_token_counter.py @@ -15,7 +15,7 @@ def test_count_message_tokens(): {"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi there!"}, ] - assert count_message_tokens(messages) == 17 + assert count_message_tokens(messages) == 15 def test_count_message_tokens_with_name(): @@ -67,3 +67,7 @@ def test_count_string_tokens_gpt_4(): string = "Hello, world!" assert count_string_tokens(string, model_name="gpt-4-0314") == 4 + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 3a35c0a0cdea75f35cff40a2b85392324268e784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 19 Dec 2023 16:32:51 +0800 Subject: [PATCH 395/398] feat: add GraphRepository --- metagpt/memory/brain_memory.py | 2 +- metagpt/repo_parser.py | 6 +- metagpt/utils/common.py | 4 + metagpt/utils/di_graph_repository.py | 69 ++++++++++ metagpt/utils/graph_repository.py | 42 ++++++ requirements.txt | 3 +- .../metagpt/utils/test_di_graph_repository.py | 121 ++++++++++++++++++ 7 files changed, 244 insertions(+), 3 deletions(-) create mode 100644 metagpt/utils/di_graph_repository.py create mode 100644 metagpt/utils/graph_repository.py create mode 100644 tests/metagpt/utils/test_di_graph_repository.py diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 034bcfa56..8aa3be2b6 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -4,7 +4,7 @@ @Time : 2023/8/18 @Author : mashenquan @File : brain_memory.py -@Desc : Support memory for multiple tasks and multiple mainlines. +@Desc : Support memory for multiple tasks and multiple mainlines. Obsoleted by `utils/*_repository.py`. @Modified By: mashenquan, 2023/9/4. + redis memory cache. """ import json diff --git a/metagpt/repo_parser.py b/metagpt/repo_parser.py index b84dbab9a..65c2959e4 100644 --- a/metagpt/repo_parser.py +++ b/metagpt/repo_parser.py @@ -51,7 +51,11 @@ class RepoParser(BaseModel): def generate_symbols(self): files_classes = [] directory = self.base_directory - for path in directory.rglob("*.py"): + matching_files = [] + extensions = ["*.py", "*.js"] + for ext in extensions: + matching_files += directory.rglob(ext) + for path in matching_files: tree = self.parse_file(path) file_info = self.extract_class_and_function_info(tree, path) files_classes.append(file_info) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 2a3d22698..575c77b5e 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -393,3 +393,7 @@ def format_value(value): for k, v in merged_opts.items(): value = value.replace("{" + f"{k}" + "}", str(v)) return value + + +def concat_namespace(*args) -> str: + return ":".join(str(value) for value in args) diff --git a/metagpt/utils/di_graph_repository.py b/metagpt/utils/di_graph_repository.py new file mode 100644 index 000000000..9bbd38d5f --- /dev/null +++ b/metagpt/utils/di_graph_repository.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/19 +@Author : mashenquan +@File : di_graph_repository.py +@Desc : Graph repository based on DiGraph +""" +from __future__ import annotations + +import json +from pathlib import Path + +import aiofiles +import networkx + +from metagpt.utils.graph_repository import GraphRepository + + +class DiGraphRepository(GraphRepository): + def __init__(self, name: str, **kwargs): + super().__init__(name=name, **kwargs) + self._repo = networkx.DiGraph() + + async def insert(self, subject: str, predicate: str, object_: str): + self._repo.add_edge(subject, object_, predicate=predicate) + + async def upsert(self, subject: str, predicate: str, object_: str): + pass + + async def update(self, subject: str, predicate: str, object_: str): + pass + + def json(self) -> str: + m = networkx.node_link_data(self._repo) + data = json.dumps(m) + return data + + async def save(self, path: str | Path = None): + data = self.json() + path = path or self._kwargs.get("root") + if not path.exists(): + path.mkdir(parents=True, exist_ok=True) + pathname = Path(path) / self.name + async with aiofiles.open(str(pathname.with_suffix(".json")), mode="w", encoding="utf-8") as writer: + await writer.write(data) + + async def load(self, pathname: str | Path): + async with aiofiles.open(str(pathname), mode="r", encoding="utf-8") as reader: + data = await reader.read(-1) + m = json.loads(data) + self._repo = networkx.node_link_graph(m) + + @staticmethod + async def load_from(pathname: str | Path) -> GraphRepository: + name = Path(pathname).with_suffix("").name + root = Path(pathname).parent + graph = DiGraphRepository(name=name, root=root) + await graph.load(pathname=pathname) + return graph + + @property + def root(self) -> str: + return self._kwargs.get("root") + + @property + def pathname(self) -> Path: + p = Path(self.root) / self.name + return p.with_suffix(".json") diff --git a/metagpt/utils/graph_repository.py b/metagpt/utils/graph_repository.py new file mode 100644 index 000000000..600575b4e --- /dev/null +++ b/metagpt/utils/graph_repository.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/19 +@Author : mashenquan +@File : graph_repository.py +@Desc : Superclass for graph repository. +""" +from abc import ABC, abstractmethod +from enum import Enum + + +class GraphKeyword(Enum): + IS = "is" + CLASS = "class" + FUNCTION = "function" + GLOBAL_VARIABLE = "global_variable" + CLASS_FUNCTION = "class_function" + CLASS_PROPERTY = "class_property" + HAS_CLASS = "has_class" + + +class GraphRepository(ABC): + def __init__(self, name: str, **kwargs): + self._repo_name = name + self._kwargs = kwargs + + @abstractmethod + async def insert(self, subject: str, predicate: str, object_: str): + pass + + @abstractmethod + async def upsert(self, subject: str, predicate: str, object_: str): + pass + + @abstractmethod + async def update(self, subject: str, predicate: str, object_: str): + pass + + @property + def name(self) -> str: + return self._repo_name diff --git a/requirements.txt b/requirements.txt index d2a4e5bb4..4310aec6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,4 +56,5 @@ zhipuai==1.0.7 socksio~=1.0.0 gitignore-parser==0.1.9 connexion[swagger-ui] -websockets~=12.0 \ No newline at end of file +websockets~=12.0 +networkx~=3.2.1 \ No newline at end of file diff --git a/tests/metagpt/utils/test_di_graph_repository.py b/tests/metagpt/utils/test_di_graph_repository.py new file mode 100644 index 000000000..7a9e58d1c --- /dev/null +++ b/tests/metagpt/utils/test_di_graph_repository.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/19 +@Author : mashenquan +@File : test_di_graph_repository.py +@Desc : Unit tests for di_graph_repository.py +""" + +from pathlib import Path + +import pytest +from pydantic import BaseModel + +from metagpt.const import DEFAULT_WORKSPACE_ROOT +from metagpt.repo_parser import RepoParser +from metagpt.utils.common import concat_namespace +from metagpt.utils.di_graph_repository import DiGraphRepository +from metagpt.utils.graph_repository import GraphKeyword + + +@pytest.mark.asyncio +async def test_di_graph_repository(): + class Input(BaseModel): + s: str + p: str + o: str + + inputs = [ + {"s": "main.py:Game:draw", "p": "method:hasDescription", "o": "Draw image"}, + {"s": "main.py:Game:draw", "p": "method:hasDescription", "o": "Show image"}, + ] + path = Path(__file__).parent + graph = DiGraphRepository(name="test", root=path) + for i in inputs: + data = Input(**i) + await graph.insert(subject=data.s, predicate=data.p, object_=data.o) + v = graph.json() + assert v + await graph.save() + assert graph.pathname.exists() + graph.pathname.unlink() + + +async def test_js_parser(): + class Input(BaseModel): + path: str + + inputs = [ + {"path": str(Path(__file__).parent / "../../data/code")}, + ] + path = Path(__file__).parent + graph = DiGraphRepository(name="test", root=path) + for i in inputs: + data = Input(**i) + repo_parser = RepoParser(base_directory=data.path) + symbols = repo_parser.generate_symbols() + for s in symbols: + ns = s.get("file", "") + for c in s.get("classes", []): + await graph.insert( + subject=concat_namespace(ns, c), predicate=GraphKeyword.IS.value, object_=GraphKeyword.CLASS.value + ) + for f in s.get("functions", []): + await graph.insert( + subject=concat_namespace(ns, f), + predicate=GraphKeyword.IS.value, + object_=GraphKeyword.FUNCTION.value, + ) + for g in s.get("globals", []): + await graph.insert( + subject=concat_namespace(ns, g), + predicate=GraphKeyword.IS.value, + object_=GraphKeyword.GLOBAL_VARIABLE.value, + ) + data = graph.json() + assert data + + +async def test_codes(): + path = DEFAULT_WORKSPACE_ROOT / "snake_game" + repo_parser = RepoParser(base_directory=path) + + graph = DiGraphRepository(name="test", root=path) + symbols = repo_parser.generate_symbols() + for s in symbols: + ns = s.get("file", "") + for c in s.get("classes", []): + class_name = c.get("name", "") + await graph.insert( + subject=ns, predicate=GraphKeyword.HAS_CLASS.value, object_=concat_namespace(ns, class_name) + ) + await graph.insert( + subject=concat_namespace(ns, class_name), + predicate=GraphKeyword.IS.value, + object_=GraphKeyword.CLASS.value, + ) + methods = c.get("methods", []) + for fn in methods: + await graph.insert( + subject=concat_namespace(ns, class_name, fn), + predicate=GraphKeyword.IS.value, + object_=GraphKeyword.CLASS_FUNCTION.value, + ) + for f in s.get("functions", []): + await graph.insert( + subject=concat_namespace(ns, f), predicate=GraphKeyword.IS.value, object_=GraphKeyword.FUNCTION.value + ) + for g in s.get("globals", []): + await graph.insert( + subject=concat_namespace(ns, g), + predicate=GraphKeyword.IS.value, + object_=GraphKeyword.GLOBAL_VARIABLE.value, + ) + data = graph.json() + assert data + print(data) + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 81b1e5bb1c0935f8773c3f0b6e66a7229d7f04db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 19 Dec 2023 16:37:01 +0800 Subject: [PATCH 396/398] feat: disable -- max_auto_summarize_code feat: repo_parser + page info --- metagpt/repo_parser.py | 43 ++++++++++++++++++- metagpt/startup.py | 2 +- .../metagpt/utils/test_di_graph_repository.py | 2 + 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/metagpt/repo_parser.py b/metagpt/repo_parser.py index 975ead8cd..03cf7be79 100644 --- a/metagpt/repo_parser.py +++ b/metagpt/repo_parser.py @@ -5,16 +5,20 @@ @Author : alexanderwu @File : repo_parser.py """ +from __future__ import annotations + import ast import json from pathlib import Path from pprint import pformat +from typing import List import pandas as pd from pydantic import BaseModel, Field from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.utils.common import any_to_str from metagpt.utils.exceptions import handle_exception @@ -36,7 +40,10 @@ class RepoParser(BaseModel): "globals": [], } + page_info = [] for node in tree: + info = RepoParser.node_to_str(node) + page_info.append(info) if isinstance(node, ast.ClassDef): class_methods = [m.name for m in node.body if is_func(m)] file_info["classes"].append({"name": node.name, "methods": class_methods}) @@ -46,6 +53,7 @@ class RepoParser(BaseModel): for target in node.targets if isinstance(node, ast.Assign) else [node.target]: if isinstance(target, ast.Name): file_info["globals"].append(target.id) + file_info["page_info"] = page_info return file_info def generate_symbols(self): @@ -57,7 +65,7 @@ class RepoParser(BaseModel): for ext in extensions: matching_files += directory.rglob(ext) for path in matching_files: - tree = self.parse_file(path) + tree = self._parse_file(path) file_info = self.extract_class_and_function_info(tree, path) files_classes.append(file_info) @@ -84,6 +92,39 @@ class RepoParser(BaseModel): elif mode == "csv": self.generate_dataframe_structure(output_path) + @staticmethod + def node_to_str(node) -> (int, int, str, str | List): + def _parse_name(n): + if n.asname: + return f"{n.name} as {n.asname}" + return n.name + + if any_to_str(node) == any_to_str(ast.Expr): + return node.lineno, node.end_lineno, any_to_str(node), RepoParser._parse_expr(node) + mappings = { + any_to_str(ast.Import): lambda x: [_parse_name(n) for n in x.names], + any_to_str(ast.Assign): lambda x: [n.id for n in x.targets], + any_to_str(ast.ClassDef): lambda x: x.name, + any_to_str(ast.FunctionDef): lambda x: x.name, + any_to_str(ast.ImportFrom): lambda x: {"module": x.module, "names": [_parse_name(n) for n in x.names]}, + any_to_str(ast.If): lambda x: x.test.left.id, + } + func = mappings.get(any_to_str(node)) + if func: + return node.lineno, node.end_lineno, any_to_str(node), func(node) + return node.lineno, node.end_lineno, any_to_str(node), None + + @staticmethod + def _parse_expr(node) -> (int, int, str, str | List): + if isinstance(node.value, ast.Constant): + return any_to_str(ast.Constant), node.value.value + if isinstance(node.value, ast.Call): + if isinstance(node.value.func, ast.Attribute): + return any_to_str(ast.Call), f"{node.value.func.value.id}.{node.value.func.attr}" + if isinstance(node.value.func, ast.Name): + return any_to_str(ast.Call), node.value.func.id + return any_to_str(node.value), None + def is_func(node): return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) diff --git a/metagpt/startup.py b/metagpt/startup.py index f930c386b..e886ad2a4 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -26,7 +26,7 @@ def startup( ), reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."), max_auto_summarize_code: int = typer.Option( - default=-1, + default=0, help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the workflow.", ), ): diff --git a/tests/metagpt/utils/test_di_graph_repository.py b/tests/metagpt/utils/test_di_graph_repository.py index 7a9e58d1c..ec2cb4d01 100644 --- a/tests/metagpt/utils/test_di_graph_repository.py +++ b/tests/metagpt/utils/test_di_graph_repository.py @@ -42,6 +42,7 @@ async def test_di_graph_repository(): graph.pathname.unlink() +@pytest.mark.asyncio async def test_js_parser(): class Input(BaseModel): path: str @@ -77,6 +78,7 @@ async def test_js_parser(): assert data +@pytest.mark.asyncio async def test_codes(): path = DEFAULT_WORKSPACE_ROOT / "snake_game" repo_parser = RepoParser(base_directory=path) From 863a30e903f04d42a7859a5b79bf278fa58c0969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Dec 2023 12:09:39 +0800 Subject: [PATCH 397/398] feat: +pylint class view --- metagpt/actions/action_node.py | 10 +- metagpt/actions/rebuild_class_view.py | 68 +++++ metagpt/actions/rebuild_class_view_an.py | 33 +++ metagpt/const.py | 2 + metagpt/repo_parser.py | 270 +++++++++++++++--- metagpt/utils/common.py | 23 +- metagpt/utils/di_graph_repository.py | 23 +- metagpt/utils/graph_repository.py | 112 +++++++- requirements.txt | 3 +- .../actions/test_rebuild_class_view.py | 24 ++ tests/metagpt/test_repo_parser.py | 0 .../metagpt/utils/test_di_graph_repository.py | 58 +--- 12 files changed, 528 insertions(+), 98 deletions(-) create mode 100644 metagpt/actions/rebuild_class_view.py create mode 100644 metagpt/actions/rebuild_class_view_an.py create mode 100644 tests/metagpt/actions/test_rebuild_class_view.py create mode 100644 tests/metagpt/test_repo_parser.py diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 6f1215920..4ed8bf22e 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -39,7 +39,7 @@ SIMPLE_TEMPLATE = """ {constraint} ## action -Fill in the above nodes based on the format example. +Based on the 'context' content, fill in the {node_name} using the 'format example' format above." """ @@ -247,8 +247,13 @@ class ActionNode: # FIXME: json instruction会带来格式问题，如："Project name": "web_2048 # 项目名称使用下划线", self.instruction = self.compile_instruction(to="markdown", mode=mode) self.example = self.compile_example(to=to, tag="CONTENT", mode=mode) + node_name = "nodes" if template != SIMPLE_TEMPLATE else f'"{list(self.children.keys())[0]}" node' prompt = template.format( - context=context, example=self.example, instruction=self.instruction, constraint=CONSTRAINT + context=context, + example=self.example, + instruction=self.instruction, + constraint=CONSTRAINT, + node_name=node_name, ) return prompt @@ -302,6 +307,7 @@ class ActionNode: mapping = self.get_mapping(mode) class_name = f"{self.key}_AN" + print(prompt) output = await self._aask_v1(prompt, class_name, mapping, format=to) self.content = output.content self.instruct_content = output.instruct_content diff --git a/metagpt/actions/rebuild_class_view.py b/metagpt/actions/rebuild_class_view.py new file mode 100644 index 000000000..6da3e2989 --- /dev/null +++ b/metagpt/actions/rebuild_class_view.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/19 +@Author : mashenquan +@File : rebuild_class_view.py +@Desc : Rebuild class view info +""" +import re +from pathlib import Path + +from metagpt.actions import Action +from metagpt.config import CONFIG +from metagpt.const import CLASS_VIEW_FILE_REPO, GRAPH_REPO_FILE_REPO +from metagpt.repo_parser import RepoParser +from metagpt.utils.di_graph_repository import DiGraphRepository +from metagpt.utils.graph_repository import GraphKeyword, GraphRepository + + +class RebuildClassView(Action): + def __init__(self, name="", context=None, llm=None): + super().__init__(name=name, context=context, llm=llm) + + async def run(self, with_messages=None, format=CONFIG.prompt_format): + graph_repo_pathname = CONFIG.git_repo.workdir / GRAPH_REPO_FILE_REPO / CONFIG.git_repo.workdir.name + graph_db = await DiGraphRepository.load_from(str(graph_repo_pathname.with_suffix(".json"))) + repo_parser = RepoParser(base_directory=self.context) + class_views = await repo_parser.rebuild_class_views(path=Path(self.context)) # use pylint + await GraphRepository.update_graph_db_with_class_views(graph_db, class_views) + symbols = repo_parser.generate_symbols() # use ast + for file_info in symbols: + await GraphRepository.update_graph_db_with_file_info(graph_db, file_info) + await self._create_mermaid_class_view(graph_db=graph_db) + await self._save(graph_db=graph_db) + + async def _create_mermaid_class_view(self, graph_db): + pass + # dataset = await graph_db.select(subject=concat_namespace(filename, class_name), predicate=GraphKeyword.HAS_PAGE_INFO) + # if not dataset: + # logger.warning(f"No page info for {concat_namespace(filename, class_name)}") + # return + # code_block_info = CodeBlockInfo.parse_raw(dataset[0].object_) + # src_code = await read_file_block(filename=Path(self.context) / filename, lineno=code_block_info.lineno, end_lineno=code_block_info.end_lineno) + # code_type = "" + # dataset = await graph_db.select(subject=filename, predicate=GraphKeyword.IS) + # for spo in dataset: + # if spo.object_ in ["javascript", "python"]: + # code_type = spo.object_ + # break + + # try: + # node = await REBUILD_CLASS_VIEW_NODE.fill(context=f"```{code_type}\n{src_code}\n```", llm=self.llm, to=format) + # class_view = node.instruct_content.dict()["Class View"] + # except Exception as e: + # class_view = RepoParser.rebuild_class_view(src_code, code_type) + # await graph_db.insert(subject=concat_namespace(filename, class_name), predicate=GraphKeyword.HAS_CLASS_VIEW, object_=class_view) + # logger.info(f"{concat_namespace(filename, class_name)} {GraphKeyword.HAS_CLASS_VIEW} {class_view}") + + async def _save(self, graph_db): + class_view_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CLASS_VIEW_FILE_REPO) + dataset = await graph_db.select(predicate=GraphKeyword.HAS_CLASS_VIEW) + all_class_view = [] + for spo in dataset: + title = f"---\ntitle: {spo.subject}\n---\n" + filename = re.sub(r"[/:]", "_", spo.subject) + ".mmd" + await class_view_file_repo.save(filename=filename, content=title + spo.object_) + all_class_view.append(spo.object_) + await class_view_file_repo.save(filename="all.mmd", content="\n".join(all_class_view)) diff --git a/metagpt/actions/rebuild_class_view_an.py b/metagpt/actions/rebuild_class_view_an.py new file mode 100644 index 000000000..da32a9b5e --- /dev/null +++ b/metagpt/actions/rebuild_class_view_an.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/19 +@Author : mashenquan +@File : rebuild_class_view_an.py +@Desc : Defines `ActionNode` objects used by rebuild_class_view.py +""" +from metagpt.actions.action_node import ActionNode + +CLASS_SOURCE_CODE_BLOCK = ActionNode( + key="Class View", + expected_type=str, + instruction='Generate the mermaid class diagram corresponding to source code in "context."', + example=""" + classDiagram + class A { + -int x + +int y + -int speed + -int direction + +__init__(x: int, y: int, speed: int, direction: int) + +change_direction(new_direction: int) None + +move() None + } + """, +) + +REBUILD_CLASS_VIEW_NODES = [ + CLASS_SOURCE_CODE_BLOCK, +] + +REBUILD_CLASS_VIEW_NODE = ActionNode.from_children("RebuildClassView", REBUILD_CLASS_VIEW_NODES) diff --git a/metagpt/const.py b/metagpt/const.py index fcb3a2b3e..53f797001 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -99,6 +99,8 @@ CODE_SUMMARIES_FILE_REPO = "docs/code_summaries" CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries" RESOURCES_FILE_REPO = "resources" SD_OUTPUT_FILE_REPO = "resources/SD_Output" +GRAPH_REPO_FILE_REPO = "docs/graph_repo" +CLASS_VIEW_FILE_REPO = "docs/class_views" YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/repo_parser.py b/metagpt/repo_parser.py index 03cf7be79..ff34257a6 100644 --- a/metagpt/repo_parser.py +++ b/metagpt/repo_parser.py @@ -9,10 +9,13 @@ from __future__ import annotations import ast import json +import re +import subprocess from pathlib import Path from pprint import pformat -from typing import List +from typing import Dict, List, Optional, Tuple +import aiofiles import pandas as pd from pydantic import BaseModel, Field @@ -22,6 +25,29 @@ from metagpt.utils.common import any_to_str from metagpt.utils.exceptions import handle_exception +class RepoFileInfo(BaseModel): + file: str + classes: List = Field(default_factory=list) + functions: List = Field(default_factory=list) + globals: List = Field(default_factory=list) + page_info: List = Field(default_factory=list) + + +class CodeBlockInfo(BaseModel): + lineno: int + end_lineno: int + type_name: str + tokens: List = Field(default_factory=list) + properties: Dict = Field(default_factory=dict) + + +class ClassInfo(BaseModel): + name: str + package: Optional[str] = None + attributes: Dict[str, str] = Field(default_factory=dict) + methods: Dict[str, str] = Field(default_factory=dict) + + class RepoParser(BaseModel): base_directory: Path = Field(default=None) @@ -31,32 +57,24 @@ class RepoParser(BaseModel): """Parse a Python file in the repository.""" return ast.parse(file_path.read_text()).body - def extract_class_and_function_info(self, tree, file_path): + def extract_class_and_function_info(self, tree, file_path) -> RepoFileInfo: """Extract class, function, and global variable information from the AST.""" - file_info = { - "file": str(file_path.relative_to(self.base_directory)), - "classes": [], - "functions": [], - "globals": [], - } - - page_info = [] + file_info = RepoFileInfo(file=str(file_path.relative_to(self.base_directory))) for node in tree: info = RepoParser.node_to_str(node) - page_info.append(info) + file_info.page_info.append(info) if isinstance(node, ast.ClassDef): class_methods = [m.name for m in node.body if is_func(m)] - file_info["classes"].append({"name": node.name, "methods": class_methods}) + file_info.classes.append({"name": node.name, "methods": class_methods}) elif is_func(node): - file_info["functions"].append(node.name) + file_info.functions.append(node.name) elif isinstance(node, (ast.Assign, ast.AnnAssign)): for target in node.targets if isinstance(node, ast.Assign) else [node.target]: if isinstance(target, ast.Name): - file_info["globals"].append(target.id) - file_info["page_info"] = page_info + file_info.globals.append(target.id) return file_info - def generate_symbols(self): + def generate_symbols(self) -> List[RepoFileInfo]: files_classes = [] directory = self.base_directory @@ -93,37 +111,213 @@ class RepoParser(BaseModel): self.generate_dataframe_structure(output_path) @staticmethod - def node_to_str(node) -> (int, int, str, str | List): - def _parse_name(n): - if n.asname: - return f"{n.name} as {n.asname}" - return n.name - + def node_to_str(node) -> (int, int, str, str | Tuple): if any_to_str(node) == any_to_str(ast.Expr): - return node.lineno, node.end_lineno, any_to_str(node), RepoParser._parse_expr(node) + return CodeBlockInfo( + lineno=node.lineno, + end_lineno=node.end_lineno, + type_name=any_to_str(node), + tokens=RepoParser._parse_expr(node), + ) mappings = { - any_to_str(ast.Import): lambda x: [_parse_name(n) for n in x.names], - any_to_str(ast.Assign): lambda x: [n.id for n in x.targets], + any_to_str(ast.Import): lambda x: [RepoParser._parse_name(n) for n in x.names], + any_to_str(ast.Assign): RepoParser._parse_assign, any_to_str(ast.ClassDef): lambda x: x.name, any_to_str(ast.FunctionDef): lambda x: x.name, - any_to_str(ast.ImportFrom): lambda x: {"module": x.module, "names": [_parse_name(n) for n in x.names]}, - any_to_str(ast.If): lambda x: x.test.left.id, + any_to_str(ast.ImportFrom): lambda x: { + "module": x.module, + "names": [RepoParser._parse_name(n) for n in x.names], + }, + any_to_str(ast.If): RepoParser._parse_if, + any_to_str(ast.AsyncFunctionDef): lambda x: x.name, } func = mappings.get(any_to_str(node)) if func: - return node.lineno, node.end_lineno, any_to_str(node), func(node) - return node.lineno, node.end_lineno, any_to_str(node), None + code_block = CodeBlockInfo(lineno=node.lineno, end_lineno=node.end_lineno, type_name=any_to_str(node)) + val = func(node) + if isinstance(val, dict): + code_block.properties = val + elif isinstance(val, list): + code_block.tokens = val + elif isinstance(val, str): + code_block.tokens = [val] + else: + raise NotImplementedError(f"Not implement:{val}") + return code_block + raise NotImplementedError(f"Not implement code block:{node.lineno}, {node.end_lineno}, {any_to_str(node)}") @staticmethod - def _parse_expr(node) -> (int, int, str, str | List): - if isinstance(node.value, ast.Constant): - return any_to_str(ast.Constant), node.value.value - if isinstance(node.value, ast.Call): - if isinstance(node.value.func, ast.Attribute): - return any_to_str(ast.Call), f"{node.value.func.value.id}.{node.value.func.attr}" - if isinstance(node.value.func, ast.Name): - return any_to_str(ast.Call), node.value.func.id - return any_to_str(node.value), None + def _parse_expr(node) -> List: + funcs = { + any_to_str(ast.Constant): lambda x: [any_to_str(x.value), RepoParser._parse_variable(x.value)], + any_to_str(ast.Call): lambda x: [any_to_str(x.value), RepoParser._parse_variable(x.value.func)], + } + func = funcs.get(any_to_str(node.value)) + if func: + return func(node) + raise NotImplementedError(f"Not implement: {node.value}") + + @staticmethod + def _parse_name(n): + if n.asname: + return f"{n.name} as {n.asname}" + return n.name + + @staticmethod + def _parse_if(n): + tokens = [RepoParser._parse_variable(n.test.left)] + for item in n.test.comparators: + tokens.append(RepoParser._parse_variable(item)) + return tokens + + @staticmethod + def _parse_variable(node): + funcs = { + any_to_str(ast.Constant): lambda x: x.value, + any_to_str(ast.Name): lambda x: x.id, + any_to_str(ast.Attribute): lambda x: f"{x.value.id}.{x.attr}", + } + func = funcs.get(any_to_str(node)) + if not func: + raise NotImplementedError(f"Not implement:{node}") + return func(node) + + @staticmethod + def _parse_assign(node): + return [RepoParser._parse_variable(t) for t in node.targets] + + async def rebuild_class_views(self, path: str | Path = None): + if not path: + path = self.base_directory + path = Path(path) + if not path.exists(): + return + command = f"pyreverse {str(path)} -o dot" + result = subprocess.run(command, shell=True, check=True, cwd=str(path)) + if result.returncode != 0: + raise ValueError(f"{result}") + class_view_pathname = path / "classes.dot" + class_views = await self._parse_classes(class_view_pathname) + packages_pathname = path / "packages.dot" + class_views = RepoParser._repair_namespaces(class_views=class_views, path=path) + class_view_pathname.unlink(missing_ok=True) + packages_pathname.unlink(missing_ok=True) + return class_views + + async def _parse_classes(self, class_view_pathname): + class_views = [] + if not class_view_pathname.exists(): + return class_views + async with aiofiles.open(str(class_view_pathname), mode="r") as reader: + lines = await reader.readlines() + for line in lines: + package_name, info = RepoParser._split_class_line(line) + if not package_name: + continue + class_name, members, functions = re.split(r"(?" + if begin_flag not in left or end_flag not in left: + return None, None + bix = left.find(begin_flag) + eix = left.rfind(end_flag) + info = left[bix + len(begin_flag) : eix] + info = re.sub(r"]*>", "\n", info) + return class_name, info + + @staticmethod + def _create_path_mapping(path: str | Path) -> Dict[str, str]: + mappings = { + str(path).replace("/", "."): str(path), + } + files = [] + try: + directory_path = Path(path) + if not directory_path.exists(): + return mappings + for file_path in directory_path.iterdir(): + if file_path.is_file(): + files.append(str(file_path)) + else: + subfolder_files = RepoParser._create_path_mapping(path=file_path) + mappings.update(subfolder_files) + except Exception as e: + logger.error(f"Error: {e}") + for f in files: + mappings[str(Path(f).with_suffix("")).replace("/", ".")] = str(f) + + return mappings + + @staticmethod + def _repair_namespaces(class_views: List[ClassInfo], path: str | Path) -> List[ClassInfo]: + if not class_views: + return [] + c = class_views[0] + full_key = str(path).lstrip("/").replace("/", ".") + root_namespace = RepoParser._find_root(full_key, c.package) + root_path = root_namespace.replace(".", "/") + + mappings = RepoParser._create_path_mapping(path=path) + new_mappings = {} + ix_root_namespace = len(root_namespace) + ix_root_path = len(root_path) + for k, v in mappings.items(): + nk = k[ix_root_namespace:] + nv = v[ix_root_path:] + new_mappings[nk] = nv + + for c in class_views: + c.package = RepoParser._repair_ns(c.package, new_mappings) + return class_views + + @staticmethod + def _repair_ns(package, mappings): + file_ns = package + while file_ns != "": + if file_ns not in mappings: + ix = file_ns.rfind(".") + file_ns = file_ns[0:ix] + continue + break + internal_ns = package[ix + 1 :] + ns = mappings[file_ns] + ":" + internal_ns.replace(".", ":") + return ns + + @staticmethod + def _find_root(full_key, package) -> str: + left = full_key + while left != "": + if left in package: + break + if "." not in left: + break + ix = left.find(".") + left = left[ix + 1 :] + ix = full_key.rfind(left) + return "." + full_key[0:ix] def is_func(node): diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 8fa729556..a5d2100cc 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -17,8 +17,8 @@ import inspect import os import platform import re -import typing -from typing import List, Tuple, Union +from pathlib import Path +from typing import Callable, List, Tuple, Union import aiofiles import loguru @@ -332,7 +332,7 @@ def get_class_name(cls) -> str: return f"{cls.__module__}.{cls.__name__}" -def any_to_str(val: str | typing.Callable) -> str: +def any_to_str(val: str | Callable) -> str: """Return the class name or the class name of the object, or 'val' if it's a string type.""" if isinstance(val, str): return val @@ -443,3 +443,20 @@ async def aread(file_path: str) -> str: async with aiofiles.open(str(file_path), mode="r") as reader: content = await reader.read() return content + + +async def read_file_block(filename: str | Path, lineno: int, end_lineno: int): + if not Path(filename).exists(): + return "" + lines = [] + async with aiofiles.open(str(filename), mode="r") as reader: + ix = 0 + while ix < end_lineno: + ix += 1 + line = await reader.readline() + if ix < lineno: + continue + if ix > end_lineno: + break + lines.append(line) + return "".join(lines) diff --git a/metagpt/utils/di_graph_repository.py b/metagpt/utils/di_graph_repository.py index 9bbd38d5f..08f4327fa 100644 --- a/metagpt/utils/di_graph_repository.py +++ b/metagpt/utils/di_graph_repository.py @@ -10,11 +10,12 @@ from __future__ import annotations import json from pathlib import Path +from typing import List import aiofiles import networkx -from metagpt.utils.graph_repository import GraphRepository +from metagpt.utils.graph_repository import SPO, GraphRepository class DiGraphRepository(GraphRepository): @@ -31,6 +32,18 @@ class DiGraphRepository(GraphRepository): async def update(self, subject: str, predicate: str, object_: str): pass + async def select(self, subject: str = None, predicate: str = None, object_: str = None) -> List[SPO]: + result = [] + for s, o, p in self._repo.edges(data="predicate"): + if subject and subject != s: + continue + if predicate and predicate != p: + continue + if object_ and object_ != o: + continue + result.append(SPO(subject=s, predicate=p, object_=o)) + return result + def json(self) -> str: m = networkx.node_link_data(self._repo) data = json.dumps(m) @@ -53,10 +66,12 @@ class DiGraphRepository(GraphRepository): @staticmethod async def load_from(pathname: str | Path) -> GraphRepository: - name = Path(pathname).with_suffix("").name - root = Path(pathname).parent + pathname = Path(pathname) + name = pathname.with_suffix("").name + root = pathname.parent graph = DiGraphRepository(name=name, root=root) - await graph.load(pathname=pathname) + if pathname.exists(): + await graph.load(pathname=pathname) return graph @property diff --git a/metagpt/utils/graph_repository.py b/metagpt/utils/graph_repository.py index 600575b4e..37da3dee4 100644 --- a/metagpt/utils/graph_repository.py +++ b/metagpt/utils/graph_repository.py @@ -6,18 +6,38 @@ @File : graph_repository.py @Desc : Superclass for graph repository. """ + from abc import ABC, abstractmethod -from enum import Enum +from pathlib import Path +from typing import List + +from pydantic import BaseModel + +from metagpt.repo_parser import ClassInfo, RepoFileInfo +from metagpt.utils.common import concat_namespace -class GraphKeyword(Enum): +class GraphKeyword: IS = "is" CLASS = "class" FUNCTION = "function" + SOURCE_CODE = "source_code" + NULL = "" GLOBAL_VARIABLE = "global_variable" CLASS_FUNCTION = "class_function" CLASS_PROPERTY = "class_property" HAS_CLASS = "has_class" + HAS_PAGE_INFO = "has_page_info" + HAS_CLASS_VIEW = "has_class_view" + HAS_SEQUENCE_VIEW = "has_sequence_view" + HAS_ARGS_DESC = "has_args_desc" + HAS_TYPE_DESC = "has_type_desc" + + +class SPO(BaseModel): + subject: str + predicate: str + object_: str class GraphRepository(ABC): @@ -37,6 +57,94 @@ class GraphRepository(ABC): async def update(self, subject: str, predicate: str, object_: str): pass + @abstractmethod + async def select(self, subject: str = None, predicate: str = None, object_: str = None) -> List[SPO]: + pass + @property def name(self) -> str: return self._repo_name + + @staticmethod + async def update_graph_db_with_file_info(graph_db: "GraphRepository", file_info: RepoFileInfo): + await graph_db.insert(subject=file_info.file, predicate=GraphKeyword.IS, object_=GraphKeyword.SOURCE_CODE) + file_types = {".py": "python", ".js": "javascript"} + file_type = file_types.get(Path(file_info.file).suffix, GraphKeyword.NULL) + await graph_db.insert(subject=file_info.file, predicate=GraphKeyword.IS, object_=file_type) + for c in file_info.classes: + class_name = c.get("name", "") + await graph_db.insert( + subject=file_info.file, + predicate=GraphKeyword.HAS_CLASS, + object_=concat_namespace(file_info.file, class_name), + ) + await graph_db.insert( + subject=concat_namespace(file_info.file, class_name), + predicate=GraphKeyword.IS, + object_=GraphKeyword.CLASS, + ) + methods = c.get("methods", []) + for fn in methods: + await graph_db.insert( + subject=concat_namespace(file_info.file, class_name, fn), + predicate=GraphKeyword.IS, + object_=GraphKeyword.CLASS_FUNCTION, + ) + for f in file_info.functions: + await graph_db.insert( + subject=concat_namespace(file_info.file, f), predicate=GraphKeyword.IS, object_=GraphKeyword.FUNCTION + ) + for g in file_info.globals: + await graph_db.insert( + subject=concat_namespace(file_info.file, g), + predicate=GraphKeyword.IS, + object_=GraphKeyword.GLOBAL_VARIABLE, + ) + for code_block in file_info.page_info: + if code_block.tokens: + await graph_db.insert( + subject=concat_namespace(file_info.file, *code_block.tokens), + predicate=GraphKeyword.HAS_PAGE_INFO, + object_=code_block.json(ensure_ascii=False), + ) + for k, v in code_block.properties.items(): + await graph_db.insert( + subject=concat_namespace(file_info.file, k, v), + predicate=GraphKeyword.HAS_PAGE_INFO, + object_=code_block.json(ensure_ascii=False), + ) + + @staticmethod + async def update_graph_db_with_class_views(graph_db: "GraphRepository", class_views: List[ClassInfo]): + for c in class_views: + filename, class_name = c.package.split(":", 1) + await graph_db.insert(subject=filename, predicate=GraphKeyword.IS, object_=GraphKeyword.SOURCE_CODE) + file_types = {".py": "python", ".js": "javascript"} + file_type = file_types.get(Path(filename).suffix, GraphKeyword.NULL) + await graph_db.insert(subject=filename, predicate=GraphKeyword.IS, object_=file_type) + await graph_db.insert(subject=filename, predicate=GraphKeyword.HAS_CLASS, object_=class_name) + await graph_db.insert( + subject=c.package, + predicate=GraphKeyword.IS, + object_=GraphKeyword.CLASS, + ) + for vn, vt in c.attributes.items(): + await graph_db.insert( + subject=concat_namespace(c.package, vn), + predicate=GraphKeyword.IS, + object_=GraphKeyword.CLASS_PROPERTY, + ) + await graph_db.insert( + subject=concat_namespace(c.package, vn), predicate=GraphKeyword.HAS_TYPE_DESC, object_=vt + ) + for fn, desc in c.methods.items(): + await graph_db.insert( + subject=concat_namespace(c.package, fn), + predicate=GraphKeyword.IS, + object_=GraphKeyword.CLASS_FUNCTION, + ) + await graph_db.insert( + subject=concat_namespace(c.package, fn), + predicate=GraphKeyword.HAS_ARGS_DESC, + object_=desc, + ) diff --git a/requirements.txt b/requirements.txt index 4310aec6c..c4e674569 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,4 +57,5 @@ socksio~=1.0.0 gitignore-parser==0.1.9 connexion[swagger-ui] websockets~=12.0 -networkx~=3.2.1 \ No newline at end of file +networkx~=3.2.1 +pylint~=3.0.3 \ No newline at end of file diff --git a/tests/metagpt/actions/test_rebuild_class_view.py b/tests/metagpt/actions/test_rebuild_class_view.py new file mode 100644 index 000000000..955c6ae3b --- /dev/null +++ b/tests/metagpt/actions/test_rebuild_class_view.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/20 +@Author : mashenquan +@File : test_rebuild_class_view.py +@Desc : Unit tests for rebuild_class_view.py +""" +from pathlib import Path + +import pytest + +from metagpt.actions.rebuild_class_view import RebuildClassView +from metagpt.llm import LLM + + +@pytest.mark.asyncio +async def test_rebuild(): + action = RebuildClassView(name="RedBean", context=Path(__file__).parent.parent, llm=LLM()) + await action.run() + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_repo_parser.py b/tests/metagpt/test_repo_parser.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/metagpt/utils/test_di_graph_repository.py b/tests/metagpt/utils/test_di_graph_repository.py index ec2cb4d01..0a8011e51 100644 --- a/tests/metagpt/utils/test_di_graph_repository.py +++ b/tests/metagpt/utils/test_di_graph_repository.py @@ -14,9 +14,8 @@ from pydantic import BaseModel from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.repo_parser import RepoParser -from metagpt.utils.common import concat_namespace from metagpt.utils.di_graph_repository import DiGraphRepository -from metagpt.utils.graph_repository import GraphKeyword +from metagpt.utils.graph_repository import GraphRepository @pytest.mark.asyncio @@ -57,23 +56,7 @@ async def test_js_parser(): repo_parser = RepoParser(base_directory=data.path) symbols = repo_parser.generate_symbols() for s in symbols: - ns = s.get("file", "") - for c in s.get("classes", []): - await graph.insert( - subject=concat_namespace(ns, c), predicate=GraphKeyword.IS.value, object_=GraphKeyword.CLASS.value - ) - for f in s.get("functions", []): - await graph.insert( - subject=concat_namespace(ns, f), - predicate=GraphKeyword.IS.value, - object_=GraphKeyword.FUNCTION.value, - ) - for g in s.get("globals", []): - await graph.insert( - subject=concat_namespace(ns, g), - predicate=GraphKeyword.IS.value, - object_=GraphKeyword.GLOBAL_VARIABLE.value, - ) + await GraphRepository.update_graph_db(graph_db=graph, file_info=s) data = graph.json() assert data @@ -85,35 +68,14 @@ async def test_codes(): graph = DiGraphRepository(name="test", root=path) symbols = repo_parser.generate_symbols() - for s in symbols: - ns = s.get("file", "") - for c in s.get("classes", []): - class_name = c.get("name", "") - await graph.insert( - subject=ns, predicate=GraphKeyword.HAS_CLASS.value, object_=concat_namespace(ns, class_name) - ) - await graph.insert( - subject=concat_namespace(ns, class_name), - predicate=GraphKeyword.IS.value, - object_=GraphKeyword.CLASS.value, - ) - methods = c.get("methods", []) - for fn in methods: - await graph.insert( - subject=concat_namespace(ns, class_name, fn), - predicate=GraphKeyword.IS.value, - object_=GraphKeyword.CLASS_FUNCTION.value, - ) - for f in s.get("functions", []): - await graph.insert( - subject=concat_namespace(ns, f), predicate=GraphKeyword.IS.value, object_=GraphKeyword.FUNCTION.value - ) - for g in s.get("globals", []): - await graph.insert( - subject=concat_namespace(ns, g), - predicate=GraphKeyword.IS.value, - object_=GraphKeyword.GLOBAL_VARIABLE.value, - ) + for file_info in symbols: + for code_block in file_info.page_info: + try: + val = code_block.json(ensure_ascii=False) + assert val + except TypeError as e: + assert not e + await GraphRepository.update_graph_db(graph_db=graph, file_info=file_info) data = graph.json() assert data print(data) From bf15613f632b73d21ce7ce354730d75d3177f79b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 22 Dec 2023 16:50:04 +0800 Subject: [PATCH 398/398] feat: merge geekan:main --- metagpt/actions/write_teaching_plan.py | 20 +++++++++++++++++++- metagpt/utils/common.py | 19 ------------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 529c563db..534f5ded9 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -6,9 +6,9 @@ @File : write_teaching_plan.py """ from metagpt.actions import Action +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Message -from metagpt.utils.common import format_value class TeachingPlanRequirement(Action): @@ -81,6 +81,24 @@ class WriteTeachingPlanPart(Action): """Show `topic` value when debug""" return self.topic + @staticmethod + def format_value(value): + """Fill parameters inside `value` with `options`.""" + if not isinstance(value, str): + return value + if "{" not in value: + return value + + merged_opts = CONFIG.options or {} + try: + return value.format(**merged_opts) + except KeyError as e: + logger.warning(f"Parameter is missing:{e}") + + for k, v in merged_opts.items(): + value = value.replace("{" + f"{k}" + "}", str(v)) + return value + FORMATION = ( '"Capacity and role" defines the role you are currently playing;\n' '\t"[LESSON_BEGIN]" and "[LESSON_END]" tags enclose the content of textbook;\n' diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index a1cb71c6f..382523083 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -30,7 +30,6 @@ import loguru from pydantic.json import pydantic_encoder from tenacity import RetryCallState, _utils -from metagpt.config import CONFIG from metagpt.const import MESSAGE_ROUTE_TO_ALL from metagpt.logs import logger from metagpt.utils.exceptions import handle_exception @@ -418,24 +417,6 @@ def any_to_name(val): return any_to_str(val).split(".")[-1] -def format_value(value): - """Fill parameters inside `value` with `options`.""" - if not isinstance(value, str): - return value - if "{" not in value: - return value - - merged_opts = CONFIG.options or {} - try: - return value.format(**merged_opts) - except KeyError as e: - logger.warning(f"Parameter is missing:{e}") - - for k, v in merged_opts.items(): - value = value.replace("{" + f"{k}" + "}", str(v)) - return value - - def concat_namespace(*args) -> str: return ":".join(str(value) for value in args)