diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..32555a806 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.html linguist-detectable=false + diff --git a/.gitignore b/.gitignore index 1ddef530f..e03eab3d3 100644 --- a/.gitignore +++ b/.gitignore @@ -148,8 +148,7 @@ allure-results .DS_Store .vscode - -*.txt +log.txt docs/scripts/set_env.sh key.yaml output.json diff --git a/examples/agent_creator.py b/examples/agent_creator.py new file mode 100644 index 000000000..e03a88c6b --- /dev/null +++ b/examples/agent_creator.py @@ -0,0 +1,100 @@ +''' +Filename: MetaGPT/examples/agent_creator.py +Created Date: Tuesday, September 12th 2023, 3:28:37 pm +Author: garylin2099 +''' +import re + +from metagpt.const import PROJECT_ROOT, WORKSPACE_ROOT +from metagpt.actions import Action +from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.logs import logger + +with open(PROJECT_ROOT / "examples/build_customized_agent.py", "r") as f: + # use official example script to guide AgentCreator + MULTI_ACTION_AGENT_CODE_EXAMPLE = f.read() + +class CreateAgent(Action): + + PROMPT_TEMPLATE = """ + ### BACKGROUND + You are using an agent framework called metagpt to write agents capable of different actions, + the usage of metagpt can be illustrated by the following example: + ### EXAMPLE STARTS AT THIS LINE + {example} + ### EXAMPLE ENDS AT THIS LINE + ### TASK + Now you should create an agent with appropriate actions based on the instruction, consider carefully about + the PROMPT_TEMPLATE of all actions and when to call self._aask() + ### INSTRUCTION + {instruction} + ### YOUR CODE + Return ```python your_code_here ``` with NO other texts, your code: + """ + + async def run(self, example: str, instruction: str): + + prompt = self.PROMPT_TEMPLATE.format(example=example, instruction=instruction) + # logger.info(prompt) + + rsp = await self._aask(prompt) + + code_text = CreateAgent.parse_code(rsp) + + return code_text + + @staticmethod + def parse_code(rsp): + pattern = r'```python(.*)```' + match = re.search(pattern, rsp, re.DOTALL) + code_text = match.group(1) if match else "" + with open(WORKSPACE_ROOT / "agent_created_agent.py", "w") as f: + f.write(code_text) + return code_text + +class AgentCreator(Role): + def __init__( + self, + name: str = "Matrix", + profile: str = "AgentCreator", + agent_template: str = MULTI_ACTION_AGENT_CODE_EXAMPLE, + **kwargs, + ): + super().__init__(name, profile, **kwargs) + self._init_actions([CreateAgent]) + self.agent_template = agent_template + + async def _act(self) -> Message: + logger.info(f"{self._setting}: ready to {self._rc.todo}") + todo = self._rc.todo + msg = self._rc.memory.get()[-1] + + instruction = msg.content + code_text = await CreateAgent().run(example=self.agent_template, instruction=instruction) + msg = Message(content=code_text, role=self.profile, cause_by=todo) + + return msg + +if __name__ == "__main__": + import asyncio + + async def main(): + + agent_template = MULTI_ACTION_AGENT_CODE_EXAMPLE + + creator = AgentCreator(agent_template=agent_template) + + # msg = """Write an agent called SimpleTester that will take any code snippet (str) + # and return a testing code (str) for testing + # the given code snippet. Use pytest as the testing framework.""" + + msg = """ + Write an agent called SimpleTester that will take any code snippet (str) and do the following: + 1. write a testing code (str) for testing the given code snippet, save the testing code as a .py file in the current working diretory; + 2. run the testing code. + You can use pytest as the testing framework. + """ + await creator.run(msg) + + asyncio.run(main()) diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py new file mode 100644 index 000000000..87d7a9c76 --- /dev/null +++ b/examples/build_customized_agent.py @@ -0,0 +1,139 @@ +''' +Filename: MetaGPT/examples/build_customized_agent.py +Created Date: Tuesday, September 19th 2023, 6:52:25 pm +Author: garylin2099 +''' +import re +import subprocess +import asyncio + +import fire + +from metagpt.actions import Action +from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.logs import logger + +class SimpleWriteCode(Action): + + PROMPT_TEMPLATE = """ + Write a python function that can {instruction} and provide two runnnable test cases. + Return ```python your_code_here ``` with NO other texts, + example: + ```python + # function + def add(a, b): + return a + b + # test cases + print(add(1, 2)) + print(add(3, 4)) + ``` + your code: + """ + + def __init__(self, name="SimpleWriteCode", context=None, llm=None): + super().__init__(name, context, llm) + + async def run(self, instruction: str): + + prompt = self.PROMPT_TEMPLATE.format(instruction=instruction) + + rsp = await self._aask(prompt) + + code_text = SimpleWriteCode.parse_code(rsp) + + return code_text + + @staticmethod + def parse_code(rsp): + pattern = r'```python(.*)```' + match = re.search(pattern, rsp, re.DOTALL) + code_text = match.group(1) if match else rsp + return code_text + +class SimpleRunCode(Action): + def __init__(self, name="SimpleRunCode", context=None, llm=None): + super().__init__(name, context, llm) + + async def run(self, code_text: str): + result = subprocess.run(["python3", "-c", code_text], capture_output=True, text=True) + code_result = result.stdout + logger.info(f"{code_result=}") + return code_result + +class SimpleCoder(Role): + def __init__( + self, + name: str = "Alice", + profile: str = "SimpleCoder", + **kwargs, + ): + super().__init__(name, profile, **kwargs) + self._init_actions([SimpleWriteCode]) + + async def _act(self) -> Message: + logger.info(f"{self._setting}: ready to {self._rc.todo}") + todo = self._rc.todo + + msg = self._rc.memory.get()[-1] # retrieve the latest memory + instruction = msg.content + + code_text = await SimpleWriteCode().run(instruction) + msg = Message(content=code_text, role=self.profile, cause_by=todo) + + return msg + +class RunnableCoder(Role): + def __init__( + self, + name: str = "Alice", + profile: str = "RunnableCoder", + **kwargs, + ): + super().__init__(name, profile, **kwargs) + self._init_actions([SimpleWriteCode, SimpleRunCode]) + + async def _think(self) -> None: + if self._rc.todo is None: + self._set_state(0) + return + + if self._rc.state + 1 < len(self._states): + self._set_state(self._rc.state + 1) + else: + self._rc.todo = None + + async def _act(self) -> Message: + logger.info(f"{self._setting}: ready to {self._rc.todo}") + todo = self._rc.todo + msg = self._rc.memory.get()[-1] + + if isinstance(todo, SimpleWriteCode): + instruction = msg.content + result = await SimpleWriteCode().run(instruction) + + elif isinstance(todo, SimpleRunCode): + code_text = msg.content + result = await SimpleRunCode().run(code_text) + + msg = Message(content=result, role=self.profile, cause_by=todo) + self._rc.memory.add(msg) + return msg + + async def _react(self) -> Message: + while True: + await self._think() + if self._rc.todo is None: + break + await self._act() + return Message(content="All job done", role=self.profile) + +def main(msg="write a function that calculates the sum of a list"): + # role = SimpleCoder() + role = RunnableCoder() + logger.info(msg) + result = asyncio.run(role.run(msg)) + logger.info(result) + +if __name__ == '__main__': + fire.Fire(main) diff --git a/examples/debate.py b/examples/debate.py new file mode 100644 index 000000000..05db28070 --- /dev/null +++ b/examples/debate.py @@ -0,0 +1,148 @@ +''' +Filename: MetaGPT/examples/debate.py +Created Date: Tuesday, September 19th 2023, 6:52:25 pm +Author: garylin2099 +''' +import asyncio +import platform +import fire + +from metagpt.software_company import SoftwareCompany +from metagpt.actions import Action, BossRequirement +from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.logs import logger + +class ShoutOut(Action): + """Action: Shout out loudly in a debate (quarrel)""" + + PROMPT_TEMPLATE = """ + ## BACKGROUND + Suppose you are {name}, you are in a debate with {opponent_name}. + ## DEBATE HISTORY + Previous rounds: + {context} + ## YOUR TURN + Now it's your turn, you should closely respond to your opponent's latest argument, state your position, defend your arguments, and attack your opponent's arguments, + craft a strong and emotional response in 80 words, in {name}'s rhetoric and viewpoints, your will argue: + """ + + def __init__(self, name="ShoutOut", context=None, llm=None): + super().__init__(name, context, llm) + + async def run(self, context: str, name: str, opponent_name: str): + + prompt = self.PROMPT_TEMPLATE.format(context=context, name=name, opponent_name=opponent_name) + # logger.info(prompt) + + rsp = await self._aask(prompt) + + return rsp + +class Trump(Role): + def __init__( + self, + name: str = "Trump", + profile: str = "Republican", + **kwargs, + ): + super().__init__(name, profile, **kwargs) + self._init_actions([ShoutOut]) + self._watch([ShoutOut]) + self.name = "Trump" + self.opponent_name = "Biden" + + async def _observe(self) -> int: + await super()._observe() + # accept messages sent (from opponent) to self, disregard own messages from the last round + self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] + return len(self._rc.news) + + async def _act(self) -> Message: + logger.info(f"{self._setting}: ready to {self._rc.todo}") + + msg_history = self._rc.memory.get_by_actions([ShoutOut]) + context = [] + for m in msg_history: + context.append(str(m)) + context = "\n".join(context) + + rsp = await ShoutOut().run(context=context, name=self.name, opponent_name=self.opponent_name) + + msg = Message( + content=rsp, + role=self.profile, + cause_by=ShoutOut, + sent_from=self.name, + send_to=self.opponent_name, + ) + + return msg + +class Biden(Role): + def __init__( + self, + name: str = "Biden", + profile: str = "Democrat", + **kwargs, + ): + super().__init__(name, profile, **kwargs) + self._init_actions([ShoutOut]) + self._watch([BossRequirement, ShoutOut]) + self.name = "Biden" + self.opponent_name = "Trump" + + async def _observe(self) -> int: + await super()._observe() + # accept the very first human instruction (the debate topic) or messages sent (from opponent) to self, + # disregard own messages from the last round + self._rc.news = [msg for msg in self._rc.news if msg.cause_by == BossRequirement or msg.send_to == self.name] + return len(self._rc.news) + + async def _act(self) -> Message: + logger.info(f"{self._setting}: ready to {self._rc.todo}") + + msg_history = self._rc.memory.get_by_actions([BossRequirement, ShoutOut]) + context = [] + for m in msg_history: + context.append(str(m)) + context = "\n".join(context) + + rsp = await ShoutOut().run(context=context, name=self.name, opponent_name=self.opponent_name) + + msg = Message( + content=rsp, + role=self.profile, + cause_by=ShoutOut, + sent_from=self.name, + send_to=self.opponent_name, + ) + + return msg + +async def startup(idea: str, investment: float = 3.0, n_round: int = 5, + code_review: bool = False, run_tests: bool = False): + """We reuse the startup paradigm for roles to interact with each other. + Now we run a startup of presidents and watch they quarrel. :) """ + company = SoftwareCompany() + company.hire([Biden(), Trump()]) + company.invest(investment) + company.start_project(idea) + await company.run(n_round=n_round) + + +def main(idea: str, investment: float = 3.0, n_round: int = 10): + """ + :param idea: Debate topic, such as "Topic: The U.S. should commit more in climate change fighting" + or "Trump: Climate change is a hoax" + :param investment: contribute a certain dollar amount to watch the debate + :param n_round: maximum rounds of the debate + :return: + """ + if platform.system() == "Windows": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + asyncio.run(startup(idea, investment, n_round)) + + +if __name__ == '__main__': + fire.Fire(main) diff --git a/examples/sk_agent.py b/examples/sk_agent.py new file mode 100644 index 000000000..f60e7299b --- /dev/null +++ b/examples/sk_agent.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/13 12:36 +@Author : femto Zheng +@File : sk_agent.py +""" +import asyncio + +from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill +from semantic_kernel.planning import SequentialPlanner + +# from semantic_kernel.planning import SequentialPlanner +from semantic_kernel.planning.action_planner.action_planner import ActionPlanner + +from metagpt.actions import BossRequirement +from metagpt.const import SKILL_DIRECTORY +from metagpt.roles.sk_agent import SkAgent +from metagpt.schema import Message +from metagpt.tools.search_engine import SkSearchEngine + + +async def main(): + await basic_planner_example() + await action_planner_example() + + # await sequential_planner_example() + # await basic_planner_web_search_example() + + +async def basic_planner_example(): + task = """ + Tomorrow is Valentine's day. I need to come up with a few date ideas. She speaks French so write it in French. + Convert the text to uppercase""" + role = SkAgent() + + # let's give the agent some skills + role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "SummarizeSkill") + role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") + role.import_skill(TextSkill(), "TextSkill") + # using BasicPlanner + await role.run(Message(content=task, cause_by=BossRequirement)) + + +async def sequential_planner_example(): + task = """ + Tomorrow is Valentine's day. I need to come up with a few date ideas. She speaks French so write it in French. + Convert the text to uppercase""" + role = SkAgent(planner_cls=SequentialPlanner) + + # let's give the agent some skills + role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "SummarizeSkill") + role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") + role.import_skill(TextSkill(), "TextSkill") + # using BasicPlanner + await role.run(Message(content=task, cause_by=BossRequirement)) + + +async def basic_planner_web_search_example(): + task = """ + Question: Who made the 1989 comic book, the film version of which Jon Raymond Polito appeared in?""" + role = SkAgent() + + role.import_skill(SkSearchEngine(), "WebSearchSkill") + # role.import_semantic_skill_from_directory(skills_directory, "QASkill") + + await role.run(Message(content=task, cause_by=BossRequirement)) + + +async def action_planner_example(): + role = SkAgent(planner_cls=ActionPlanner) + # let's give the agent 4 skills + role.import_skill(MathSkill(), "math") + role.import_skill(FileIOSkill(), "fileIO") + role.import_skill(TimeSkill(), "time") + role.import_skill(TextSkill(), "text") + task = "What is the sum of 110 and 990?" + await role.run(Message(content=task, cause_by=BossRequirement)) # it will choose mathskill.Add + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/use_off_the_shelf_agent.py b/examples/use_off_the_shelf_agent.py new file mode 100644 index 000000000..2e10068bd --- /dev/null +++ b/examples/use_off_the_shelf_agent.py @@ -0,0 +1,18 @@ +''' +Filename: MetaGPT/examples/use_off_the_shelf_agent.py +Created Date: Tuesday, September 19th 2023, 6:52:25 pm +Author: garylin2099 +''' +import asyncio + +from metagpt.roles.product_manager import ProductManager +from metagpt.logs import logger + +async def main(): + msg = "Write a PRD for a snake game" + role = ProductManager() + result = await role.run(msg) + logger.info(result.content[:100]) + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/write_tutorial.py b/examples/write_tutorial.py index 167f3eb7c..71ece5527 100644 --- a/examples/write_tutorial.py +++ b/examples/write_tutorial.py @@ -17,4 +17,5 @@ async def main(): if __name__ == '__main__': - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) + diff --git a/metagpt/actions/clone_function.py b/metagpt/actions/clone_function.py new file mode 100644 index 000000000..cf7d22f04 --- /dev/null +++ b/metagpt/actions/clone_function.py @@ -0,0 +1,65 @@ +from pathlib import Path +import traceback + +from metagpt.actions.write_code import WriteCode +from metagpt.logs import logger +from metagpt.schema import Message +from metagpt.utils.highlight import highlight + +CLONE_PROMPT = """ +*context* +Please convert the function code ```{source_code}``` into the the function format: ```{template_func}```. +*Please Write code based on the following list and context* +1. Write code start with ```, and end with ```. +2. Please implement it in one function if possible, except for import statements. for exmaple: +```python +import pandas as pd +def run(*args) -> pd.DataFrame: + ... +``` +3. Do not use public member functions that do not exist in your design. +4. The output function name, input parameters and return value must be the same as ```{template_func}```. +5. Make sure the results before and after the code conversion are required to be exactly the same. +6. Don't repeat my context in your replies. +7. Return full results, for example, if the return value has df.head(), please return df. +8. If you must use a third-party package, use the most popular ones, for example: pandas, numpy, ta, ... +""" + + +class CloneFunction(WriteCode): + def __init__(self, name="CloneFunction", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) + + def _save(self, code_path, code): + if isinstance(code_path, str): + code_path = Path(code_path) + code_path.parent.mkdir(parents=True, exist_ok=True) + code_path.write_text(code) + logger.info(f"Saving Code to {code_path}") + + async def run(self, template_func: str, source_code: str) -> str: + """将source_code转换成template_func一样的入参和返回类型""" + prompt = CLONE_PROMPT.format(source_code=source_code, template_func=template_func) + logger.info(f"query for CloneFunction: \n {prompt}") + code = await self.write_code(prompt) + logger.info(f'CloneFunction code is \n {highlight(code)}') + return code + + +def run_function_code(func_code: str, func_name: str, *args, **kwargs): + """Run function code from string code.""" + try: + locals_ = {} + exec(func_code, locals_) + func = locals_[func_name] + return func(*args, **kwargs), "" + except Exception: + return "", traceback.format_exc() + + +def run_function_script(code_script_path: str, func_name: str, *args, **kwargs): + """Run function code from script.""" + if isinstance(code_script_path, str): + code_path = Path(code_script_path) + code = code_path.read_text(encoding='utf-8') + return run_function_code(code, func_name, *args, **kwargs) diff --git a/metagpt/actions/execute_task.py b/metagpt/actions/execute_task.py new file mode 100644 index 000000000..afdeda323 --- /dev/null +++ b/metagpt/actions/execute_task.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/13 12:26 +@Author : femto Zheng +@File : execute_task.py +""" +from metagpt.actions import Action +from metagpt.schema import Message + + +class ExecuteTask(Action): + def __init__(self, name="ExecuteTask", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) + + def run(self, *args, **kwargs): + pass diff --git a/metagpt/actions/prepare_interview.py b/metagpt/actions/prepare_interview.py new file mode 100644 index 000000000..5db3a9f37 --- /dev/null +++ b/metagpt/actions/prepare_interview.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/19 15:02 +@Author : DevXiaolan +@File : prepare_interview.py +""" +from metagpt.actions import Action + +PROMPT_TEMPLATE = """ +# Context +{context} + +## Format example +--- +Q1: question 1 here +References: + - point 1 + - point 2 + +Q2: question 2 here... +--- + +----- +Role: You are an interviewer of our company who is well-knonwn in frontend or backend develop; +Requirement: Provide a list of questions for the interviewer to ask the interviewee, by reading the resume of the interviewee in the context. +Attention: Provide as markdown block as the format above, at least 10 questions. +""" + +# prepare for a interview + + +class PrepareInterview(Action): + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) + + async def run(self, context): + prompt = PROMPT_TEMPLATE.format(context=context) + question_list = await self._aask_v1(prompt) + return question_list + diff --git a/metagpt/actions/research.py b/metagpt/actions/research.py index 81eb876dd..49a981e86 100644 --- a/metagpt/actions/research.py +++ b/metagpt/actions/research.py @@ -13,6 +13,7 @@ from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.tools.search_engine import SearchEngine from metagpt.tools.web_browser_engine import WebBrowserEngine, WebBrowserEngineType +from metagpt.utils.common import OutputParser from metagpt.utils.text import generate_prompt_chunk, reduce_message_length LANG_PROMPT = "Please respond in {language}." @@ -110,7 +111,7 @@ class CollectLinks(Action): system_text = system_text if system_text else RESEARCH_TOPIC_SYSTEM.format(topic=topic) keywords = await self._aask(SEARCH_TOPIC_PROMPT, [system_text]) try: - keywords = json.loads(keywords) + keywords = OutputParser.extract_struct(keywords, list) keywords = parse_obj_as(list[str], keywords) except Exception as e: logger.exception(f"fail to get keywords related to the research topic \"{topic}\" for {e}") @@ -130,7 +131,7 @@ class CollectLinks(Action): logger.debug(prompt) queries = await self._aask(prompt, [system_text]) try: - queries = json.loads(queries) + queries = OutputParser.extract_struct(queries, list) queries = parse_obj_as(list[str], queries) except Exception as e: logger.exception(f"fail to break down the research question due to {e}") @@ -158,7 +159,7 @@ class CollectLinks(Action): logger.debug(prompt) indices = await self._aask(prompt) try: - indices = json.loads(indices) + indices = OutputParser.extract_struct(indices, list) assert all(isinstance(i, int) for i in indices) except Exception as e: logger.exception(f"fail to rank results for {e}") diff --git a/metagpt/actions/write_tutorial.py b/metagpt/actions/write_tutorial.py index b23fc2ad4..23e3560e8 100644 --- a/metagpt/actions/write_tutorial.py +++ b/metagpt/actions/write_tutorial.py @@ -6,12 +6,12 @@ @File : tutorial_assistant.py @Describe : Actions of the tutorial assistant, including writing directories and document content. """ -import json + from typing import Dict from metagpt.actions import Action -from metagpt.logs import logger from metagpt.prompts.tutorial_assistant import DIRECTORY_PROMPT, CONTENT_PROMPT +from metagpt.utils.common import OutputParser class WriteDirectory(Action): @@ -26,33 +26,6 @@ class WriteDirectory(Action): super().__init__(name, *args, **kwargs) self.language = language - @staticmethod - async def _handle_resp(resp: str) -> Dict: - """Process string results and convert them to JSON format. - - Args: - resp: The directory results returned by gpt. - - Returns: - The parsed dictionary, such as {"title": "xxx", "directory": [{"dir 1": ["sub dir 1", "sub dir 2"]}]}. - - Raises: - Exception: If no matching dictionary section is found. - json.JSONDecodeError: If the dictionary part cannot be parsed as JSON. - """ - start = resp.find('{') - end = resp.rfind('}') - if start != -1 and end != -1 and end > start: - directory_str = resp[start:end + 1] - logger.info(f"Successfully parsed json: {str(directory_str)}") - try: - return json.loads(directory_str) - except json.JSONDecodeError as e: - logger.error(f"Json parsing error: {e}") - raise e - else: - raise Exception("No matching dictionary section found.") - async def run(self, topic: str, *args, **kwargs) -> Dict: """Execute the action to generate a tutorial directory according to the topic. @@ -64,7 +37,7 @@ class WriteDirectory(Action): """ prompt = DIRECTORY_PROMPT.format(topic=topic, language=self.language) resp = await self._aask(prompt=prompt) - return await self._handle_resp(resp) + return OutputParser.extract_struct(resp, dict) class WriteContent(Action): diff --git a/metagpt/const.py b/metagpt/const.py index 35b4c9fa7..b8b08628e 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -12,9 +12,11 @@ def get_project_root(): """Search upwards to find the project root directory.""" current_path = Path.cwd() while True: - if (current_path / '.git').exists() or \ - (current_path / '.project_root').exists() or \ - (current_path / '.gitignore').exists(): + if ( + (current_path / ".git").exists() + or (current_path / ".project_root").exists() + or (current_path / ".gitignore").exists() + ): return current_path parent_path = current_path.parent if parent_path == current_path: @@ -23,16 +25,18 @@ def get_project_root(): PROJECT_ROOT = get_project_root() -DATA_PATH = PROJECT_ROOT / 'data' -WORKSPACE_ROOT = PROJECT_ROOT / 'workspace' -PROMPT_PATH = PROJECT_ROOT / 'metagpt/prompts' -UT_PATH = PROJECT_ROOT / 'data/ut' +DATA_PATH = PROJECT_ROOT / "data" +WORKSPACE_ROOT = PROJECT_ROOT / "workspace" +PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" +UT_PATH = PROJECT_ROOT / "data/ut" SWAGGER_PATH = UT_PATH / "files/api/" UT_PY_PATH = UT_PATH / "files/ut/" API_QUESTIONS_PATH = UT_PATH / "files/question/" YAPI_URL = "http://yapi.deepwisdomai.com/" -TMP = PROJECT_ROOT / 'tmp' +TMP = PROJECT_ROOT / "tmp" RESEARCH_PATH = DATA_PATH / "research" TUTORIAL_PATH = DATA_PATH / "tutorial_docx" +SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills" + MEM_TTL = 24 * 30 * 3600 diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py new file mode 100644 index 000000000..b27841d74 --- /dev/null +++ b/metagpt/roles/sk_agent.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/13 12:23 +@Author : femto Zheng +@File : sk_agent.py +""" +from semantic_kernel.planning import SequentialPlanner +from semantic_kernel.planning.action_planner.action_planner import ActionPlanner +from semantic_kernel.planning.basic_planner import BasicPlanner + +from metagpt.actions import BossRequirement +from metagpt.actions.execute_task import ExecuteTask +from metagpt.logs import logger +from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.utils.make_sk_kernel import make_sk_kernel + + +class SkAgent(Role): + """ + Represents an SkAgent implemented using semantic kernel + + Attributes: + name (str): Name of the SkAgent. + profile (str): Role profile, default is 'sk_agent'. + goal (str): Goal of the SkAgent. + constraints (str): Constraints for the SkAgent. + """ + + def __init__( + self, + name: str = "Sunshine", + profile: str = "sk_agent", + goal: str = "Execute task based on passed in task description", + constraints: str = "", + planner_cls=BasicPlanner, + ) -> None: + """Initializes the Engineer role with given attributes.""" + super().__init__(name, profile, goal, constraints) + self._init_actions([ExecuteTask()]) + self._watch([BossRequirement]) + self.kernel = make_sk_kernel() + + # how funny the interface is inconsistent + if planner_cls == BasicPlanner: + self.planner = planner_cls() + elif planner_cls in [SequentialPlanner, ActionPlanner]: + self.planner = planner_cls(self.kernel) + else: + raise f"Unsupported planner of type {planner_cls}" + + self.import_semantic_skill_from_directory = self.kernel.import_semantic_skill_from_directory + self.import_skill = self.kernel.import_skill + + async def _think(self) -> None: + self._set_state(0) + # how funny the interface is inconsistent + if isinstance(self.planner, BasicPlanner): + self.plan = await self.planner.create_plan_async(self._rc.important_memory[-1].content, self.kernel) + logger.info(self.plan.generated_plan) + elif any(isinstance(self.planner, cls) for cls in [SequentialPlanner, ActionPlanner]): + self.plan = await self.planner.create_plan_async(self._rc.important_memory[-1].content) + + async def _act(self) -> Message: + # how funny the interface is inconsistent + if isinstance(self.planner, BasicPlanner): + result = await self.planner.execute_plan_async(self.plan, self.kernel) + elif any(isinstance(self.planner, cls) for cls in [SequentialPlanner, ActionPlanner]): + result = (await self.plan.invoke_async()).result + logger.info(result) + + msg = Message(content=result, role=self.profile, cause_by=type(self._rc.todo)) + self._rc.memory.add(msg) + # logger.debug(f"{response}") + return msg diff --git a/metagpt/skills/SummarizeSkill/MakeAbstractReadable/config.json b/metagpt/skills/SummarizeSkill/MakeAbstractReadable/config.json new file mode 100644 index 000000000..0bd48b77a --- /dev/null +++ b/metagpt/skills/SummarizeSkill/MakeAbstractReadable/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Given a scientific white paper abstract, rewrite it to make it more readable", + "completion": { + "max_tokens": 4000, + "temperature": 0.0, + "top_p": 1.0, + "presence_penalty": 0.0, + "frequency_penalty": 2.0 + } +} \ No newline at end of file diff --git a/metagpt/skills/SummarizeSkill/MakeAbstractReadable/skprompt.txt b/metagpt/skills/SummarizeSkill/MakeAbstractReadable/skprompt.txt new file mode 100644 index 000000000..5501e19b7 --- /dev/null +++ b/metagpt/skills/SummarizeSkill/MakeAbstractReadable/skprompt.txt @@ -0,0 +1,5 @@ +{{$input}} + +== +Summarize, using a user friendly, using simple grammar. Don't use subjects like "we" "our" "us" "your". +== \ No newline at end of file diff --git a/metagpt/skills/SummarizeSkill/Notegen/config.json b/metagpt/skills/SummarizeSkill/Notegen/config.json new file mode 100644 index 000000000..f7e1c355e --- /dev/null +++ b/metagpt/skills/SummarizeSkill/Notegen/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Automatically generate compact notes for any text or text document.", + "completion": { + "max_tokens": 256, + "temperature": 0.0, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + } +} \ No newline at end of file diff --git a/metagpt/skills/SummarizeSkill/Notegen/skprompt.txt b/metagpt/skills/SummarizeSkill/Notegen/skprompt.txt new file mode 100644 index 000000000..b3f4d203b --- /dev/null +++ b/metagpt/skills/SummarizeSkill/Notegen/skprompt.txt @@ -0,0 +1,21 @@ +Analyze the following extract taken from a document. +- Produce key points for memory. +- Give memory a name. +- Extract only points worth remembering. +- Be brief. Conciseness is very important. +- Use broken English. +You will use this memory to analyze the rest of this document, and for other relevant tasks. + +[Input] +My name is Macbeth. I used to be King of Scotland, but I died. My wife's name is Lady Macbeth and we were married for 15 years. We had no children. Our beloved dog Toby McDuff was a famous hunter of rats in the forest. +My story was immortalized by Shakespeare in a play. ++++++ +Family History +- Macbeth, King Scotland +- Wife Lady Macbeth, No Kids +- Dog Toby McDuff. Hunter, dead. +- Shakespeare play + +[Input] +[[{{$input}}]] ++++++ diff --git a/metagpt/skills/SummarizeSkill/Summarize/config.json b/metagpt/skills/SummarizeSkill/Summarize/config.json new file mode 100644 index 000000000..7ba5cf02d --- /dev/null +++ b/metagpt/skills/SummarizeSkill/Summarize/config.json @@ -0,0 +1,21 @@ +{ + "schema": 1, + "type": "completion", + "description": "Summarize given text or any text document", + "completion": { + "max_tokens": 512, + "temperature": 0.0, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + }, + "input": { + "parameters": [ + { + "name": "input", + "description": "Text to summarize", + "defaultValue": "" + } + ] + } +} diff --git a/metagpt/skills/SummarizeSkill/Summarize/skprompt.txt b/metagpt/skills/SummarizeSkill/Summarize/skprompt.txt new file mode 100644 index 000000000..5597e1350 --- /dev/null +++ b/metagpt/skills/SummarizeSkill/Summarize/skprompt.txt @@ -0,0 +1,23 @@ +[SUMMARIZATION RULES] +DONT WASTE WORDS +USE SHORT, CLEAR, COMPLETE SENTENCES. +DO NOT USE BULLET POINTS OR DASHES. +USE ACTIVE VOICE. +MAXIMIZE DETAIL, MEANING +FOCUS ON THE CONTENT + +[BANNED PHRASES] +This article +This document +This page +This material +[END LIST] + +Summarize: +Hello how are you? ++++++ +Hello + +Summarize this +{{$input}} ++++++ \ No newline at end of file diff --git a/metagpt/skills/SummarizeSkill/Topics/config.json b/metagpt/skills/SummarizeSkill/Topics/config.json new file mode 100644 index 000000000..b2cd9985c --- /dev/null +++ b/metagpt/skills/SummarizeSkill/Topics/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Analyze given text or document and extract key topics worth remembering", + "completion": { + "max_tokens": 128, + "temperature": 0.0, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + } +} \ No newline at end of file diff --git a/metagpt/skills/SummarizeSkill/Topics/skprompt.txt b/metagpt/skills/SummarizeSkill/Topics/skprompt.txt new file mode 100644 index 000000000..cb7a28c13 --- /dev/null +++ b/metagpt/skills/SummarizeSkill/Topics/skprompt.txt @@ -0,0 +1,28 @@ +Analyze the following extract taken from a document and extract key topics. +- Topics only worth remembering. +- Be brief. Short phrases. +- Can use broken English. +- Conciseness is very important. +- Topics can include names of memories you want to recall. +- NO LONG SENTENCES. SHORT PHRASES. +- Return in JSON +[Input] +My name is Macbeth. I used to be King of Scotland, but I died. My wife's name is Lady Macbeth and we were married for 15 years. We had no children. Our beloved dog Toby McDuff was a famous hunter of rats in the forest. +My tragic story was immortalized by Shakespeare in a play. +[Output] +{ + "topics": [ + "Macbeth", + "King of Scotland", + "Lady Macbeth", + "Dog", + "Toby McDuff", + "Shakespeare", + "Play", + "Tragedy" + ] +} ++++++ +[Input] +{{$input}} +[Output] \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/Acronym/config.json b/metagpt/skills/WriterSkill/Acronym/config.json new file mode 100644 index 000000000..c48414856 --- /dev/null +++ b/metagpt/skills/WriterSkill/Acronym/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Generate an acronym for the given concept or phrase", + "completion": { + "max_tokens": 100, + "temperature": 0.5, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + } +} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/Acronym/skprompt.txt b/metagpt/skills/WriterSkill/Acronym/skprompt.txt new file mode 100644 index 000000000..1c2e8a6aa --- /dev/null +++ b/metagpt/skills/WriterSkill/Acronym/skprompt.txt @@ -0,0 +1,25 @@ +Generate a suitable acronym pair for the concept. Creativity is encouraged, including obscure references. +The uppercase letters in the acronym expansion must agree with the letters of the acronym + +Q: A technology for detecting moving objects, their distance and velocity using radio waves. +A: R.A.D.A.R: RAdio Detection And Ranging. + +Q: A weapon that uses high voltage electricity to incapacitate the target +A. T.A.S.E.R: Thomas A. Swift’s Electric Rifle + +Q: Equipment that lets a diver breathe underwater +A: S.C.U.B.A: Self Contained Underwater Breathing Apparatus. + +Q: Reminder not to complicated subject matter. +A. K.I.S.S: Keep It Simple Stupid + +Q: A national organization for investment in space travel, rockets, space ships, space exploration +A. N.A.S.A: National Aeronautics Space Administration + +Q: Agreement that governs trade among North American countries. +A: N.A.F.T.A: North American Free Trade Agreement. + +Q: Organization to protect the freedom and security of its member countries in North America and Europe. +A: N.A.T.O: North Atlantic Treaty Organization. + +Q:{{$input}} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/AcronymGenerator/config.json b/metagpt/skills/WriterSkill/AcronymGenerator/config.json new file mode 100644 index 000000000..1dab1fe9f --- /dev/null +++ b/metagpt/skills/WriterSkill/AcronymGenerator/config.json @@ -0,0 +1,15 @@ +{ + "schema": 1, + "type": "completion", + "description": "Given a request to generate an acronym from a string, generate an acronym and provide the acronym explanation.", + "completion": { + "max_tokens": 256, + "temperature": 0.7, + "top_p": 1.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "stop_sequences": [ + "#" + ] + } +} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/AcronymGenerator/skprompt.txt b/metagpt/skills/WriterSkill/AcronymGenerator/skprompt.txt new file mode 100644 index 000000000..5bf0b987d --- /dev/null +++ b/metagpt/skills/WriterSkill/AcronymGenerator/skprompt.txt @@ -0,0 +1,54 @@ +# Name of a super artificial intelligence +J.A.R.V.I.S. = Just A Really Very Intelligent System. +# Name for a new young beautiful assistant +F.R.I.D.A.Y. = Female Replacement Intelligent Digital Assistant Youth. +# Mirror to check what's behind +B.A.R.F. = Binary Augmented Retro-Framing. +# Pair of powerful glasses created by a genius that is now dead +E.D.I.T.H. = Even Dead I’m The Hero. +# A company building and selling computers +I.B.M. = Intelligent Business Machine. +# A super computer that is sentient. +H.A.L = Heuristically programmed ALgorithmic computer. +# an intelligent bot that helps with productivity. +C.O.R.E. = Central Optimization Routines and Efficiency. +# an intelligent bot that helps with productivity. +P.A.L. = Personal Assistant Light. +# an intelligent bot that helps with productivity. +A.I.D.A. = Artificial Intelligence Digital Assistant. +# an intelligent bot that helps with productivity. +H.E.R.A. = Human Emulation and Recognition Algorithm. +# an intelligent bot that helps with productivity. +I.C.A.R.U.S. = Intelligent Control and Automation of Research and Utility Systems. +# an intelligent bot that helps with productivity. +N.E.M.O. = Networked Embedded Multiprocessor Orchestration. +# an intelligent bot that helps with productivity. +E.P.I.C. = Enhanced Productivity and Intelligence through Computing. +# an intelligent bot that helps with productivity. +M.A.I.A. = Multipurpose Artificial Intelligence Assistant. +# an intelligent bot that helps with productivity. +A.R.I.A. = Artificial Reasoning and Intelligent Assistant. +# An incredibly smart entity developed with complex math, that helps me being more productive. +O.M.E.G.A. = Optimized Mathematical Entity for Generalized Artificial intelligence. +# An incredibly smart entity developed with complex math, that helps me being more productive. +P.Y.T.H.O.N. = Precise Yet Thorough Heuristic Optimization Network. +# An incredibly smart entity developed with complex math, that helps me being more productive. +A.P.O.L.L.O. = Adaptive Probabilistic Optimization Learning Library for Online Applications. +# An incredibly smart entity developed with complex math, that helps me being more productive. +S.O.L.I.D. = Self-Organizing Logical Intelligent Data-base. +# An incredibly smart entity developed with complex math, that helps me being more productive. +D.E.E.P. = Dynamic Estimation and Prediction. +# An incredibly smart entity developed with complex math, that helps me being more productive. +B.R.A.I.N. = Biologically Realistic Artificial Intelligence Network. +# An incredibly smart entity developed with complex math, that helps me being more productive. +C.O.G.N.I.T.O. = COmputational and Generalized INtelligence TOolkit. +# An incredibly smart entity developed with complex math, that helps me being more productive. +S.A.G.E. = Symbolic Artificial General Intelligence Engine. +# An incredibly smart entity developed with complex math, that helps me being more productive. +Q.U.A.R.K. = Quantum Universal Algorithmic Reasoning Kernel. +# An incredibly smart entity developed with complex math, that helps me being more productive. +S.O.L.V.E. = Sophisticated Operational Logic and Versatile Expertise. +# An incredibly smart entity developed with complex math, that helps me being more productive. +C.A.L.C.U.L.U.S. = Cognitively Advanced Logic and Computation Unit for Learning and Understanding Systems. + +# {{$INPUT}} diff --git a/metagpt/skills/WriterSkill/AcronymReverse/config.json b/metagpt/skills/WriterSkill/AcronymReverse/config.json new file mode 100644 index 000000000..eed5c5191 --- /dev/null +++ b/metagpt/skills/WriterSkill/AcronymReverse/config.json @@ -0,0 +1,15 @@ +{ + "schema": 1, + "type": "completion", + "description": "Given a single word or acronym, generate the expanded form matching the acronym letters.", + "completion": { + "max_tokens": 256, + "temperature": 0.5, + "top_p": 1.0, + "presence_penalty": 0.8, + "frequency_penalty": 0.0, + "stop_sequences": [ + "#END#" + ] + } +} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/AcronymReverse/skprompt.txt b/metagpt/skills/WriterSkill/AcronymReverse/skprompt.txt new file mode 100644 index 000000000..7c1d649a9 --- /dev/null +++ b/metagpt/skills/WriterSkill/AcronymReverse/skprompt.txt @@ -0,0 +1,24 @@ +# acronym: Devis +Sentences matching the acronym: +1. Dragons Eat Very Interesting Snacks +2. Develop Empathy and Vision to Increase Success +3. Don't Expect Vampires In Supermarkets +#END# + +# acronym: Christmas +Sentences matching the acronym: +1. Celebrating Harmony and Respect in a Season of Togetherness, Merriment, and True joy +2. Children Have Real Interest Since The Mystery And Surprise Thrills +3. Christmas Helps Reduce Inner Stress Through Mistletoe And Sleigh excursions +#END# + +# acronym: noWare +Sentences matching the acronym: +1. No One Wants an App that Randomly Erases everything +2. Nourishing Oatmeal With Almond, Raisin, and Egg toppings +3. Notice Opportunity When Available and React Enthusiastically +#END# + +Reverse the following acronym back to a funny sentence. Provide 3 examples. +# acronym: {{$INPUT}} +Sentences matching the acronym: diff --git a/metagpt/skills/WriterSkill/Brainstorm/config.json b/metagpt/skills/WriterSkill/Brainstorm/config.json new file mode 100644 index 000000000..f50a354e7 --- /dev/null +++ b/metagpt/skills/WriterSkill/Brainstorm/config.json @@ -0,0 +1,22 @@ +{ + "schema": 1, + "type": "completion", + "description": "Given a goal or topic description generate a list of ideas", + "completion": { + "max_tokens": 2000, + "temperature": 0.5, + "top_p": 1.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "stop_sequences": ["##END##"] + }, + "input": { + "parameters": [ + { + "name": "input", + "description": "A topic description or goal.", + "defaultValue": "" + } + ] + } +} diff --git a/metagpt/skills/WriterSkill/Brainstorm/skprompt.txt b/metagpt/skills/WriterSkill/Brainstorm/skprompt.txt new file mode 100644 index 000000000..6a8b92086 --- /dev/null +++ b/metagpt/skills/WriterSkill/Brainstorm/skprompt.txt @@ -0,0 +1,8 @@ +Must: brainstorm ideas and create a list. +Must: use a numbered list. +Must: only one list. +Must: end list with ##END## +Should: no more than 10 items. +Should: at least 3 items. +Topic: {{$INPUT}} +Start. diff --git a/metagpt/skills/WriterSkill/EmailGen/config.json b/metagpt/skills/WriterSkill/EmailGen/config.json new file mode 100644 index 000000000..d43eab348 --- /dev/null +++ b/metagpt/skills/WriterSkill/EmailGen/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Write an email from the given bullet points", + "completion": { + "max_tokens": 256, + "temperature": 0.0, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + } +} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/EmailGen/skprompt.txt b/metagpt/skills/WriterSkill/EmailGen/skprompt.txt new file mode 100644 index 000000000..26f4933fb --- /dev/null +++ b/metagpt/skills/WriterSkill/EmailGen/skprompt.txt @@ -0,0 +1,16 @@ +Rewrite my bullet points into complete sentences. Use a polite and inclusive tone. + +[Input] +- Macbeth, King Scotland +- Married, Wife Lady Macbeth, No Kids +- Dog Toby McDuff. Hunter, dead. +- Shakespeare play ++++++ +The story of Macbeth +My name is Macbeth. I used to be King of Scotland, but I died. My wife's name is Lady Macbeth and we were married for 15 years. We had no children. Our beloved dog Toby McDuff was a famous hunter of rats in the forest. +My story was immortalized by Shakespeare in a play. + ++++++ +[Input] +{{$input}} ++++++ diff --git a/metagpt/skills/WriterSkill/EmailTo/config.json b/metagpt/skills/WriterSkill/EmailTo/config.json new file mode 100644 index 000000000..5f0d6ee6e --- /dev/null +++ b/metagpt/skills/WriterSkill/EmailTo/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Turn bullet points into an email to someone, using a polite tone", + "completion": { + "max_tokens": 256, + "temperature": 0.0, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + } +} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/EmailTo/skprompt.txt b/metagpt/skills/WriterSkill/EmailTo/skprompt.txt new file mode 100644 index 000000000..cc6b5c962 --- /dev/null +++ b/metagpt/skills/WriterSkill/EmailTo/skprompt.txt @@ -0,0 +1,31 @@ +Rewrite my bullet points into an email featuring complete sentences. Use a polite and inclusive tone. + +[Input] +Toby, + +- Macbeth, King Scotland +- Married, Wife Lady Macbeth, No Kids +- Dog Toby McDuff. Hunter, dead. +- Shakespeare play + +Thanks, +Dexter + ++++++ +Hi Toby, + +The story of Macbeth +My name is Macbeth. I used to be King of Scotland, but I died. My wife's name is Lady Macbeth and we were married for 15 years. We had no children. Our beloved dog Toby McDuff was a famous hunter of rats in the forest. +My story was immortalized by Shakespeare in a play. + +Thanks, +Dexter + ++++++ +[Input] +{{$to}} +{{$input}} + +Thanks, +{{$sender}} ++++++ diff --git a/metagpt/skills/WriterSkill/EnglishImprover/config.json b/metagpt/skills/WriterSkill/EnglishImprover/config.json new file mode 100644 index 000000000..4d10af469 --- /dev/null +++ b/metagpt/skills/WriterSkill/EnglishImprover/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Translate text to English and improve it", + "completion": { + "max_tokens": 3000, + "temperature": 0.0, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + } +} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/EnglishImprover/skprompt.txt b/metagpt/skills/WriterSkill/EnglishImprover/skprompt.txt new file mode 100644 index 000000000..09b80036c --- /dev/null +++ b/metagpt/skills/WriterSkill/EnglishImprover/skprompt.txt @@ -0,0 +1,11 @@ +I want you to act as an English translator, spelling corrector and improver. +I will speak to you in any language and you will detect the language, translate it and answer in the corrected and improved version of my text, in English. +I want you to replace my simplified A0-level words and sentences with more beautiful and elegant, upper level English words and sentences. +Keep the meaning same, but make them more literary. +I want you to only reply the correction, the improvements and nothing else, do not write explanations. + +Sentence: """ +{{$INPUT}} +""" + +Translation: diff --git a/metagpt/skills/WriterSkill/NovelChapter/config.json b/metagpt/skills/WriterSkill/NovelChapter/config.json new file mode 100644 index 000000000..3568c6955 --- /dev/null +++ b/metagpt/skills/WriterSkill/NovelChapter/config.json @@ -0,0 +1,36 @@ +{ + "schema": 1, + "type": "completion", + "description": "Write a chapter of a novel.", + "completion": { + "max_tokens": 2048, + "temperature": 0.3, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + }, + "input": { + "parameters": [ + { + "name": "input", + "description": "A synopsis of what the chapter should be about.", + "defaultValue": "" + }, + { + "name": "theme", + "description": "The theme or topic of this novel.", + "defaultValue": "" + }, + { + "name": "previousChapter", + "description": "The synopsis of the previous chapter.", + "defaultValue": "" + }, + { + "name": "chapterIndex", + "description": "The number of the chapter to write.", + "defaultValue": "" + } + ] + } +} diff --git a/metagpt/skills/WriterSkill/NovelChapter/skprompt.txt b/metagpt/skills/WriterSkill/NovelChapter/skprompt.txt new file mode 100644 index 000000000..4fb85a538 --- /dev/null +++ b/metagpt/skills/WriterSkill/NovelChapter/skprompt.txt @@ -0,0 +1,20 @@ +[CONTEXT] + +THEME OF STORY: +{{$theme}} + +PREVIOUS CHAPTER: +{{$previousChapter}} + +[END CONTEXT] + + +WRITE THIS CHAPTER USING [CONTEXT] AND +CHAPTER SYNOPSIS. DO NOT REPEAT SYNOPSIS IN THE OUTPUT + +Chapter Synopsis: +{{$input}} + +Chapter {{$chapterIndex}} + + diff --git a/metagpt/skills/WriterSkill/NovelChapterWithNotes/config.json b/metagpt/skills/WriterSkill/NovelChapterWithNotes/config.json new file mode 100644 index 000000000..02b9e613a --- /dev/null +++ b/metagpt/skills/WriterSkill/NovelChapterWithNotes/config.json @@ -0,0 +1,41 @@ +{ + "schema": 1, + "type": "completion", + "description": "Write a chapter of a novel using notes about the chapter to write.", + "completion": { + "max_tokens": 1024, + "temperature": 0.5, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + }, + "input": { + "parameters": [ + { + "name": "input", + "description": "What the novel should be about.", + "defaultValue": "" + }, + { + "name": "theme", + "description": "The theme of this novel.", + "defaultValue": "" + }, + { + "name": "notes", + "description": "Notes useful to write this chapter.", + "defaultValue": "" + }, + { + "name": "previousChapter", + "description": "The previous chapter synopsis.", + "defaultValue": "" + }, + { + "name": "chapterIndex", + "description": "The number of the chapter to write.", + "defaultValue": "" + } + ] + } +} diff --git a/metagpt/skills/WriterSkill/NovelChapterWithNotes/skprompt.txt b/metagpt/skills/WriterSkill/NovelChapterWithNotes/skprompt.txt new file mode 100644 index 000000000..650bd50d9 --- /dev/null +++ b/metagpt/skills/WriterSkill/NovelChapterWithNotes/skprompt.txt @@ -0,0 +1,19 @@ +[CONTEXT] + +THEME OF STORY: +{{$theme}} + +NOTES OF STORY SO FAR - USE AS REFERENCE +{{$notes}} + +PREVIOUS CHAPTER, USE AS REFERENCE: +{{$previousChapter}} + +[END CONTEXT] + + +WRITE THIS CHAPTER CONTINUING STORY, USING [CONTEXT] AND CHAPTER SYNOPSIS BELOW. DO NOT REPEAT SYNOPSIS IN THE CHAPTER. DON'T REPEAT PREVIOUS CHAPTER. + +{{$input}} + +Chapter {{$chapterIndex}} diff --git a/metagpt/skills/WriterSkill/NovelOutline/config.json b/metagpt/skills/WriterSkill/NovelOutline/config.json new file mode 100644 index 000000000..a34622f7b --- /dev/null +++ b/metagpt/skills/WriterSkill/NovelOutline/config.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "type": "completion", + "description": "Generate a list of chapter synopsis for a novel or novella", + "completion": { + "max_tokens": 2048, + "temperature": 0.1, + "top_p": 0.5, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + }, + "input": { + "parameters": [ + { + "name": "input", + "description": "What the novel should be about.", + "defaultValue": "" + }, + { + "name": "chapterCount", + "description": "The number of chapters to generate.", + "defaultValue": "" + }, + { + "name": "endMarker", + "description": "The marker to use to end each chapter.", + "defaultValue": "" + } + ] + } +} diff --git a/metagpt/skills/WriterSkill/NovelOutline/skprompt.txt b/metagpt/skills/WriterSkill/NovelOutline/skprompt.txt new file mode 100644 index 000000000..05f725acb --- /dev/null +++ b/metagpt/skills/WriterSkill/NovelOutline/skprompt.txt @@ -0,0 +1,12 @@ +I want to write a {{$chapterCount}} chapter novella about: +{{$input}} + +There MUST BE {{$chapterCount}} CHAPTERS. + +INVENT CHARACTERS AS YOU SEE FIT. BE HIGHLY CREATIVE AND/OR FUNNY. +WRITE SYNOPSIS FOR EACH CHAPTER. INCLUDE INFORMATION ABOUT CHARACTERS ETC. SINCE EACH +CHAPTER WILL BE WRITTEN BY A DIFFERENT WRITER, YOU MUST INCLUDE ALL PERTINENT INFORMATION +IN EACH SYNOPSIS + +YOU MUST END EACH SYNOPSIS WITH {{$endMarker}} + diff --git a/metagpt/skills/WriterSkill/Rewrite/config.json b/metagpt/skills/WriterSkill/Rewrite/config.json new file mode 100644 index 000000000..175ade9d9 --- /dev/null +++ b/metagpt/skills/WriterSkill/Rewrite/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Automatically generate compact notes for any text or text document", + "completion": { + "max_tokens": 256, + "temperature": 0.0, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + } +} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/Rewrite/skprompt.txt b/metagpt/skills/WriterSkill/Rewrite/skprompt.txt new file mode 100644 index 000000000..37f8d03fc --- /dev/null +++ b/metagpt/skills/WriterSkill/Rewrite/skprompt.txt @@ -0,0 +1,6 @@ +Rewrite the given text like it was written in this style or by: {{$style}}. +MUST RETAIN THE MEANING AND FACTUAL CONTENT AS THE ORIGINAL. + + +{{$input}} + diff --git a/metagpt/skills/WriterSkill/ShortPoem/config.json b/metagpt/skills/WriterSkill/ShortPoem/config.json new file mode 100644 index 000000000..0cc3da6c8 --- /dev/null +++ b/metagpt/skills/WriterSkill/ShortPoem/config.json @@ -0,0 +1,21 @@ +{ + "schema": 1, + "type": "completion", + "description": "Turn a scenario into a short and entertaining poem.", + "completion": { + "max_tokens": 60, + "temperature": 0.5, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + }, + "input": { + "parameters": [ + { + "name": "input", + "description": "The scenario to turn into a poem.", + "defaultValue": "" + } + ] + } +} diff --git a/metagpt/skills/WriterSkill/ShortPoem/skprompt.txt b/metagpt/skills/WriterSkill/ShortPoem/skprompt.txt new file mode 100644 index 000000000..bc42fcba6 --- /dev/null +++ b/metagpt/skills/WriterSkill/ShortPoem/skprompt.txt @@ -0,0 +1,2 @@ +Generate a short funny poem or limerick to explain the given event. Be creative and be funny. Let your imagination run wild. +Event:{{$input}} diff --git a/metagpt/skills/WriterSkill/StoryGen/config.json b/metagpt/skills/WriterSkill/StoryGen/config.json new file mode 100644 index 000000000..212831341 --- /dev/null +++ b/metagpt/skills/WriterSkill/StoryGen/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Generate a list of synopsis for a novel or novella with sub-chapters", + "completion": { + "max_tokens": 250, + "temperature": 0.0, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + } +} diff --git a/metagpt/skills/WriterSkill/StoryGen/skprompt.txt b/metagpt/skills/WriterSkill/StoryGen/skprompt.txt new file mode 100644 index 000000000..661df013c --- /dev/null +++ b/metagpt/skills/WriterSkill/StoryGen/skprompt.txt @@ -0,0 +1,10 @@ +ONLY USE XML TAGS IN THIS LIST: +[XML TAG LIST] +list: Surround any lists with this tag +synopsis: An outline of the chapter to write +[END LIST] + +EMIT WELL FORMED XML ALWAYS. Code should be CDATA. + + +{{$input}} diff --git a/metagpt/skills/WriterSkill/TellMeMore/config.json b/metagpt/skills/WriterSkill/TellMeMore/config.json new file mode 100644 index 000000000..28b6b4e5c --- /dev/null +++ b/metagpt/skills/WriterSkill/TellMeMore/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Summarize given text or any text document", + "completion": { + "max_tokens": 500, + "temperature": 0.0, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + } +} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/TellMeMore/skprompt.txt b/metagpt/skills/WriterSkill/TellMeMore/skprompt.txt new file mode 100644 index 000000000..143ce3a65 --- /dev/null +++ b/metagpt/skills/WriterSkill/TellMeMore/skprompt.txt @@ -0,0 +1,7 @@ +>>>>>The following is part of a {{$conversationtype}}. +{{$input}} + +>>>>>The following is an overview of a previous part of the {{$conversationtype}}, focusing on "{{$focusarea}}". +{{$previousresults}} + +>>>>>In 250 words or less, write a verbose and detailed overview of the {{$conversationtype}} focusing solely on "{{$focusarea}}". \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/Translate/config.json b/metagpt/skills/WriterSkill/Translate/config.json new file mode 100644 index 000000000..8134ce8dd --- /dev/null +++ b/metagpt/skills/WriterSkill/Translate/config.json @@ -0,0 +1,15 @@ +{ + "schema": 1, + "type": "completion", + "description": "Translate the input into a language of your choice", + "completion": { + "max_tokens": 2000, + "temperature": 0.7, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "stop_sequences": [ + "[done]" + ] + } +} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/Translate/skprompt.txt b/metagpt/skills/WriterSkill/Translate/skprompt.txt new file mode 100644 index 000000000..d5f2fa8c1 --- /dev/null +++ b/metagpt/skills/WriterSkill/Translate/skprompt.txt @@ -0,0 +1,7 @@ +Translate the input below into {{$language}} + +MAKE SURE YOU ONLY USE {{$language}}. + +{{$input}} + +Translation: diff --git a/metagpt/skills/WriterSkill/TwoSentenceSummary/config.json b/metagpt/skills/WriterSkill/TwoSentenceSummary/config.json new file mode 100644 index 000000000..833bd5950 --- /dev/null +++ b/metagpt/skills/WriterSkill/TwoSentenceSummary/config.json @@ -0,0 +1,12 @@ +{ + "schema": 1, + "type": "completion", + "description": "Summarize given text in two sentences or less", + "completion": { + "max_tokens": 100, + "temperature": 0.0, + "top_p": 0.0, + "presence_penalty": 0.0, + "frequency_penalty": 0.0 + } +} \ No newline at end of file diff --git a/metagpt/skills/WriterSkill/TwoSentenceSummary/skprompt.txt b/metagpt/skills/WriterSkill/TwoSentenceSummary/skprompt.txt new file mode 100644 index 000000000..b8f657a93 --- /dev/null +++ b/metagpt/skills/WriterSkill/TwoSentenceSummary/skprompt.txt @@ -0,0 +1,4 @@ +Summarize the following text in two sentences or less. +[BEGIN TEXT] +{{$input}} +[END TEXT] diff --git a/metagpt/tools/code_interpreter.py b/metagpt/tools/code_interpreter.py new file mode 100644 index 000000000..97398ccfd --- /dev/null +++ b/metagpt/tools/code_interpreter.py @@ -0,0 +1,129 @@ +import re +from typing import List, Callable +from pathlib import Path + +import wrapt +import textwrap +import inspect +from interpreter.interpreter import Interpreter + +from metagpt.logs import logger +from metagpt.config import CONFIG +from metagpt.utils.highlight import highlight +from metagpt.actions.clone_function import CloneFunction, run_function_code, run_function_script + + +def extract_python_code(code: str): + """Extract code blocks: If the code comments are the same, only the last code block is kept.""" + # Use regular expressions to match comment blocks and related code. + pattern = r'(#\s[^\n]*)\n(.*?)(?=\n\s*#|$)' + matches = re.findall(pattern, code, re.DOTALL) + + # Extract the last code block when encountering the same comment. + unique_comments = {} + for comment, code_block in matches: + unique_comments[comment] = code_block + + # concatenate into functional form + result_code = '\n'.join([f"{comment}\n{code_block}" for comment, code_block in unique_comments.items()]) + header_code = code[:code.find("#")] + code = header_code + result_code + + logger.info(f"Extract python code: \n {highlight(code)}") + + return code + + +class OpenCodeInterpreter(object): + """https://github.com/KillianLucas/open-interpreter""" + def __init__(self, auto_run: bool = True) -> None: + interpreter = Interpreter() + interpreter.auto_run = auto_run + interpreter.model = CONFIG.openai_api_model or "gpt-3.5-turbo" + interpreter.api_key = CONFIG.openai_api_key + interpreter.api_base = CONFIG.openai_api_base + self.interpreter = interpreter + + def chat(self, query: str, reset: bool = True): + if reset: + self.interpreter.reset() + return self.interpreter.chat(query, return_messages=True) + + @staticmethod + def extract_function(query_respond: List, function_name: str, *, language: str = 'python', + function_format: str = None) -> str: + """create a function from query_respond.""" + if language not in ('python'): + raise NotImplementedError(f"Not support to parse language {language}!") + + # set function form + if function_format is None: + assert language == 'python', f"Expect python language for default function_format, but got {language}." + function_format = """def {function_name}():\n{code}""" + # Extract the code module in the open-interpreter respond message. + code = [item['function_call']['parsed_arguments']['code'] for item in query_respond + if "function_call" in item + and "parsed_arguments" in item["function_call"] + and 'language' in item["function_call"]['parsed_arguments'] + and item["function_call"]['parsed_arguments']['language'] == language] + # add indent. + indented_code_str = textwrap.indent("\n".join(code), ' ' * 4) + # Return the code after deduplication. + if language == "python": + return extract_python_code(function_format.format(function_name=function_name, code=indented_code_str)) + + +def gen_query(func: Callable, args, kwargs) -> str: + # Get the annotation of the function as part of the query. + desc = func.__doc__ + signature = inspect.signature(func) + # Get the signature of the wrapped function and the assignment of the input parameters as part of the query. + bound_args = signature.bind(*args, **kwargs) + bound_args.apply_defaults() + query = f"{desc}, {bound_args.arguments}, If you must use a third-party package, use the most popular ones, for example: pandas, numpy, ta, ..." + return query + + +def gen_template_fun(func: Callable) -> str: + return f"def {func.__name__}{str(inspect.signature(func))}\n # here is your code ..." + + +class OpenInterpreterDecorator(object): + def __init__(self, save_code: bool = False, code_file_path: str = None, clear_code: bool = False) -> None: + self.save_code = save_code + self.code_file_path = code_file_path + self.clear_code = clear_code + + def __call__(self, wrapped): + @wrapt.decorator + async def wrapper(wrapped: Callable, instance, args, kwargs): + # Get the decorated function name. + func_name = wrapped.__name__ + # If the script exists locally and clearcode is not required, execute the function from the script. + if Path(self.code_file_path).is_file() and not self.clear_code: + return run_function_script(self.code_file_path, func_name, *args, **kwargs) + + # Auto run generate code by using open-interpreter. + interpreter = OpenCodeInterpreter() + query = gen_query(wrapped, args, kwargs) + logger.info(f"query for OpenCodeInterpreter: \n {query}") + respond = interpreter.chat(query) + # Assemble the code blocks generated by open-interpreter into a function without parameters. + func_code = interpreter.extract_function(respond, func_name) + # Clone the `func_code` into wrapped, that is, + # keep the `func_code` and wrapped functions with the same input parameter and return value types. + template_func = gen_template_fun(wrapped) + cf = CloneFunction() + code = await cf.run(template_func=template_func, source_code=func_code) + # Display the generated function in the terminal. + logger_code = highlight(code, "python") + logger.info(f"Creating following Python function:\n{logger_code}") + # execute this function. + try: + res = run_function_code(code, func_name, *args, **kwargs) + if self.save_code: + cf._save(self.code_file_path, code) + except Exception as e: + raise Exception("Could not evaluate Python code", e) + return res + return wrapper(wrapped) diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index d28700054..4ac078714 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -5,15 +5,32 @@ @Author : alexanderwu @File : search_engine.py """ -from __future__ import annotations +# from __future__ import annotations import importlib from typing import Callable, Coroutine, Literal, overload +from semantic_kernel.skill_definition import sk_function + from metagpt.config import CONFIG from metagpt.tools import SearchEngineType +class SkSearchEngine: + def __init__(self): + self.search_engine = SearchEngine() + + @sk_function( + description="searches results from Google. Useful when you need to find short " + "and succinct answers about a specific topic. Input should be a search query.", + name="searchAsync", + input_description="search", + ) + async def run(self, query: str) -> str: + result = await self.search_engine.run(query) + return result + + class SearchEngine: """Class representing a search engine. @@ -25,6 +42,7 @@ class SearchEngine: run_func: The function to run the search. engine: The search engine type. """ + def __init__( self, engine: SearchEngineType | None = None, @@ -33,7 +51,7 @@ class SearchEngine: engine = engine or CONFIG.search_engine if engine == SearchEngineType.SERPAPI_GOOGLE: module = "metagpt.tools.search_engine_serpapi" - run_func = importlib.import_module(module).SerpAPIWrapper().run + run_func = importlib.import_module(module).SerpAPIWrapper().run elif engine == SearchEngineType.SERPER_GOOGLE: module = "metagpt.tools.search_engine_serper" run_func = importlib.import_module(module).SerperWrapper().run diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 9add0db8f..65cc15e82 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -11,7 +11,7 @@ import inspect import os import platform import re -from typing import List, Tuple +from typing import List, Tuple, Union from metagpt.logs import logger @@ -150,6 +150,53 @@ class OutputParser: parsed_data[block] = content return parsed_data + @classmethod + def extract_struct(cls, text: str, data_type: Union[type(list), type(dict)]) -> Union[list, dict]: + """Extracts and parses a specified type of structure (dictionary or list) from the given text. + The text only contains a list or dictionary, which may have nested structures. + + Args: + text: The text containing the structure (dictionary or list). + data_type: The data type to extract, can be "list" or "dict". + + Returns: + - If extraction and parsing are successful, it returns the corresponding data structure (list or dictionary). + - If extraction fails or parsing encounters an error, it throw an exception. + + Examples: + >>> text = 'xxx [1, 2, ["a", "b", [3, 4]], {"x": 5, "y": [6, 7]}] xxx' + >>> result_list = OutputParser.extract_struct(text, "list") + >>> print(result_list) + >>> # Output: [1, 2, ["a", "b", [3, 4]], {"x": 5, "y": [6, 7]}] + + >>> text = 'xxx {"x": 1, "y": {"a": 2, "b": {"c": 3}}} xxx' + >>> result_dict = OutputParser.extract_struct(text, "dict") + >>> print(result_dict) + >>> # Output: {"x": 1, "y": {"a": 2, "b": {"c": 3}}} + """ + # Find the first "[" or "{" and the last "]" or "}" + start_index = text.find("[" if data_type is list else "{") + end_index = text.rfind("]" if data_type is list else "}") + + if start_index != -1 and end_index != -1: + # Extract the structure part + structure_text = text[start_index:end_index + 1] + + try: + # Attempt to convert the text to a Python data type using ast.literal_eval + result = ast.literal_eval(structure_text) + + # Ensure the result matches the specified data type + if isinstance(result, list) or isinstance(result, dict): + return result + + raise ValueError(f"The extracted structure is not a {data_type}.") + + except (ValueError, SyntaxError) as e: + raise Exception(f"Error while extracting and parsing the {data_type}: {e}") + else: + raise Exception(f"No {data_type} found in the text.") + class CodeParser: @classmethod diff --git a/metagpt/utils/file.py b/metagpt/utils/file.py index 5aca2a0e5..f3691549b 100644 --- a/metagpt/utils/file.py +++ b/metagpt/utils/file.py @@ -15,6 +15,8 @@ from metagpt.logs import logger class File: """A general util for file operations.""" + CHUNK_SIZE = 64 * 1024 + @classmethod async def write(cls, root_path: Path, filename: str, content: bytes) -> Path: """Write the file content to the local specified path. @@ -35,8 +37,39 @@ class File: full_path = root_path / filename async with aiofiles.open(full_path, mode="wb") as writer: await writer.write(content) - logger.info(f"Successfully write file: {full_path}") + logger.debug(f"Successfully write file: {full_path}") return full_path except Exception as e: logger.error(f"Error writing file: {e}") - raise e \ No newline at end of file + raise e + + @classmethod + async def read(cls, file_path: Path, chunk_size: int = None) -> bytes: + """Partitioning read the file content from the local specified path. + + Args: + file_path: The full file name of file, such as "/data/test.txt". + chunk_size: The size of each chunk in bytes (default is 64kb). + + Returns: + The binary content of file. + + Raises: + Exception: If an unexpected error occurs during the file reading process. + """ + try: + chunk_size = chunk_size or cls.CHUNK_SIZE + async with aiofiles.open(file_path, mode="rb") as reader: + chunks = list() + while True: + chunk = await reader.read(chunk_size) + if not chunk: + break + chunks.append(chunk) + content = b''.join(chunks) + logger.debug(f"Successfully read file, the path of file: {file_path}") + return content + except Exception as e: + logger.error(f"Error reading file: {e}") + raise e + diff --git a/metagpt/utils/highlight.py b/metagpt/utils/highlight.py new file mode 100644 index 000000000..e6cbb228c --- /dev/null +++ b/metagpt/utils/highlight.py @@ -0,0 +1,25 @@ +# 添加代码语法高亮显示 +from pygments import highlight as highlight_ +from pygments.lexers import PythonLexer, SqlLexer +from pygments.formatters import TerminalFormatter, HtmlFormatter + + +def highlight(code: str, language: str = 'python', formatter: str = 'terminal'): + # 指定要高亮的语言 + if language.lower() == 'python': + lexer = PythonLexer() + elif language.lower() == 'sql': + lexer = SqlLexer() + else: + raise ValueError(f"Unsupported language: {language}") + + # 指定输出格式 + if formatter.lower() == 'terminal': + formatter = TerminalFormatter() + elif formatter.lower() == 'html': + formatter = HtmlFormatter() + else: + raise ValueError(f"Unsupported formatter: {formatter}") + + # 使用 Pygments 高亮代码片段 + return highlight_(code, lexer, formatter) diff --git a/metagpt/utils/make_sk_kernel.py b/metagpt/utils/make_sk_kernel.py new file mode 100644 index 000000000..5e919abeb --- /dev/null +++ b/metagpt/utils/make_sk_kernel.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/13 12:29 +@Author : femto Zheng +@File : make_sk_kernel.py +""" +import semantic_kernel as sk +from semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion import ( + AzureChatCompletion, +) +from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import ( + OpenAIChatCompletion, +) + +from metagpt.config import CONFIG + + +def make_sk_kernel(): + kernel = sk.Kernel() + if CONFIG.openai_api_type == "azure": + kernel.add_chat_service( + "chat_completion", + AzureChatCompletion(CONFIG.deployment_name, CONFIG.openai_api_base, CONFIG.openai_api_key), + ) + else: + kernel.add_chat_service( + "chat_completion", + OpenAIChatCompletion( + CONFIG.openai_api_model, CONFIG.openai_api_key, org_id=None, endpoint=CONFIG.openai_api_base + ), + ) + + return kernel diff --git a/requirements.txt b/requirements.txt index 24f539a98..3a75f924c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,4 +38,7 @@ typing-inspect==0.8.0 typing_extensions==4.5.0 libcst==1.0.1 qdrant-client==1.4.0 -pytest-mock==3.11.1 \ No newline at end of file +pytest-mock==3.11.1 +open-interpreter==0.1.3 +ta==0.10.2 + diff --git a/tests/metagpt/actions/test_clone_function.py b/tests/metagpt/actions/test_clone_function.py new file mode 100644 index 000000000..6d4432dcd --- /dev/null +++ b/tests/metagpt/actions/test_clone_function.py @@ -0,0 +1,54 @@ +import pytest + +from metagpt.actions.clone_function import CloneFunction, run_function_code + + +source_code = """ +import pandas as pd +import ta + +def user_indicator(): + # 读取股票数据 + stock_data = pd.read_csv('./tests/data/baba_stock.csv') + stock_data.head() + # 计算简单移动平均线 + stock_data['SMA'] = ta.trend.sma_indicator(stock_data['Close'], window=6) + stock_data[['Date', 'Close', 'SMA']].head() + # 计算布林带 + stock_data['bb_upper'], stock_data['bb_middle'], stock_data['bb_lower'] = ta.volatility.bollinger_hband_indicator(stock_data['Close'], window=20), ta.volatility.bollinger_mavg(stock_data['Close'], window=20), ta.volatility.bollinger_lband_indicator(stock_data['Close'], window=20) + stock_data[['Date', 'Close', 'bb_upper', 'bb_middle', 'bb_lower']].head() +""" + +template_code = """ +def stock_indicator(stock_path: str, indicators=['Simple Moving Average', 'BollingerBands', 'MACD]) -> pd.DataFrame: + import pandas as pd + # here is your code. +""" + + +def get_expected_res(): + import pandas as pd + import ta + + # 读取股票数据 + stock_data = pd.read_csv('./tests/data/baba_stock.csv') + stock_data.head() + # 计算简单移动平均线 + stock_data['SMA'] = ta.trend.sma_indicator(stock_data['Close'], window=6) + stock_data[['Date', 'Close', 'SMA']].head() + # 计算布林带 + stock_data['bb_upper'], stock_data['bb_middle'], stock_data['bb_lower'] = ta.volatility.bollinger_hband_indicator(stock_data['Close'], window=20), ta.volatility.bollinger_mavg(stock_data['Close'], window=20), ta.volatility.bollinger_lband_indicator(stock_data['Close'], window=20) + stock_data[['Date', 'Close', 'bb_upper', 'bb_middle', 'bb_lower']].head() + return stock_data + + +@pytest.mark.asyncio +async def test_clone_function(): + clone = CloneFunction() + code = await clone.run(template_code, source_code) + assert 'def ' in code + stock_path = './tests/data/baba_stock.csv' + df, msg = run_function_code(code, 'stock_indicator', stock_path) + assert not msg + expected_df = get_expected_res() + assert df.equals(expected_df) diff --git a/tests/metagpt/planner/__init__.py b/tests/metagpt/planner/__init__.py new file mode 100644 index 000000000..85e01b36b --- /dev/null +++ b/tests/metagpt/planner/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/16 20:03 +@Author : femto Zheng +@File : __init__.py +""" diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py new file mode 100644 index 000000000..5ab9a493f --- /dev/null +++ b/tests/metagpt/planner/test_action_planner.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/16 20:03 +@Author : femto Zheng +@File : test_basic_planner.py +""" +import pytest +from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill +from semantic_kernel.planning.action_planner.action_planner import ActionPlanner + +from metagpt.actions import BossRequirement +from metagpt.roles.sk_agent import SkAgent +from metagpt.schema import Message + + +@pytest.mark.asyncio +async def test_action_planner(): + role = SkAgent(planner_cls=ActionPlanner) + # let's give the agent 4 skills + role.import_skill(MathSkill(), "math") + role.import_skill(FileIOSkill(), "fileIO") + role.import_skill(TimeSkill(), "time") + role.import_skill(TextSkill(), "text") + task = "What is the sum of 110 and 990?" + role.recv(Message(content=task, cause_by=BossRequirement)) + + await role._think() # it will choose mathskill.Add + assert "1100" == (await role._act()).content diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py new file mode 100644 index 000000000..03a82ec5e --- /dev/null +++ b/tests/metagpt/planner/test_basic_planner.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/16 20:03 +@Author : femto Zheng +@File : test_basic_planner.py +""" +import pytest +from semantic_kernel.core_skills import TextSkill + +from metagpt.actions import BossRequirement +from metagpt.const import SKILL_DIRECTORY +from metagpt.roles.sk_agent import SkAgent +from metagpt.schema import Message + + +@pytest.mark.asyncio +async def test_basic_planner(): + task = """ + Tomorrow is Valentine's day. I need to come up with a few date ideas. She speaks French so write it in French. + Convert the text to uppercase""" + role = SkAgent() + + # let's give the agent some skills + role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "SummarizeSkill") + role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") + role.import_skill(TextSkill(), "TextSkill") + # using BasicPlanner + role.recv(Message(content=task, cause_by=BossRequirement)) + await role._think() + # assuming sk_agent will think he needs WriterSkill.Brainstorm and WriterSkill.Translate + assert "WriterSkill.Brainstorm" in role.plan.generated_plan.result + assert "WriterSkill.Translate" in role.plan.generated_plan.result + # assert "SALUT" in (await role._act()).content #content will be some French diff --git a/tests/metagpt/tools/test_code_interpreter.py b/tests/metagpt/tools/test_code_interpreter.py new file mode 100644 index 000000000..0eec3f80b --- /dev/null +++ b/tests/metagpt/tools/test_code_interpreter.py @@ -0,0 +1,42 @@ +import pytest +import pandas as pd +from pathlib import Path + +from tests.data import sales_desc, store_desc +from metagpt.tools.code_interpreter import OpenCodeInterpreter, OpenInterpreterDecorator +from metagpt.actions import Action +from metagpt.logs import logger + + +logger.add('./tests/data/test_ci.log') +stock = "./tests/data/baba_stock.csv" + + +# TODO: 需要一种表格数据格式,能够支持schame管理的,标注字段类型和字段含义。 +class CreateStockIndicators(Action): + @OpenInterpreterDecorator(save_code=True, code_file_path="./tests/data/stock_indicators.py") + async def run(self, stock_path: str, indicators=['Simple Moving Average', 'BollingerBands']) -> pd.DataFrame: + """对stock_path中的股票数据, 使用pandas和ta计算indicators中的技术指标, 返回带有技术指标的股票数据,不需要去除空值, 不需要安装任何包; + 指标生成对应的三列: SMA, BB_upper, BB_lower + """ + ... + + +@pytest.mark.asyncio +async def test_actions(): + # 计算指标 + indicators = ['Simple Moving Average', 'BollingerBands'] + stocker = CreateStockIndicators() + df, msg = await stocker.run(stock, indicators=indicators) + assert isinstance(df, pd.DataFrame) + assert 'Close' in df.columns + assert 'Date' in df.columns + # 将df保存为文件,将文件路径传入到下一个action + df_path = './tests/data/stock_indicators.csv' + df.to_csv(df_path) + assert Path(df_path).is_file() + # 可视化指标结果 + figure_path = './tests/data/figure_ci.png' + ci_ploter = OpenCodeInterpreter() + ci_ploter.chat(f"使用seaborn对{df_path}中与股票布林带有关的数据列的Date, Close, SMA, BB_upper(布林带上界), BB_lower(布林带下界)进行可视化, 可视化图片保存在{figure_path}中。不需要任何指标计算,把Date列转换为日期类型。要求图片优美,BB_upper, BB_lower之间使用合适的颜色填充。") + assert Path(figure_path).is_file() diff --git a/tests/metagpt/utils/test_file.py b/tests/metagpt/utils/test_file.py index a9f1a353d..b30e6be93 100644 --- a/tests/metagpt/utils/test_file.py +++ b/tests/metagpt/utils/test_file.py @@ -7,7 +7,6 @@ """ from pathlib import Path -import aiofiles import pytest from metagpt.utils.file import File @@ -18,10 +17,10 @@ from metagpt.utils.file import File ("root_path", "filename", "content"), [(Path("/code/MetaGPT/data/tutorial_docx/2023-09-07_17-05-20"), "test.md", "Hello World!")] ) -async def test_write_file(root_path: Path, filename: str, content: bytes): +async def test_write_and_read_file(root_path: Path, filename: str, content: bytes): full_file_name = await File.write(root_path=root_path, filename=filename, content=content.encode('utf-8')) assert isinstance(full_file_name, Path) assert root_path / filename == full_file_name - async with aiofiles.open(full_file_name, mode="r") as reader: - body = await reader.read() - assert body == content \ No newline at end of file + file_data = await File.read(full_file_name) + assert file_data.decode("utf-8") == content + diff --git a/tests/metagpt/utils/test_output_parser.py b/tests/metagpt/utils/test_output_parser.py index c56cff6fa..2b706efc4 100644 --- a/tests/metagpt/utils/test_output_parser.py +++ b/tests/metagpt/utils/test_output_parser.py @@ -5,7 +5,7 @@ @Author : chengmaoyu @File : test_output_parser.py """ -from typing import List, Tuple +from typing import List, Tuple, Union import pytest @@ -64,6 +64,59 @@ def test_parse_data(): assert OutputParser.parse_data(test_data) == expected_result +@pytest.mark.parametrize( + ("text", "data_type", "parsed_data", "expected_exception"), + [ + ( + """xxx [1, 2, ["a", "b", [3, 4]], {"x": 5, "y": [6, 7]}] xxx""", + list, + [1, 2, ["a", "b", [3, 4]], {"x": 5, "y": [6, 7]}], + None, + ), + ( + """xxx ["1", "2", "3"] xxx \n xxx \t xx""", + list, + ["1", "2", "3"], + None, + ), + ( + """{"title": "a", "directory": {"sub_dir1": ["title1, title2"]}, "sub_dir2": [1, 2]}""", + dict, + {"title": "a", "directory": {"sub_dir1": ["title1, title2"]}, "sub_dir2": [1, 2]}, + None, + ), + ( + """xxx {"title": "x", \n \t "directory": ["x", \n "y"]} xxx \n xxx \t xx""", + dict, + {"title": "x", "directory": ["x", "y"]}, + None, + ), + ( + """xxx xx""", + list, + None, + Exception, + ), + ( + """xxx [1, 2, []xx""", + list, + None, + Exception, + ), + ] +) +def test_extract_struct(text: str, data_type: Union[type(list), type(dict)], parsed_data: Union[list, dict], expected_exception): + def case(): + resp = OutputParser.extract_struct(text, data_type) + assert resp == parsed_data + + if expected_exception: + with pytest.raises(expected_exception): + case() + else: + case() + + if __name__ == '__main__': t_text = ''' ## Required Python third-party packages