From 791ab749adab3d527c6a700bfd89d855a2ba347a Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 21 Feb 2024 16:49:59 +0800 Subject: [PATCH 01/21] refine code to avoid config error --- metagpt/software_company.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 26bb29cd1..f290d497a 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -2,14 +2,11 @@ # -*- coding: utf-8 -*- import asyncio -import shutil from pathlib import Path import typer -from metagpt.config2 import config -from metagpt.const import CONFIG_ROOT, METAGPT_ROOT -from metagpt.context import Context +from metagpt.const import CONFIG_ROOT from metagpt.utils.project_repo import ProjectRepo app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False) @@ -30,6 +27,8 @@ def generate_repo( recover_path=None, ) -> ProjectRepo: """Run the startup logic. Can be called from CLI or other Python scripts.""" + from metagpt.config2 import config + from metagpt.context import Context from metagpt.roles import ( Architect, Engineer, @@ -122,7 +121,17 @@ def startup( ) -def copy_config_to(config_path=METAGPT_ROOT / "config" / "config2.yaml"): +DEFAULT_CONFIG = """# Full Example: https://github.com/geekan/MetaGPT/blob/main/config/config2.example.yaml +# Reflected Code: https://github.com/geekan/MetaGPT/blob/main/metagpt/config2.py +llm: + api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options + model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview + base_url: "https://api.openai.com/v1" # or forward url / other llm url + api_key: "YOUR_API_KEY" +""" + + +def copy_config_to(): """Initialize the configuration file for MetaGPT.""" target_path = CONFIG_ROOT / "config2.yaml" @@ -136,7 +145,7 @@ def copy_config_to(config_path=METAGPT_ROOT / "config" / "config2.yaml"): print(f"Existing configuration file backed up at {backup_path}") # 复制文件 - shutil.copy(str(config_path), target_path) + target_path.write_text(DEFAULT_CONFIG, encoding="utf-8") print(f"Configuration file initialized at {target_path}") From 0a6dc8f7e7d888f7556a0c009c422d61933c0406 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Fri, 23 Feb 2024 10:15:49 +0800 Subject: [PATCH 02/21] update version to 0.7.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ed3d5f78f..f5d880ac9 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.2", + version="0.7.3", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", From 08bf756ae8cc81bfa483d5fcfbb9a45b49decca6 Mon Sep 17 00:00:00 2001 From: yzlin Date: Fri, 1 Mar 2024 18:33:49 +0800 Subject: [PATCH 03/21] rename interpreter --- README.md | 4 +++- examples/di/README.md | 18 ++++++++++++++++++ examples/{mi => di}/crawl_webpage.py | 6 +++--- examples/{mi => di}/data_visualization.py | 6 +++--- examples/{mi => di}/email_summary.py | 6 +++--- examples/{mi => di}/imitate_webpage.py | 6 +++--- examples/{mi => di}/machine_learning.py | 6 +++--- examples/{mi => di}/ml_engineer_with_tools.py | 2 +- examples/{mi => di}/ocr_receipt.py | 6 +++--- examples/{mi => di}/rm_image_background.py | 6 +++--- examples/{mi => di}/sd_tool_usage.py | 6 +++--- examples/{mi => di}/solve_math_problems.py | 6 +++--- examples/mi/README.md | 18 ------------------ metagpt/actions/__init__.py | 6 +++--- metagpt/actions/{mi => di}/__init__.py | 0 metagpt/actions/{mi => di}/ask_review.py | 0 metagpt/actions/{mi => di}/debug_code.py | 2 +- metagpt/actions/{mi => di}/execute_nb_code.py | 0 metagpt/actions/{mi => di}/ml_action.py | 6 +++--- .../actions/{mi => di}/write_analysis_code.py | 2 +- metagpt/actions/{mi => di}/write_plan.py | 2 +- metagpt/prompts/{mi => di}/__init__.py | 0 metagpt/prompts/{mi => di}/ml_action.py | 0 .../prompts/{mi => di}/write_analysis_code.py | 0 metagpt/roles/{mi => di}/__init__.py | 0 .../interpreter.py => di/data_interpreter.py} | 12 ++++++------ metagpt/roles/{mi => di}/ml_engineer.py | 10 +++++----- metagpt/strategy/planner.py | 4 ++-- metagpt/strategy/solver.py | 4 ++-- setup.py | 2 +- .../actions/{mi => di}/test_ask_review.py | 2 +- .../actions/{mi => di}/test_debug_code.py | 2 +- .../actions/{mi => di}/test_execute_nb_code.py | 2 +- .../actions/{mi => di}/test_ml_action.py | 2 +- .../{mi => di}/test_write_analysis_code.py | 4 ++-- .../actions/{mi => di}/test_write_plan.py | 2 +- .../test_data_interpreter.py} | 10 +++++----- .../roles/{mi => di}/test_ml_engineer.py | 6 +++--- tests/metagpt/utils/test_save_code.py | 2 +- 39 files changed, 90 insertions(+), 88 deletions(-) create mode 100644 examples/di/README.md rename examples/{mi => di}/crawl_webpage.py (78%) rename examples/{mi => di}/data_visualization.py (59%) rename examples/{mi => di}/email_summary.py (90%) rename examples/{mi => di}/imitate_webpage.py (81%) rename examples/{mi => di}/machine_learning.py (67%) rename examples/{mi => di}/ml_engineer_with_tools.py (94%) rename examples/{mi => di}/ocr_receipt.py (81%) rename examples/{mi => di}/rm_image_background.py (75%) rename examples/{mi => di}/sd_tool_usage.py (73%) rename examples/{mi => di}/solve_math_problems.py (70%) delete mode 100644 examples/mi/README.md rename metagpt/actions/{mi => di}/__init__.py (100%) rename metagpt/actions/{mi => di}/ask_review.py (100%) rename metagpt/actions/{mi => di}/debug_code.py (98%) rename metagpt/actions/{mi => di}/execute_nb_code.py (100%) rename metagpt/actions/{mi => di}/ml_action.py (93%) rename metagpt/actions/{mi => di}/write_analysis_code.py (99%) rename metagpt/actions/{mi => di}/write_plan.py (98%) rename metagpt/prompts/{mi => di}/__init__.py (100%) rename metagpt/prompts/{mi => di}/ml_action.py (100%) rename metagpt/prompts/{mi => di}/write_analysis_code.py (100%) rename metagpt/roles/{mi => di}/__init__.py (100%) rename metagpt/roles/{mi/interpreter.py => di/data_interpreter.py} (91%) rename metagpt/roles/{mi => di}/ml_engineer.py (89%) rename tests/metagpt/actions/{mi => di}/test_ask_review.py (84%) rename tests/metagpt/actions/{mi => di}/test_debug_code.py (96%) rename tests/metagpt/actions/{mi => di}/test_execute_nb_code.py (98%) rename tests/metagpt/actions/{mi => di}/test_ml_action.py (95%) rename tests/metagpt/actions/{mi => di}/test_write_analysis_code.py (99%) rename tests/metagpt/actions/{mi => di}/test_write_plan.py (95%) rename tests/metagpt/roles/{mi/test_interpreter.py => di/test_data_interpreter.py} (65%) rename tests/metagpt/roles/{mi => di}/test_ml_engineer.py (94%) diff --git a/README.md b/README.md index a1aa5ded8..197c387fd 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,9 @@ # MetaGPT: The Multi-Agent Framework

## News -🚀 Feb. 08, 2024: [v0.7.0](https://github.com/geekan/MetaGPT/releases/tag/v0.7.0) released, supporting assigning different LLMs to different Roles. We also introduced [Interpreter](https://github.com/geekan/MetaGPT/blob/main/examples/mi/README.md), a powerful agent capable of solving a wide range of real-world problems. +🚀 March. 01, 2024: Our Data Interpreter paper is on arxiv. Find all design and benchmark details [here](https://arxiv.org/abs/2402.18679)! + +🚀 Feb. 08, 2024: [v0.7.0](https://github.com/geekan/MetaGPT/releases/tag/v0.7.0) released, supporting assigning different LLMs to different Roles. We also introduced [Data Interpreter](https://github.com/geekan/MetaGPT/blob/main/examples/di/README.md), a powerful agent capable of solving a wide range of real-world problems. 🚀 Jan. 16, 2024: Our paper [MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework ](https://arxiv.org/abs/2308.00352) accepted for oral presentation **(top 1.2%)** at ICLR 2024, **ranking #1** in the LLM-based Agent category. diff --git a/examples/di/README.md b/examples/di/README.md new file mode 100644 index 000000000..fb4e8044b --- /dev/null +++ b/examples/di/README.md @@ -0,0 +1,18 @@ +# Data Interpreter (DI) + +## What is Data Interpreter +Data Interpreter is an agent who solves problems through codes. It understands user requirements, makes plans, writes codes for execution, and uses tools if necessary. These capabilities enable it to tackle a wide range of scenarios, please check out the examples below. + +## Example List +- Data visualization +- Machine learning modeling +- Image background removal +- Solve math problems +- Receipt OCR +- Tool usage: web page imitation +- Tool usage: web crawling +- Tool usage: text2image +- Tool usage: email summarization and response +- More on the way! + +Please see [here](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/intro.html) for detailed explanation. \ No newline at end of file diff --git a/examples/mi/crawl_webpage.py b/examples/di/crawl_webpage.py similarity index 78% rename from examples/mi/crawl_webpage.py rename to examples/di/crawl_webpage.py index b5d2fb3d0..f06b85d9b 100644 --- a/examples/mi/crawl_webpage.py +++ b/examples/di/crawl_webpage.py @@ -5,15 +5,15 @@ @File : crawl_webpage.py """ -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): prompt = """Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/, and save it to a csv file. paper title must include `multiagent` or `large language model`. *notice: print key variables*""" - mi = Interpreter(use_tools=True) + di = DataInterpreter(use_tools=True) - await mi.run(prompt) + await di.run(prompt) if __name__ == "__main__": diff --git a/examples/mi/data_visualization.py b/examples/di/data_visualization.py similarity index 59% rename from examples/mi/data_visualization.py rename to examples/di/data_visualization.py index 2e4acc9b4..9af72dc42 100644 --- a/examples/mi/data_visualization.py +++ b/examples/di/data_visualization.py @@ -1,11 +1,11 @@ import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=False) - await mi.run(requirement) + di = DataInterpreter(use_tools=False) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/email_summary.py b/examples/di/email_summary.py similarity index 90% rename from examples/mi/email_summary.py rename to examples/di/email_summary.py index e1511c5b0..af081fee2 100644 --- a/examples/mi/email_summary.py +++ b/examples/di/email_summary.py @@ -6,7 +6,7 @@ """ import os -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): @@ -22,9 +22,9 @@ async def main(): Firstly, Please help me fetch the latest 5 senders and full letter contents. Then, summarize each of the 5 emails into one sentence (you can do this by yourself, no need to import other models to do this) and output them in a markdown format.""" - mi = Interpreter(use_tools=True) + di = DataInterpreter(use_tools=True) - await mi.run(prompt) + await di.run(prompt) if __name__ == "__main__": diff --git a/examples/mi/imitate_webpage.py b/examples/di/imitate_webpage.py similarity index 81% rename from examples/mi/imitate_webpage.py rename to examples/di/imitate_webpage.py index 0e9ca731d..e2c99b874 100644 --- a/examples/mi/imitate_webpage.py +++ b/examples/di/imitate_webpage.py @@ -5,7 +5,7 @@ @Author : mannaandpoem @File : imitate_webpage.py """ -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): @@ -15,9 +15,9 @@ Firstly, utilize Selenium and WebDriver for rendering. Secondly, convert image to a webpage including HTML, CSS and JS in one go. Finally, save webpage in a text file. Note: All required dependencies and environments have been fully installed and configured.""" - mi = Interpreter(use_tools=True) + di = DataInterpreter(use_tools=True) - await mi.run(prompt) + await di.run(prompt) if __name__ == "__main__": diff --git a/examples/mi/machine_learning.py b/examples/di/machine_learning.py similarity index 67% rename from examples/mi/machine_learning.py rename to examples/di/machine_learning.py index a8ab5051e..a58735831 100644 --- a/examples/mi/machine_learning.py +++ b/examples/di/machine_learning.py @@ -1,12 +1,12 @@ import fire -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(auto_run: bool = True): requirement = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy." - mi = Interpreter(auto_run=auto_run) - await mi.run(requirement) + di = DataInterpreter(auto_run=auto_run) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/ml_engineer_with_tools.py b/examples/di/ml_engineer_with_tools.py similarity index 94% rename from examples/mi/ml_engineer_with_tools.py rename to examples/di/ml_engineer_with_tools.py index 9d0e7f951..6119ad843 100644 --- a/examples/mi/ml_engineer_with_tools.py +++ b/examples/di/ml_engineer_with_tools.py @@ -1,6 +1,6 @@ import asyncio -from metagpt.roles.mi.ml_engineer import MLEngineer +from metagpt.roles.di.ml_engineer import MLEngineer async def main(requirement: str): diff --git a/examples/mi/ocr_receipt.py b/examples/di/ocr_receipt.py similarity index 81% rename from examples/mi/ocr_receipt.py rename to examples/di/ocr_receipt.py index ffa5cff05..8b48be4f1 100644 --- a/examples/mi/ocr_receipt.py +++ b/examples/di/ocr_receipt.py @@ -1,4 +1,4 @@ -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): @@ -8,9 +8,9 @@ async def main(): requirement = f"""This is a {language} receipt image. Your goal is to perform OCR on images using PaddleOCR, then extract the total amount from ocr text results, and finally save as table. Image path: {image_path}. NOTE: The environments for Paddle and PaddleOCR are all ready and has been fully installed.""" - mi = Interpreter() + di = DataInterpreter() - await mi.run(requirement) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/rm_image_background.py b/examples/di/rm_image_background.py similarity index 75% rename from examples/mi/rm_image_background.py rename to examples/di/rm_image_background.py index 57e89b103..b74a79eeb 100644 --- a/examples/mi/rm_image_background.py +++ b/examples/di/rm_image_background.py @@ -1,11 +1,11 @@ import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=False) - await mi.run(requirement) + di = DataInterpreter(use_tools=False) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/sd_tool_usage.py b/examples/di/sd_tool_usage.py similarity index 73% rename from examples/mi/sd_tool_usage.py rename to examples/di/sd_tool_usage.py index f8507a411..69c7df5bd 100644 --- a/examples/mi/sd_tool_usage.py +++ b/examples/di/sd_tool_usage.py @@ -4,12 +4,12 @@ # @Desc : import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=True, goal=requirement) - await mi.run(requirement) + di = DataInterpreter(use_tools=True, goal=requirement) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/solve_math_problems.py b/examples/di/solve_math_problems.py similarity index 70% rename from examples/mi/solve_math_problems.py rename to examples/di/solve_math_problems.py index dce2edb00..35a69c953 100644 --- a/examples/mi/solve_math_problems.py +++ b/examples/di/solve_math_problems.py @@ -1,11 +1,11 @@ import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=False) - await mi.run(requirement) + di = DataInterpreter(use_tools=False) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/README.md b/examples/mi/README.md deleted file mode 100644 index 1734ba388..000000000 --- a/examples/mi/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# MetaGPT Interpreter (MI) - -## What is Interpreter -Interpreter is an agent who solves problems through codes. It understands user requirements, makes plans, writes codes for execution, and uses tools if necessary. These capabilities enable it to tackle a wide range of scenarios, please check out the examples below. - -## Example List -- Data visualization -- Machine learning modeling -- Image background removal -- Solve math problems -- Receipt OCR -- Tool usage: web page imitation -- Tool usage: web crawling -- Tool usage: text2image -- Tool usage: email summarization and response -- More on the way! - -Please see [here](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/mi_intro.html) for detailed explanation. \ No newline at end of file diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index 19a7c10d5..29af1dad1 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -22,9 +22,9 @@ from metagpt.actions.write_code_review import WriteCodeReview from metagpt.actions.write_prd import WritePRD from metagpt.actions.write_prd_review import WritePRDReview from metagpt.actions.write_test import WriteTest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.write_analysis_code import WriteCodeWithoutTools, WriteCodeWithTools -from metagpt.actions.mi.write_plan import WritePlan +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.write_analysis_code import WriteCodeWithoutTools, WriteCodeWithTools +from metagpt.actions.di.write_plan import WritePlan class ActionType(Enum): diff --git a/metagpt/actions/mi/__init__.py b/metagpt/actions/di/__init__.py similarity index 100% rename from metagpt/actions/mi/__init__.py rename to metagpt/actions/di/__init__.py diff --git a/metagpt/actions/mi/ask_review.py b/metagpt/actions/di/ask_review.py similarity index 100% rename from metagpt/actions/mi/ask_review.py rename to metagpt/actions/di/ask_review.py diff --git a/metagpt/actions/mi/debug_code.py b/metagpt/actions/di/debug_code.py similarity index 98% rename from metagpt/actions/mi/debug_code.py rename to metagpt/actions/di/debug_code.py index db3b7a9db..f589436f5 100644 --- a/metagpt/actions/mi/debug_code.py +++ b/metagpt/actions/di/debug_code.py @@ -1,6 +1,6 @@ from __future__ import annotations -from metagpt.actions.mi.write_analysis_code import BaseWriteAnalysisCode +from metagpt.actions.di.write_analysis_code import BaseWriteAnalysisCode from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import create_func_call_config diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/di/execute_nb_code.py similarity index 100% rename from metagpt/actions/mi/execute_nb_code.py rename to metagpt/actions/di/execute_nb_code.py diff --git a/metagpt/actions/mi/ml_action.py b/metagpt/actions/di/ml_action.py similarity index 93% rename from metagpt/actions/mi/ml_action.py rename to metagpt/actions/di/ml_action.py index 60b2fb547..d49b7b67d 100644 --- a/metagpt/actions/mi/ml_action.py +++ b/metagpt/actions/di/ml_action.py @@ -3,14 +3,14 @@ from __future__ import annotations from typing import Tuple from metagpt.actions import Action -from metagpt.actions.mi.write_analysis_code import WriteCodeWithTools -from metagpt.prompts.mi.ml_action import ( +from metagpt.actions.di.write_analysis_code import WriteCodeWithTools +from metagpt.prompts.di.ml_action import ( ML_GENERATE_CODE_PROMPT, ML_TOOL_USAGE_PROMPT, PRINT_DATA_COLUMNS, UPDATE_DATA_COLUMNS, ) -from metagpt.prompts.mi.write_analysis_code import CODE_GENERATOR_WITH_TOOLS +from metagpt.prompts.di.write_analysis_code import CODE_GENERATOR_WITH_TOOLS from metagpt.schema import Message, Plan from metagpt.utils.common import create_func_call_config, remove_comments diff --git a/metagpt/actions/mi/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py similarity index 99% rename from metagpt/actions/mi/write_analysis_code.py rename to metagpt/actions/di/write_analysis_code.py index b3d0632b6..0c4980c2b 100644 --- a/metagpt/actions/mi/write_analysis_code.py +++ b/metagpt/actions/di/write_analysis_code.py @@ -10,7 +10,7 @@ from typing import Tuple from metagpt.actions import Action from metagpt.logs import logger -from metagpt.prompts.mi.write_analysis_code import ( +from metagpt.prompts.di.write_analysis_code import ( CODE_GENERATOR_WITH_TOOLS, SELECT_FUNCTION_TOOLS, TOOL_RECOMMENDATION_PROMPT, diff --git a/metagpt/actions/mi/write_plan.py b/metagpt/actions/di/write_plan.py similarity index 98% rename from metagpt/actions/mi/write_plan.py rename to metagpt/actions/di/write_plan.py index 8067d7b87..518dfb6c6 100644 --- a/metagpt/actions/mi/write_plan.py +++ b/metagpt/actions/di/write_plan.py @@ -12,7 +12,7 @@ from typing import Tuple from metagpt.actions import Action from metagpt.logs import logger -from metagpt.prompts.mi.write_analysis_code import ( +from metagpt.prompts.di.write_analysis_code import ( ASSIGN_TASK_TYPE_CONFIG, ASSIGN_TASK_TYPE_PROMPT, ) diff --git a/metagpt/prompts/mi/__init__.py b/metagpt/prompts/di/__init__.py similarity index 100% rename from metagpt/prompts/mi/__init__.py rename to metagpt/prompts/di/__init__.py diff --git a/metagpt/prompts/mi/ml_action.py b/metagpt/prompts/di/ml_action.py similarity index 100% rename from metagpt/prompts/mi/ml_action.py rename to metagpt/prompts/di/ml_action.py diff --git a/metagpt/prompts/mi/write_analysis_code.py b/metagpt/prompts/di/write_analysis_code.py similarity index 100% rename from metagpt/prompts/mi/write_analysis_code.py rename to metagpt/prompts/di/write_analysis_code.py diff --git a/metagpt/roles/mi/__init__.py b/metagpt/roles/di/__init__.py similarity index 100% rename from metagpt/roles/mi/__init__.py rename to metagpt/roles/di/__init__.py diff --git a/metagpt/roles/mi/interpreter.py b/metagpt/roles/di/data_interpreter.py similarity index 91% rename from metagpt/roles/mi/interpreter.py rename to metagpt/roles/di/data_interpreter.py index fa50098e9..b3a2e789f 100644 --- a/metagpt/roles/mi/interpreter.py +++ b/metagpt/roles/di/data_interpreter.py @@ -2,9 +2,9 @@ from __future__ import annotations from pydantic import Field -from metagpt.actions.mi.ask_review import ReviewConst -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.write_analysis_code import ( +from metagpt.actions.di.ask_review import ReviewConst +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.write_analysis_code import ( WriteCodeWithoutTools, WriteCodeWithTools, ) @@ -13,9 +13,9 @@ from metagpt.roles import Role from metagpt.schema import Message, Task, TaskResult -class Interpreter(Role): - name: str = "Ivy" - profile: str = "Interpreter" +class DataInterpreter(Role): + name: str = "David" + profile: str = "DataInterpreter" auto_run: bool = True use_tools: bool = False execute_code: ExecuteNbCode = Field(default_factory=ExecuteNbCode, exclude=True) diff --git a/metagpt/roles/mi/ml_engineer.py b/metagpt/roles/di/ml_engineer.py similarity index 89% rename from metagpt/roles/mi/ml_engineer.py rename to metagpt/roles/di/ml_engineer.py index 78d605d3e..b33b166cf 100644 --- a/metagpt/roles/mi/ml_engineer.py +++ b/metagpt/roles/di/ml_engineer.py @@ -1,13 +1,13 @@ -from metagpt.actions.mi.debug_code import DebugCode -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.ml_action import UpdateDataColumns, WriteCodeWithToolsML +from metagpt.actions.di.debug_code import DebugCode +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.ml_action import UpdateDataColumns, WriteCodeWithToolsML from metagpt.logs import logger -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter from metagpt.tools.tool_type import ToolType from metagpt.utils.common import any_to_str -class MLEngineer(Interpreter): +class MLEngineer(DataInterpreter): name: str = "Mark" profile: str = "MLEngineer" debug_context: list = [] diff --git a/metagpt/strategy/planner.py b/metagpt/strategy/planner.py index 99d16f78b..44294be00 100644 --- a/metagpt/strategy/planner.py +++ b/metagpt/strategy/planner.py @@ -4,8 +4,8 @@ import json from pydantic import BaseModel, Field -from metagpt.actions.mi.ask_review import AskReview, ReviewConst -from metagpt.actions.mi.write_plan import ( +from metagpt.actions.di.ask_review import AskReview, ReviewConst +from metagpt.actions.di.write_plan import ( WritePlan, precheck_update_plan_from_rsp, update_plan_from_rsp, diff --git a/metagpt/strategy/solver.py b/metagpt/strategy/solver.py index ab16d82bd..e532f736b 100644 --- a/metagpt/strategy/solver.py +++ b/metagpt/strategy/solver.py @@ -49,8 +49,8 @@ class TOTSolver(BaseSolver): raise NotImplementedError -class InterpreterSolver(BaseSolver): - """InterpreterSolver: Write&Run code in the graph""" +class DataInterpreterSolver(BaseSolver): + """DataInterpreterSolver: Write&Run code in the graph""" async def solve(self): raise NotImplementedError diff --git a/setup.py b/setup.py index f5d880ac9..2044e577a 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.3", + version="0.7.4", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/metagpt/actions/mi/test_ask_review.py b/tests/metagpt/actions/di/test_ask_review.py similarity index 84% rename from tests/metagpt/actions/mi/test_ask_review.py rename to tests/metagpt/actions/di/test_ask_review.py index 92e8bd046..6bb1accf5 100644 --- a/tests/metagpt/actions/mi/test_ask_review.py +++ b/tests/metagpt/actions/di/test_ask_review.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.ask_review import AskReview +from metagpt.actions.di.ask_review import AskReview @pytest.mark.asyncio diff --git a/tests/metagpt/actions/mi/test_debug_code.py b/tests/metagpt/actions/di/test_debug_code.py similarity index 96% rename from tests/metagpt/actions/mi/test_debug_code.py rename to tests/metagpt/actions/di/test_debug_code.py index 24cfcef10..67f72ad63 100644 --- a/tests/metagpt/actions/mi/test_debug_code.py +++ b/tests/metagpt/actions/di/test_debug_code.py @@ -5,7 +5,7 @@ import pytest -from metagpt.actions.mi.debug_code import DebugCode +from metagpt.actions.di.debug_code import DebugCode from metagpt.schema import Message ErrorStr = """Tested passed: diff --git a/tests/metagpt/actions/mi/test_execute_nb_code.py b/tests/metagpt/actions/di/test_execute_nb_code.py similarity index 98% rename from tests/metagpt/actions/mi/test_execute_nb_code.py rename to tests/metagpt/actions/di/test_execute_nb_code.py index 59a814054..b491dd212 100644 --- a/tests/metagpt/actions/mi/test_execute_nb_code.py +++ b/tests/metagpt/actions/di/test_execute_nb_code.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode, truncate +from metagpt.actions.di.execute_nb_code import ExecuteNbCode, truncate @pytest.mark.asyncio diff --git a/tests/metagpt/actions/mi/test_ml_action.py b/tests/metagpt/actions/di/test_ml_action.py similarity index 95% rename from tests/metagpt/actions/mi/test_ml_action.py rename to tests/metagpt/actions/di/test_ml_action.py index 27d47b0e3..826a7fcf2 100644 --- a/tests/metagpt/actions/mi/test_ml_action.py +++ b/tests/metagpt/actions/di/test_ml_action.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.ml_action import WriteCodeWithToolsML +from metagpt.actions.di.ml_action import WriteCodeWithToolsML from metagpt.schema import Plan, Task diff --git a/tests/metagpt/actions/mi/test_write_analysis_code.py b/tests/metagpt/actions/di/test_write_analysis_code.py similarity index 99% rename from tests/metagpt/actions/mi/test_write_analysis_code.py rename to tests/metagpt/actions/di/test_write_analysis_code.py index 6c2228222..d6bbae1c6 100644 --- a/tests/metagpt/actions/mi/test_write_analysis_code.py +++ b/tests/metagpt/actions/di/test_write_analysis_code.py @@ -2,8 +2,8 @@ import asyncio import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.write_analysis_code import ( +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.write_analysis_code import ( WriteCodeWithoutTools, WriteCodeWithTools, ) diff --git a/tests/metagpt/actions/mi/test_write_plan.py b/tests/metagpt/actions/di/test_write_plan.py similarity index 95% rename from tests/metagpt/actions/mi/test_write_plan.py rename to tests/metagpt/actions/di/test_write_plan.py index 97632ea44..80b3399b8 100644 --- a/tests/metagpt/actions/mi/test_write_plan.py +++ b/tests/metagpt/actions/di/test_write_plan.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.write_plan import ( +from metagpt.actions.di.write_plan import ( Plan, Task, WritePlan, diff --git a/tests/metagpt/roles/mi/test_interpreter.py b/tests/metagpt/roles/di/test_data_interpreter.py similarity index 65% rename from tests/metagpt/roles/mi/test_interpreter.py rename to tests/metagpt/roles/di/test_data_interpreter.py index 3bae4a1ac..ba50f473b 100644 --- a/tests/metagpt/roles/mi/test_interpreter.py +++ b/tests/metagpt/roles/di/test_data_interpreter.py @@ -1,23 +1,23 @@ import pytest from metagpt.logs import logger -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter @pytest.mark.asyncio @pytest.mark.parametrize("auto_run", [(True), (False)]) async def test_interpreter(mocker, auto_run): - mocker.patch("metagpt.actions.mi.execute_nb_code.ExecuteNbCode.run", return_value=("a successful run", True)) + mocker.patch("metagpt.actions.di.execute_nb_code.ExecuteNbCode.run", return_value=("a successful run", True)) mocker.patch("builtins.input", return_value="confirm") requirement = "Run data analysis on sklearn Iris dataset, include a plot" tools = [] - mi = Interpreter(auto_run=auto_run, use_tools=True, tools=tools) - rsp = await mi.run(requirement) + di = DataInterpreter(auto_run=auto_run, use_tools=True, tools=tools) + rsp = await di.run(requirement) logger.info(rsp) assert len(rsp.content) > 0 - finished_tasks = mi.planner.plan.get_finished_tasks() + finished_tasks = di.planner.plan.get_finished_tasks() assert len(finished_tasks) > 0 assert len(finished_tasks[0].code) > 0 # check one task to see if code is recorded diff --git a/tests/metagpt/roles/mi/test_ml_engineer.py b/tests/metagpt/roles/di/test_ml_engineer.py similarity index 94% rename from tests/metagpt/roles/mi/test_ml_engineer.py rename to tests/metagpt/roles/di/test_ml_engineer.py index 921ac8822..08b92cd27 100644 --- a/tests/metagpt/roles/mi/test_ml_engineer.py +++ b/tests/metagpt/roles/di/test_ml_engineer.py @@ -1,11 +1,11 @@ import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.execute_nb_code import ExecuteNbCode from metagpt.logs import logger -from metagpt.roles.mi.ml_engineer import MLEngineer +from metagpt.roles.di.ml_engineer import MLEngineer from metagpt.schema import Message, Plan, Task from metagpt.tools.tool_type import ToolType -from tests.metagpt.actions.mi.test_debug_code import CODE, DebugContext, ErrorStr +from tests.metagpt.actions.di.test_debug_code import CODE, DebugContext, ErrorStr def test_mle_init(): diff --git a/tests/metagpt/utils/test_save_code.py b/tests/metagpt/utils/test_save_code.py index 9df2650f3..aceecec3b 100644 --- a/tests/metagpt/utils/test_save_code.py +++ b/tests/metagpt/utils/test_save_code.py @@ -6,7 +6,7 @@ import nbformat import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.execute_nb_code import ExecuteNbCode from metagpt.utils.common import read_json_file from metagpt.utils.save_code import DATA_PATH, save_code_file From 97bf135da6d78eed0a740beb8a87ca6a6fadc58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Mar 2024 12:20:58 +0800 Subject: [PATCH 04/21] feat: gemini + proxy --- metagpt/provider/google_gemini_api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 09e554205..8f8f5f707 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # @Desc : Google Gemini LLM from https://ai.google.dev/tutorials/python_quickstart +import os from typing import Optional, Union import google.generativeai as genai @@ -15,7 +16,7 @@ from google.generativeai.types.generation_types import ( ) from metagpt.configs.llm_config import LLMConfig, LLMType -from metagpt.logs import log_llm_stream +from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -51,6 +52,10 @@ class GeminiLLM(BaseLLM): self.llm = GeminiGenerativeModel(model_name=self.model) def __init_gemini(self, config: LLMConfig): + if config.proxy: + logger.info(f"Use proxy: {config.proxy}") + os.environ["HTTP_PROXY"] = config.proxy + os.environ["HTTP_PROXYS"] = config.proxy genai.configure(api_key=config.api_key) def _user_msg(self, msg: str, images: Optional[Union[str, list[str]]] = None) -> dict[str, str]: From f6260ec084443d9ae96dd6aec02afc595c709ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Mar 2024 12:27:28 +0800 Subject: [PATCH 05/21] feat: gemini + proxy --- metagpt/provider/google_gemini_api.py | 6 +++++- setup.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 2647ab16b..bdbf7acd6 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # @Desc : Google Gemini LLM from https://ai.google.dev/tutorials/python_quickstart - +import os from typing import Optional, Union import google.generativeai as genai @@ -58,6 +58,10 @@ class GeminiLLM(BaseLLM): self.llm = GeminiGenerativeModel(model_name=self.model) def __init_gemini(self, config: LLMConfig): + if config.proxy: + logger.info(f"Use proxy: {config.proxy}") + os.environ["HTTP_PROXY"] = config.proxy + os.environ["HTTP_PROXYS"] = config.proxy genai.configure(api_key=config.api_key) def _user_msg(self, msg: str, images: Optional[Union[str, list[str]]] = None) -> dict[str, str]: diff --git a/setup.py b/setup.py index 2044e577a..072237e8c 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.4", + version="0.7.5", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", From 0867dad4d7b66ad098934dd9d39b3cdb261c0c00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Mar 2024 17:04:44 +0800 Subject: [PATCH 06/21] fixbug: gbk UnicodeEncodeError --- metagpt/learn/skill_loader.py | 5 ++--- metagpt/utils/common.py | 16 +++++++++++---- metagpt/utils/dependency_file.py | 7 ++----- metagpt/utils/file_repository.py | 7 ++----- metagpt/utils/mermaid.py | 8 ++------ setup.py | 2 +- .../metagpt/roles/test_tutorial_assistant.py | 7 +++---- tests/metagpt/utils/test_common.py | 20 ++++++++++++++----- tests/metagpt/utils/test_git_repository.py | 5 ++--- tests/metagpt/utils/test_s3.py | 8 ++------ 10 files changed, 43 insertions(+), 42 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index bcf28bb87..e98f73cf9 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -9,11 +9,11 @@ from pathlib import Path from typing import Dict, List, Optional -import aiofiles import yaml from pydantic import BaseModel, Field from metagpt.context import Context +from metagpt.utils.common import aread class Example(BaseModel): @@ -68,8 +68,7 @@ class SkillsDeclaration(BaseModel): async def load(skill_yaml_file_name: Path = None) -> "SkillsDeclaration": if not skill_yaml_file_name: skill_yaml_file_name = Path(__file__).parent.parent.parent / "docs/.well-known/skills.yaml" - async with aiofiles.open(str(skill_yaml_file_name), mode="r") as reader: - data = await reader.read(-1) + data = await aread(filename=skill_yaml_file_name) skill_data = yaml.safe_load(data) return SkillsDeclaration(**skill_data) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 015902c3d..aba75fbec 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -29,6 +29,7 @@ from pathlib import Path from typing import Any, Callable, List, Tuple, Union import aiofiles +import chardet import loguru import requests from PIL import Image @@ -587,14 +588,21 @@ def role_raise_decorator(func): @handle_exception -async def aread(filename: str | Path, encoding=None) -> str: +async def aread(filename: str | Path, encoding="utf-8") -> str: """Read file asynchronously.""" - async with aiofiles.open(str(filename), mode="r", encoding=encoding) as reader: - content = await reader.read() + try: + async with aiofiles.open(str(filename), mode="r", encoding=encoding) as reader: + content = await reader.read() + except UnicodeDecodeError: + async with aiofiles.open(str(filename), mode="rb") as reader: + raw = await reader.read() + result = chardet.detect(raw) + detected_encoding = result["encoding"] + content = raw.decode(detected_encoding) return content -async def awrite(filename: str | Path, data: str, encoding=None): +async def awrite(filename: str | Path, data: str, encoding="utf-8"): """Write file asynchronously.""" pathname = Path(filename) pathname.parent.mkdir(parents=True, exist_ok=True) diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py index d3add1171..0a375051c 100644 --- a/metagpt/utils/dependency_file.py +++ b/metagpt/utils/dependency_file.py @@ -13,9 +13,7 @@ import re from pathlib import Path from typing import Set -import aiofiles - -from metagpt.utils.common import aread +from metagpt.utils.common import aread, awrite from metagpt.utils.exceptions import handle_exception @@ -45,8 +43,7 @@ class DependencyFile: async def save(self): """Save dependencies to the file asynchronously.""" data = json.dumps(self._dependencies) - async with aiofiles.open(str(self._filename), mode="w") as writer: - await writer.write(data) + await awrite(filename=self._filename, data=data) async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True): """Update dependencies for a file asynchronously. diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index d2a06963a..d19f2b705 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -14,11 +14,9 @@ from datetime import datetime from pathlib import Path from typing import Dict, List, Set -import aiofiles - from metagpt.logs import logger from metagpt.schema import Document -from metagpt.utils.common import aread +from metagpt.utils.common import aread, awrite from metagpt.utils.json_to_markdown import json_to_markdown @@ -55,8 +53,7 @@ class FileRepository: pathname = self.workdir / filename pathname.parent.mkdir(parents=True, exist_ok=True) content = content if content else "" # avoid `argument must be str, not None` to make it continue - async with aiofiles.open(str(pathname), mode="w") as writer: - await writer.write(content) + await awrite(filename=str(pathname), data=content) logger.info(f"save to: {str(pathname)}") if dependencies is not None: diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index ae3c5118f..e1d140e84 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -9,11 +9,9 @@ import asyncio import os from pathlib import Path -import aiofiles - from metagpt.config2 import config from metagpt.logs import logger -from metagpt.utils.common import check_cmd_exists +from metagpt.utils.common import awrite, check_cmd_exists async def mermaid_to_file(engine, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: @@ -30,9 +28,7 @@ async def mermaid_to_file(engine, mermaid_code, output_file_without_suffix, widt if dir_name and not os.path.exists(dir_name): os.makedirs(dir_name) tmp = Path(f"{output_file_without_suffix}.mmd") - async with aiofiles.open(tmp, "w", encoding="utf-8") as f: - await f.write(mermaid_code) - # tmp.write_text(mermaid_code, encoding="utf-8") + await awrite(filename=tmp, data=mermaid_code) if engine == "nodejs": if check_cmd_exists(config.mermaid.path) != 0: diff --git a/setup.py b/setup.py index 072237e8c..813d2410c 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.5", + version="0.7.6", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/metagpt/roles/test_tutorial_assistant.py b/tests/metagpt/roles/test_tutorial_assistant.py index c12c2b26e..732f346fd 100644 --- a/tests/metagpt/roles/test_tutorial_assistant.py +++ b/tests/metagpt/roles/test_tutorial_assistant.py @@ -6,11 +6,11 @@ @File : test_tutorial_assistant.py """ -import aiofiles import pytest from metagpt.const import TUTORIAL_PATH from metagpt.roles.tutorial_assistant import TutorialAssistant +from metagpt.utils.common import aread @pytest.mark.asyncio @@ -20,9 +20,8 @@ async def test_tutorial_assistant(language: str, topic: str, context): msg = await role.run(topic) assert TUTORIAL_PATH.exists() filename = msg.content - async with aiofiles.open(filename, mode="r", encoding="utf-8") as reader: - content = await reader.read() - assert "pip" in content + content = await aread(filename=filename) + assert "pip" in content if __name__ == "__main__": diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index 9b1fa878e..7c59b8072 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -13,7 +13,6 @@ import uuid from pathlib import Path from typing import Any, Set -import aiofiles import pytest from pydantic import BaseModel @@ -125,9 +124,7 @@ class TestGetProjectRoot: async def test_parse_data_exception(self, filename, want): pathname = Path(__file__).parent.parent.parent / "data/output_parser" / filename assert pathname.exists() - async with aiofiles.open(str(pathname), mode="r") as reader: - data = await reader.read() - + data = await aread(filename=pathname) result = OutputParser.parse_data(data=data) assert want in result @@ -198,12 +195,25 @@ class TestGetProjectRoot: @pytest.mark.asyncio async def test_read_write(self): - pathname = Path(__file__).parent / uuid.uuid4().hex / "test.tmp" + pathname = Path(__file__).parent / f"../../../workspace/unittest/{uuid.uuid4().hex}" / "test.tmp" await awrite(pathname, "ABC") data = await aread(pathname) assert data == "ABC" pathname.unlink(missing_ok=True) + @pytest.mark.asyncio + async def test_read_write_error_charset(self): + pathname = Path(__file__).parent / f"../../../workspace/unittest/{uuid.uuid4().hex}" / "test.txt" + content = "中国abc123\u27f6" + await awrite(filename=pathname, data=content) + data = await aread(filename=pathname) + assert data == content + + content = "GB18030 是中国国家标准局发布的新一代中文字符集标准，是 GBK 的升级版，支持更广泛的字符范围。" + await awrite(filename=pathname, data=content, encoding="gb2312") + data = await aread(filename=pathname, encoding="utf-8") + assert data == content + if __name__ == "__main__": pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index ea28b8f0b..480a22e24 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -10,15 +10,14 @@ import shutil from pathlib import Path -import aiofiles import pytest +from metagpt.utils.common import awrite from metagpt.utils.git_repository import GitRepository async def mock_file(filename, content=""): - async with aiofiles.open(str(filename), mode="w") as file: - await file.write(content) + await awrite(filename=filename, data=content) async def mock_repo(local_path) -> (GitRepository, Path): diff --git a/tests/metagpt/utils/test_s3.py b/tests/metagpt/utils/test_s3.py index b26ebe94d..c1a85f4ff 100644 --- a/tests/metagpt/utils/test_s3.py +++ b/tests/metagpt/utils/test_s3.py @@ -9,7 +9,6 @@ import uuid from pathlib import Path import aioboto3 -import aiofiles import pytest from metagpt.config2 import Config @@ -37,7 +36,7 @@ async def test_s3(mocker): conn = S3(s3) object_name = "unittest.bak" await conn.upload_file(bucket=s3.bucket, local_path=__file__, object_name=object_name) - pathname = (Path(__file__).parent / uuid.uuid4().hex).with_suffix(".bak") + pathname = (Path(__file__).parent / "../../../workspace/unittest" / uuid.uuid4().hex).with_suffix(".bak") pathname.unlink(missing_ok=True) await conn.download_file(bucket=s3.bucket, object_name=object_name, local_path=str(pathname)) assert pathname.exists() @@ -45,8 +44,7 @@ async def test_s3(mocker): assert url bin_data = await conn.get_object(bucket=s3.bucket, object_name=object_name) assert bin_data - async with aiofiles.open(__file__, mode="r", encoding="utf-8") as reader: - data = await reader.read() + data = await aread(filename=__file__) res = await conn.cache(data, ".bak", "script") assert "http" in res @@ -60,8 +58,6 @@ async def test_s3(mocker): except Exception: pass - await reader.close() - if __name__ == "__main__": pytest.main([__file__, "-s"]) From e39cafdd580ebcf1b587b6de40a51b5deae3fef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 11 Mar 2024 22:25:38 +0800 Subject: [PATCH 07/21] feat: + tree command --- metagpt/utils/tree.py | 99 ++++++++++++++++++++++++++++++++ tests/metagpt/utils/test_tree.py | 54 +++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 metagpt/utils/tree.py create mode 100644 tests/metagpt/utils/test_tree.py diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py new file mode 100644 index 000000000..49b5634c6 --- /dev/null +++ b/metagpt/utils/tree.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/3/11 +@Author : mashenquan +@File : tree.py +@Desc : Implement the same functionality as the `tree` command. +Example: + root + +-- dir1 + | +-- file1.txt + | +-- file2.txt + +-- dir2 + | +-- subdir1 + | | +-- file1.txt + | | +-- file2.txt + | +-- subdir2 + | +-- file1.txt + | +-- file2.txt + +-- file.txt +""" +from __future__ import annotations + +from pathlib import Path +from typing import Callable, Dict, List + +from anthropic import BaseModel +from pydantic import Field + + +class Tree(BaseModel): + """ + Represents a directory tree structure. + + Attributes: + root (str): The root directory of the tree. + tree (Dict[str, Dict]): The tree structure as a dictionary. + + Methods: + print: Print the directory tree structure. + + """ + + root: str + tree: Dict[str, Dict] = Field(default_factory=dict) + + def print(self, git_ignore_rules: Callable = None) -> str: + """ + Recursively traverses the directory structure and prints it out in a tree-like format. + + Args: + git_ignore_rules (Callable): Optional. A function to filter files to ignore. + + Returns: + str: A string representation of the directory tree. + + """ + root = Path(self.root).resolve() + self.tree[root.name] = self._list_children(root=root, git_ignore_rules=git_ignore_rules) + v = self._print_tree(self.tree) + return "\n".join(v) + + @staticmethod + def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: + tree = {} + for i in root.iterdir(): + if git_ignore_rules and git_ignore_rules(str(i)): + continue + if i.is_file(): + tree[i.name] = {} + else: + tree[i.name] = Tree._list_children(root=i, git_ignore_rules=git_ignore_rules) + return tree + + @staticmethod + def _print_tree(tree: Dict[str:Dict], indent: int = 0) -> List[str]: + ret = [] + for name, children in tree.items(): + ret.append(name) + if not children: + continue + lines = Tree._print_tree(tree=children, indent=indent + 1) + for j, v in enumerate(lines): + if v[0] not in ["+", " ", "|"]: + ret = Tree._add_line(ret) + row = f"+-- {v}" + else: + row = f" {v}" + ret.append(row) + return ret + + @staticmethod + def _add_line(rows: List[str]) -> List[str]: + for i in range(len(rows) - 1, -1, -1): + v = rows[i] + if v[0] != " ": + return rows + rows[i] = "|" + v[1:] + return rows diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py new file mode 100644 index 000000000..0d48f7ce3 --- /dev/null +++ b/tests/metagpt/utils/test_tree.py @@ -0,0 +1,54 @@ +from pathlib import Path +from typing import List + +import pytest +from gitignore_parser import parse_gitignore + +from metagpt.utils.tree import Tree + + +@pytest.mark.parametrize( + ("root", "rules"), + [ + (str(Path(__file__).parent / "../.."), None), + (str(Path(__file__).parent / "../.."), str(Path(__file__).parent / "../../../.gitignore")), + ], +) +def test_tree(root: str, rules: str): + gitignore_rules = parse_gitignore(full_path=rules) if rules else None + tree = Tree(root=root).print(git_ignore_rules=gitignore_rules) + assert tree + + +@pytest.mark.parametrize( + ("tree", "want"), + [ + ({"a": {"b": {}, "c": {}}}, ["a", "+-- b", "+-- c"]), + ({"a": {"b": {}, "c": {"d": {}}}}, ["a", "+-- b", "+-- c", " +-- d"]), + ( + {"a": {"b": {"e": {"f": {}, "g": {}}}, "c": {"d": {}}}}, + ["a", "+-- b", "| +-- e", "| +-- f", "| +-- g", "+-- c", " +-- d"], + ), + ( + {"h": {"a": {"b": {"e": {"f": {}, "g": {}}}, "c": {"d": {}}}, "i": {}}}, + [ + "h", + "+-- a", + "| +-- b", + "| | +-- e", + "| | +-- f", + "| | +-- g", + "| +-- c", + "| +-- d", + "+-- i", + ], + ), + ], +) +def test__print_tree(tree: dict, want: List[str]): + v = Tree._print_tree(tree) + assert v == want + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 7d32f9efe6958bf9b76b4309d426c590e3268e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 12 Mar 2024 11:10:09 +0800 Subject: [PATCH 08/21] refactor: Replace Tree class with tree() --- metagpt/utils/tree.py | 218 ++++++++++++++++++++----------- tests/metagpt/utils/test_tree.py | 8 +- 2 files changed, 149 insertions(+), 77 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index 49b5634c6..ad3373f5f 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -6,94 +6,166 @@ @File : tree.py @Desc : Implement the same functionality as the `tree` command. Example: - root - +-- dir1 - | +-- file1.txt - | +-- file2.txt - +-- dir2 - | +-- subdir1 - | | +-- file1.txt - | | +-- file2.txt - | +-- subdir2 - | +-- file1.txt - | +-- file2.txt - +-- file.txt + Usage: + >>> print_tree(".") + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- dependency_file.py + +-- index.html + +-- make_sk_kernel.py + +-- token_counter.py + +-- embedding.py + +-- repair_llm_raw_output.py + +-- mermaid.py + +-- parse_html.py + +-- visual_graph_repo.py + +-- special_tokens.py + +-- ahttp_client.py + +-- __init__.py + +-- mmdc_ink.py + +-- di_graph_repository.py + +-- yaml_model.py + +-- cost_manager.py + +-- __pycache__ + | +-- __init__.cpython-39.pyc + | +-- redis.cpython-39.pyc + | +-- singleton.cpython-39.pyc + | +-- mmdc_ink.cpython-39.pyc + | +-- read_document.cpython-39.pyc + | +-- mermaid.cpython-39.pyc + | +-- parse_html.cpython-39.pyc + | +-- human_interaction.cpython-39.pyc + | +-- cost_manager.cpython-39.pyc + | +-- json_to_markdown.cpython-39.pyc + | +-- graph_repository.cpython-39.pyc + | +-- ahttp_client.cpython-39.pyc + | +-- visual_graph_repo.cpython-39.pyc + | +-- file.cpython-39.pyc + | +-- di_graph_repository.cpython-39.pyc + | +-- pycst.cpython-39.pyc + | +-- save_code.cpython-39.pyc + | +-- dependency_file.cpython-39.pyc + | +-- text.cpython-39.pyc + | +-- token_counter.cpython-39.pyc + | +-- project_repo.cpython-39.pyc + | +-- yaml_model.cpython-39.pyc + | +-- serialize.cpython-39.pyc + | +-- git_repository.cpython-39.pyc + | +-- custom_decoder.cpython-39.pyc + | +-- parse_docstring.cpython-39.pyc + | +-- common.cpython-39.pyc + | +-- exceptions.cpython-39.pyc + | +-- repair_llm_raw_output.cpython-39.pyc + | +-- s3.cpython-39.pyc + | +-- embedding.cpython-39.pyc + | +-- make_sk_kernel.cpython-39.pyc + | +-- file_repository.cpython-39.pyc + +-- file.py + +-- save_code.py + +-- common.py + +-- redis.py + +-- text.py + +-- graph_repository.py + +-- singleton.py + +-- recovery_util.py + +-- file_repository.py + +-- pycst.py + +-- exceptions.py + +-- human_interaction.py + +-- highlight.py + +-- mmdc_pyppeteer.py + +-- s3.py + +-- json_to_markdown.py + +-- custom_decoder.py + +-- git_repository.py + +-- read_document.py + +-- parse_docstring.py """ from __future__ import annotations from pathlib import Path from typing import Callable, Dict, List -from anthropic import BaseModel -from pydantic import Field - -class Tree(BaseModel): +def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: """ - Represents a directory tree structure. + Recursively traverses the directory structure and prints it out in a tree-like format. - Attributes: - root (str): The root directory of the tree. - tree (Dict[str, Dict]): The tree structure as a dictionary. + Args: + root (str or Path): The root directory from which to start traversing. + git_ignore_rules (Callable): Optional. A function to filter files to ignore. - Methods: - print: Print the directory tree structure. + Returns: + str: A string representation of the directory tree. + + Example: + >>> tree(".") + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- __pycache__ + | +-- __init__.cpython-39.pyc + | +-- redis.cpython-39.pyc + | +-- singleton.cpython-39.pyc + +-- parse_docstring.py + + >>> from gitignore_parser import parse_gitignore + >>> tree(".", git_ignore_rules=parse_gitignore(full_path="../../.gitignore")) + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- parse_docstring.py """ + root = Path(root).resolve() + dir_ = {root.name: _list_children(root=root, git_ignore_rules=git_ignore_rules)} + v = _print_tree(dir_) + return "\n".join(v) - root: str - tree: Dict[str, Dict] = Field(default_factory=dict) - def print(self, git_ignore_rules: Callable = None) -> str: - """ - Recursively traverses the directory structure and prints it out in a tree-like format. - - Args: - git_ignore_rules (Callable): Optional. A function to filter files to ignore. - - Returns: - str: A string representation of the directory tree. - - """ - root = Path(self.root).resolve() - self.tree[root.name] = self._list_children(root=root, git_ignore_rules=git_ignore_rules) - v = self._print_tree(self.tree) - return "\n".join(v) - - @staticmethod - def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: - tree = {} - for i in root.iterdir(): - if git_ignore_rules and git_ignore_rules(str(i)): - continue +def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: + dir_ = {} + for i in root.iterdir(): + if git_ignore_rules and git_ignore_rules(str(i)): + continue + try: if i.is_file(): - tree[i.name] = {} + dir_[i.name] = {} else: - tree[i.name] = Tree._list_children(root=i, git_ignore_rules=git_ignore_rules) - return tree + dir_[i.name] = _list_children(root=i, git_ignore_rules=git_ignore_rules) + except (FileNotFoundError, PermissionError, OSError): + dir_[i.name] = {} + return dir_ - @staticmethod - def _print_tree(tree: Dict[str:Dict], indent: int = 0) -> List[str]: - ret = [] - for name, children in tree.items(): - ret.append(name) - if not children: - continue - lines = Tree._print_tree(tree=children, indent=indent + 1) - for j, v in enumerate(lines): - if v[0] not in ["+", " ", "|"]: - ret = Tree._add_line(ret) - row = f"+-- {v}" - else: - row = f" {v}" - ret.append(row) - return ret - @staticmethod - def _add_line(rows: List[str]) -> List[str]: - for i in range(len(rows) - 1, -1, -1): - v = rows[i] - if v[0] != " ": - return rows - rows[i] = "|" + v[1:] - return rows +def _print_tree(dir_: Dict[str:Dict]) -> List[str]: + ret = [] + for name, children in dir_.items(): + ret.append(name) + if not children: + continue + lines = _print_tree(children) + for j, v in enumerate(lines): + if v[0] not in ["+", " ", "|"]: + ret = _add_line(ret) + row = f"+-- {v}" + else: + row = f" {v}" + ret.append(row) + return ret + + +def _add_line(rows: List[str]) -> List[str]: + for i in range(len(rows) - 1, -1, -1): + v = rows[i] + if v[0] != " ": + return rows + rows[i] = "|" + v[1:] + return rows diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py index 0d48f7ce3..34eae10cf 100644 --- a/tests/metagpt/utils/test_tree.py +++ b/tests/metagpt/utils/test_tree.py @@ -4,7 +4,7 @@ from typing import List import pytest from gitignore_parser import parse_gitignore -from metagpt.utils.tree import Tree +from metagpt.utils.tree import _print_tree, tree @pytest.mark.parametrize( @@ -16,8 +16,8 @@ from metagpt.utils.tree import Tree ) def test_tree(root: str, rules: str): gitignore_rules = parse_gitignore(full_path=rules) if rules else None - tree = Tree(root=root).print(git_ignore_rules=gitignore_rules) - assert tree + v = tree(root=root, git_ignore_rules=gitignore_rules) + assert v @pytest.mark.parametrize( @@ -46,7 +46,7 @@ def test_tree(root: str, rules: str): ], ) def test__print_tree(tree: dict, want: List[str]): - v = Tree._print_tree(tree) + v = _print_tree(tree) assert v == want From 6a8699cd4a8f0f2f6f3a8095c67bc72abe775b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 12 Mar 2024 11:14:35 +0800 Subject: [PATCH 09/21] refactor: Replace Tree class with tree() --- metagpt/utils/tree.py | 61 +------------------------------------------ 1 file changed, 1 insertion(+), 60 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index ad3373f5f..1c0060842 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -5,61 +5,18 @@ @Author : mashenquan @File : tree.py @Desc : Implement the same functionality as the `tree` command. -Example: - Usage: + Example: >>> print_tree(".") utils +-- serialize.py +-- project_repo.py +-- tree.py +-- mmdc_playwright.py - +-- dependency_file.py - +-- index.html - +-- make_sk_kernel.py - +-- token_counter.py - +-- embedding.py - +-- repair_llm_raw_output.py - +-- mermaid.py - +-- parse_html.py - +-- visual_graph_repo.py - +-- special_tokens.py - +-- ahttp_client.py - +-- __init__.py - +-- mmdc_ink.py - +-- di_graph_repository.py - +-- yaml_model.py +-- cost_manager.py +-- __pycache__ | +-- __init__.cpython-39.pyc | +-- redis.cpython-39.pyc | +-- singleton.cpython-39.pyc - | +-- mmdc_ink.cpython-39.pyc - | +-- read_document.cpython-39.pyc - | +-- mermaid.cpython-39.pyc - | +-- parse_html.cpython-39.pyc - | +-- human_interaction.cpython-39.pyc - | +-- cost_manager.cpython-39.pyc - | +-- json_to_markdown.cpython-39.pyc - | +-- graph_repository.cpython-39.pyc - | +-- ahttp_client.cpython-39.pyc - | +-- visual_graph_repo.cpython-39.pyc - | +-- file.cpython-39.pyc - | +-- di_graph_repository.cpython-39.pyc - | +-- pycst.cpython-39.pyc - | +-- save_code.cpython-39.pyc - | +-- dependency_file.cpython-39.pyc - | +-- text.cpython-39.pyc - | +-- token_counter.cpython-39.pyc - | +-- project_repo.cpython-39.pyc - | +-- yaml_model.cpython-39.pyc - | +-- serialize.cpython-39.pyc - | +-- git_repository.cpython-39.pyc - | +-- custom_decoder.cpython-39.pyc - | +-- parse_docstring.cpython-39.pyc - | +-- common.cpython-39.pyc - | +-- exceptions.cpython-39.pyc - | +-- repair_llm_raw_output.cpython-39.pyc - | +-- s3.cpython-39.pyc | +-- embedding.cpython-39.pyc | +-- make_sk_kernel.cpython-39.pyc | +-- file_repository.cpython-39.pyc @@ -67,22 +24,6 @@ Example: +-- save_code.py +-- common.py +-- redis.py - +-- text.py - +-- graph_repository.py - +-- singleton.py - +-- recovery_util.py - +-- file_repository.py - +-- pycst.py - +-- exceptions.py - +-- human_interaction.py - +-- highlight.py - +-- mmdc_pyppeteer.py - +-- s3.py - +-- json_to_markdown.py - +-- custom_decoder.py - +-- git_repository.py - +-- read_document.py - +-- parse_docstring.py """ from __future__ import annotations From 684730e94f7c5fe7f5bac8a31ed8fac1937b6d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 13 Mar 2024 15:06:40 +0800 Subject: [PATCH 10/21] feat: +`tree` command --- metagpt/utils/tree.py | 36 ++++++++++++++++++++++++++++---- tests/metagpt/utils/test_tree.py | 16 +++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index 1c0060842..c0386d822 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -27,17 +27,22 @@ """ from __future__ import annotations +import subprocess from pathlib import Path from typing import Callable, Dict, List +from gitignore_parser import parse_gitignore -def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: + +def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = True) -> str: """ Recursively traverses the directory structure and prints it out in a tree-like format. Args: root (str or Path): The root directory from which to start traversing. - git_ignore_rules (Callable): Optional. A function to filter files to ignore. + gitignore (str or Path): The filename of gitignore file. + run_command (bool): Whether to execute `tree` command. Execute the `tree` command and return the result if True, + otherwise execute python code instead. Returns: str: A string representation of the directory tree. @@ -55,8 +60,7 @@ def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: | +-- singleton.cpython-39.pyc +-- parse_docstring.py - >>> from gitignore_parser import parse_gitignore - >>> tree(".", git_ignore_rules=parse_gitignore(full_path="../../.gitignore")) + >>> tree(".", gitignore="../../.gitignore") utils +-- serialize.py +-- project_repo.py @@ -64,8 +68,21 @@ def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: +-- mmdc_playwright.py +-- parse_docstring.py + >>> tree(".", gitignore="../../.gitignore", run_command=True) + utils + ├── serialize.py + ├── project_repo.py + ├── tree.py + ├── mmdc_playwright.py + └── parse_docstring.py + + """ root = Path(root).resolve() + if run_command: + return _execute_tree(root, gitignore) + + git_ignore_rules = parse_gitignore(gitignore) if gitignore else None dir_ = {root.name: _list_children(root=root, git_ignore_rules=git_ignore_rules)} v = _print_tree(dir_) return "\n".join(v) @@ -110,3 +127,14 @@ def _add_line(rows: List[str]) -> List[str]: return rows rows[i] = "|" + v[1:] return rows + + +def _execute_tree(root: Path, gitignore: str | Path) -> str: + args = ["--gitignore", str(gitignore)] if gitignore else [] + try: + result = subprocess.run(["tree"] + args + [str(root)], capture_output=True, text=True, check=True) + if result.returncode != 0: + raise ValueError(f"tree exits with code {result.returncode}") + return result.stdout + except subprocess.CalledProcessError as e: + raise e diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py index 34eae10cf..03a2a5606 100644 --- a/tests/metagpt/utils/test_tree.py +++ b/tests/metagpt/utils/test_tree.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import List import pytest -from gitignore_parser import parse_gitignore from metagpt.utils.tree import _print_tree, tree @@ -15,8 +14,19 @@ from metagpt.utils.tree import _print_tree, tree ], ) def test_tree(root: str, rules: str): - gitignore_rules = parse_gitignore(full_path=rules) if rules else None - v = tree(root=root, git_ignore_rules=gitignore_rules) + v = tree(root=root, gitignore=rules) + assert v + + +@pytest.mark.parametrize( + ("root", "rules"), + [ + (str(Path(__file__).parent / "../.."), None), + (str(Path(__file__).parent / "../.."), str(Path(__file__).parent / "../../../.gitignore")), + ], +) +def test_tree_command(root: str, rules: str): + v = tree(root=root, gitignore=rules, run_command=True) assert v From 6f4d30825f84d90c35dab04e639fc1b483e1a823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 13 Mar 2024 15:10:13 +0800 Subject: [PATCH 11/21] feat: +`tree` command --- metagpt/utils/tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index c0386d822..fbf085e48 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -34,7 +34,7 @@ from typing import Callable, Dict, List from gitignore_parser import parse_gitignore -def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = True) -> str: +def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = False) -> str: """ Recursively traverses the directory structure and prints it out in a tree-like format. From f6a11d508904e6a56a9e35895abfeb439f5c4110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 20 Mar 2024 17:34:30 +0800 Subject: [PATCH 12/21] fixbug: #1016 --- metagpt/actions/di/write_analysis_code.py | 4 +-- metagpt/provider/base_llm.py | 22 +++++++++++++++++ metagpt/provider/google_gemini_api.py | 30 +++++++++++++++++++++++ metagpt/provider/openai_api.py | 9 ++----- metagpt/utils/common.py | 23 ----------------- tests/mock/mock_llm.py | 3 +-- 6 files changed, 57 insertions(+), 34 deletions(-) diff --git a/metagpt/actions/di/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py index 185926e31..711e56d39 100644 --- a/metagpt/actions/di/write_analysis_code.py +++ b/metagpt/actions/di/write_analysis_code.py @@ -18,7 +18,7 @@ from metagpt.prompts.di.write_analysis_code import ( STRUCTUAL_PROMPT, ) from metagpt.schema import Message, Plan -from metagpt.utils.common import CodeParser, process_message, remove_comments +from metagpt.utils.common import CodeParser, remove_comments class WriteAnalysisCode(Action): @@ -50,7 +50,7 @@ class WriteAnalysisCode(Action): ) working_memory = working_memory or [] - context = process_message([Message(content=structual_prompt, role="user")] + working_memory) + context = self.llm.format_msg([Message(content=structual_prompt, role="user")] + working_memory) # LLM call if use_reflection: diff --git a/metagpt/provider/base_llm.py b/metagpt/provider/base_llm.py index 71308930a..601980d5e 100644 --- a/metagpt/provider/base_llm.py +++ b/metagpt/provider/base_llm.py @@ -73,6 +73,28 @@ class BaseLLM(ABC): def _system_msg(self, msg: str) -> dict[str, str]: return {"role": "system", "content": msg} + def format_msg(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: + """convert messages to list[dict].""" + from metagpt.schema import Message + + if not isinstance(messages, list): + messages = [messages] + + processed_messages = [] + for msg in messages: + if isinstance(msg, str): + processed_messages.append({"role": "user", "content": msg}) + elif isinstance(msg, dict): + assert set(msg.keys()) == set(["role", "content"]) + processed_messages.append(msg) + elif isinstance(msg, Message): + processed_messages.append(msg.to_dict()) + else: + raise ValueError( + f"Only support message type are: str, Message, dict, but got {type(messages).__name__}!" + ) + return processed_messages + def _system_msgs(self, msgs: list[str]) -> list[dict[str, str]]: return [self._system_msg(msg) for msg in msgs] diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 09e554205..7370747a5 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -18,6 +18,7 @@ from metagpt.configs.llm_config import LLMConfig, LLMType from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider +from metagpt.schema import Message class GeminiGenerativeModel(GenerativeModel): @@ -61,6 +62,35 @@ class GeminiLLM(BaseLLM): def _assistant_msg(self, msg: str) -> dict[str, str]: return {"role": "model", "parts": [msg]} + def _system_msg(self, msg: str) -> dict[str, str]: + return {"role": "user", "parts": [msg]} + + def format_msg(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: + """convert messages to list[dict].""" + from metagpt.schema import Message + + if not isinstance(messages, list): + messages = [messages] + + # REF: https://ai.google.dev/tutorials/python_quickstart + # As a dictionary, the message requires `role` and `parts` keys. + # The role in a conversation can either be the `user`, which provides the prompts, + # or `model`, which provides the responses. + processed_messages = [] + for msg in messages: + if isinstance(msg, str): + processed_messages.append({"role": "user", "parts": [msg]}) + elif isinstance(msg, dict): + assert set(msg.keys()) == set(["role", "parts"]) + processed_messages.append(msg) + elif isinstance(msg, Message): + processed_messages.append({"role": "user" if msg.role == "user" else "model", "parts": [msg.content]}) + else: + raise ValueError( + f"Only support message type are: str, Message, dict, but got {type(messages).__name__}!" + ) + return processed_messages + def _const_kwargs(self, messages: list[dict], stream: bool = False) -> dict: kwargs = {"contents": messages, "generation_config": GenerationConfig(temperature=0.3), "stream": stream} return kwargs diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index b4f99e69f..2fb64dc85 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -29,12 +29,7 @@ from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_llm import BaseLLM from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA from metagpt.provider.llm_provider_registry import register_provider -from metagpt.utils.common import ( - CodeParser, - decode_image, - log_and_reraise, - process_message, -) +from metagpt.utils.common import CodeParser, decode_image, log_and_reraise from metagpt.utils.cost_manager import CostManager from metagpt.utils.exceptions import handle_exception from metagpt.utils.token_counter import ( @@ -150,7 +145,7 @@ class OpenAILLM(BaseLLM): async def _achat_completion_function( self, messages: list[dict], timeout: int = 3, **chat_configs ) -> ChatCompletion: - messages = process_message(messages) + messages = self.format_msg(messages) kwargs = self._cons_kwargs(messages=messages, timeout=timeout, **chat_configs) rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs) self._update_costs(rsp.usage) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index e9cef69a4..7493712c2 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -802,29 +802,6 @@ def decode_image(img_url_or_b64: str) -> Image: return img -def process_message(messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: - """convert messages to list[dict].""" - from metagpt.schema import Message - - # 全部转成list - if not isinstance(messages, list): - messages = [messages] - - # 转成list[dict] - processed_messages = [] - for msg in messages: - if isinstance(msg, str): - processed_messages.append({"role": "user", "content": msg}) - elif isinstance(msg, dict): - assert set(msg.keys()) == set(["role", "content"]) - processed_messages.append(msg) - elif isinstance(msg, Message): - processed_messages.append(msg.to_dict()) - else: - raise ValueError(f"Only support message type are: str, Message, dict, but got {type(messages).__name__}!") - return processed_messages - - def log_and_reraise(retry_state: RetryCallState): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") logger.warning( diff --git a/tests/mock/mock_llm.py b/tests/mock/mock_llm.py index b4cdfa0cf..c4262e080 100644 --- a/tests/mock/mock_llm.py +++ b/tests/mock/mock_llm.py @@ -8,7 +8,6 @@ from metagpt.provider.azure_openai_api import AzureOpenAILLM from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA from metagpt.provider.openai_api import OpenAILLM from metagpt.schema import Message -from metagpt.utils.common import process_message OriginalLLM = OpenAILLM if config.llm.api_type == LLMType.OPENAI else AzureOpenAILLM @@ -105,7 +104,7 @@ class MockLLM(OriginalLLM): return rsp async def aask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: - msg_key = json.dumps(process_message(messages), ensure_ascii=False) + msg_key = json.dumps(self.format_msg(messages), ensure_ascii=False) rsp = await self._mock_rsp(msg_key, self.original_aask_code, messages, **kwargs) return rsp From 067a39e7b923e4d9394ca8547f04cc1e24cd079d Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 21 Mar 2024 10:45:53 +0800 Subject: [PATCH 13/21] remove incorrect file header --- examples/di/arxiv_reader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/di/arxiv_reader.py b/examples/di/arxiv_reader.py index fab0e2e48..7f2c6e331 100644 --- a/examples/di/arxiv_reader.py +++ b/examples/di/arxiv_reader.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- """ @Time : 2024/01/15 -@Author : mannaandpoem -@File : imitate_webpage.py """ from metagpt.roles.di.data_interpreter import DataInterpreter From dd348d07071c9d59ec243058922d3d37c90eb6c3 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 21 Mar 2024 10:45:58 +0800 Subject: [PATCH 14/21] remove incorrect file header --- examples/di/arxiv_reader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/di/arxiv_reader.py b/examples/di/arxiv_reader.py index 7f2c6e331..6e1939b81 100644 --- a/examples/di/arxiv_reader.py +++ b/examples/di/arxiv_reader.py @@ -1,8 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" -@Time : 2024/01/15 -""" from metagpt.roles.di.data_interpreter import DataInterpreter From fcb2ea48638d1890faf277b87acc2e63e59521c5 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 21 Mar 2024 13:57:32 +0800 Subject: [PATCH 15/21] add params to search engine --- examples/search_with_specific_engine.py | 2 +- metagpt/configs/search_config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index 1eee762d5..48bcc67b8 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -13,7 +13,7 @@ async def main(): question = "What are the most interesting human facts?" search = Config.default().search - kwargs = {"api_key": search.api_key, "cse_id": search.cse_id, "proxy": None} + kwargs = {"api_key": search.api_key, "cse_id": search.cse_id, "proxy": None, "params": search.params} await Searcher(search_engine=SearchEngine(engine=search.api_type, **kwargs)).run(question) diff --git a/metagpt/configs/search_config.py b/metagpt/configs/search_config.py index af928b02a..38af4d91f 100644 --- a/metagpt/configs/search_config.py +++ b/metagpt/configs/search_config.py @@ -18,3 +18,4 @@ class SearchConfig(YamlModel): api_key: str = "" cse_id: str = "" # for google search_func: Optional[Callable] = None + params: dict = {} From 8486e520485359410a7560b23ee39895fdae6de2 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 21 Mar 2024 14:52:17 +0800 Subject: [PATCH 16/21] add params to search engine --- metagpt/actions/action_node.py | 2 +- metagpt/configs/search_config.py | 11 ++++++++++- tests/metagpt/tools/test_search_engine.py | 4 ---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 09da4a988..59559ad04 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -330,7 +330,7 @@ class ActionNode: def compile_to(self, i: Dict, schema, kv_sep) -> str: if schema == "json": - return json.dumps(i, indent=4) + return json.dumps(i, indent=4, ensure_ascii=False) elif schema == "markdown": return dict_to_markdown(i, kv_sep=kv_sep) else: diff --git a/metagpt/configs/search_config.py b/metagpt/configs/search_config.py index 38af4d91f..e28b14c99 100644 --- a/metagpt/configs/search_config.py +++ b/metagpt/configs/search_config.py @@ -7,6 +7,8 @@ """ from typing import Callable, Optional +from pydantic import Field + from metagpt.tools import SearchEngineType from metagpt.utils.yaml_model import YamlModel @@ -18,4 +20,11 @@ class SearchConfig(YamlModel): api_key: str = "" cse_id: str = "" # for google search_func: Optional[Callable] = None - params: dict = {} + params: dict = Field( + default_factory=lambda: { + "engine": "google", + "google_domain": "google.com", + "gl": "us", + "hl": "en", + } + ) diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index a1f03ef7b..964ead02f 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -11,7 +11,6 @@ from typing import Callable import pytest -from metagpt.config2 import config from metagpt.configs.search_config import SearchConfig from metagpt.logs import logger from metagpt.tools import SearchEngineType @@ -53,14 +52,11 @@ async def test_search_engine( search_engine_config = {"engine": search_engine_type, "run_func": run_func} if search_engine_type is SearchEngineType.SERPAPI_GOOGLE: - assert config.search search_engine_config["api_key"] = "mock-serpapi-key" elif search_engine_type is SearchEngineType.DIRECT_GOOGLE: - assert config.search search_engine_config["api_key"] = "mock-google-key" search_engine_config["cse_id"] = "mock-google-cse" elif search_engine_type is SearchEngineType.SERPER_GOOGLE: - assert config.search search_engine_config["api_key"] = "mock-serper-key" async def test(search_engine): From 8cda8c6f1b0f35e90ecca9467a3ee346b441860b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 14:54:14 +0800 Subject: [PATCH 17/21] fixbug: reset ocr_receipt.py --- examples/di/ocr_receipt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/di/ocr_receipt.py b/examples/di/ocr_receipt.py index 8b48be4f1..6b969ee99 100644 --- a/examples/di/ocr_receipt.py +++ b/examples/di/ocr_receipt.py @@ -6,7 +6,9 @@ async def main(): image_path = "image.jpg" language = "English" requirement = f"""This is a {language} receipt image. - Your goal is to perform OCR on images using PaddleOCR, then extract the total amount from ocr text results, and finally save as table. Image path: {image_path}. + Your goal is to perform OCR on images using PaddleOCR, output text content from the OCR results and discard + coordinates and confidence levels, then recognize the total amount from ocr text content, and finally save as table. + Image path: {image_path}. NOTE: The environments for Paddle and PaddleOCR are all ready and has been fully installed.""" di = DataInterpreter() From 587c13489320c62c8958f57bea54842e5a993614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 14:55:00 +0800 Subject: [PATCH 18/21] fixbug: reset ocr_receipt.py --- examples/di/ocr_receipt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/di/ocr_receipt.py b/examples/di/ocr_receipt.py index 6b969ee99..af54d519b 100644 --- a/examples/di/ocr_receipt.py +++ b/examples/di/ocr_receipt.py @@ -8,7 +8,7 @@ async def main(): requirement = f"""This is a {language} receipt image. Your goal is to perform OCR on images using PaddleOCR, output text content from the OCR results and discard coordinates and confidence levels, then recognize the total amount from ocr text content, and finally save as table. - Image path: {image_path}. + Image path: {image_path}. NOTE: The environments for Paddle and PaddleOCR are all ready and has been fully installed.""" di = DataInterpreter() From f051104bce125539f4ea24598b4b35a9e2c0ae69 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 21 Mar 2024 14:55:42 +0800 Subject: [PATCH 19/21] simplify code --- examples/search_with_specific_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index 48bcc67b8..276431ed8 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -13,7 +13,7 @@ async def main(): question = "What are the most interesting human facts?" search = Config.default().search - kwargs = {"api_key": search.api_key, "cse_id": search.cse_id, "proxy": None, "params": search.params} + kwargs = search.model_dump() await Searcher(search_engine=SearchEngine(engine=search.api_type, **kwargs)).run(question) From 9cfcfb1ea8bae086ed4fc5fe9d8365038c95e89e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 15:14:14 +0800 Subject: [PATCH 20/21] feat: use --gitfile --- .gitignore | 2 +- metagpt/utils/tree.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6bc67fa61..1542bbb98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ ### Python template # Byte-compiled / optimized / DLL files -__pycache__/ +__pycache__ *.py[cod] *$py.class diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index fbf085e48..bd7922290 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -130,7 +130,7 @@ def _add_line(rows: List[str]) -> List[str]: def _execute_tree(root: Path, gitignore: str | Path) -> str: - args = ["--gitignore", str(gitignore)] if gitignore else [] + args = ["--gitfile", str(gitignore)] if gitignore else [] try: result = subprocess.run(["tree"] + args + [str(root)], capture_output=True, text=True, check=True) if result.returncode != 0: From 12551ab60cc9ee0e59cf31a1a29c24e13d9e7abd Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 21 Mar 2024 21:57:59 +0800 Subject: [PATCH 21/21] fix state value extract for https://github.com/geekan/MetaGPT/issues/1067 --- metagpt/utils/repair_llm_raw_output.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index b8756e8c6..17e095c5f 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -340,7 +340,9 @@ def extract_state_value_from_output(content: str) -> str: content (str): llm's output from `Role._think` """ content = content.strip() # deal the output cases like " 0", "0\n" and so on. - pattern = r"([0-9])" # TODO find the number using a more proper method not just extract from content using pattern + pattern = ( + r"(? 0 else "-1"