From 791ab749adab3d527c6a700bfd89d855a2ba347a Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 21 Feb 2024 16:49:59 +0800 Subject: [PATCH 01/21] refine code to avoid config error --- metagpt/software_company.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 26bb29cd1..f290d497a 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -2,14 +2,11 @@ # -*- coding: utf-8 -*- import asyncio -import shutil from pathlib import Path import typer -from metagpt.config2 import config -from metagpt.const import CONFIG_ROOT, METAGPT_ROOT -from metagpt.context import Context +from metagpt.const import CONFIG_ROOT from metagpt.utils.project_repo import ProjectRepo app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False) @@ -30,6 +27,8 @@ def generate_repo( recover_path=None, ) -> ProjectRepo: """Run the startup logic. Can be called from CLI or other Python scripts.""" + from metagpt.config2 import config + from metagpt.context import Context from metagpt.roles import ( Architect, Engineer, @@ -122,7 +121,17 @@ def startup( ) -def copy_config_to(config_path=METAGPT_ROOT / "config" / "config2.yaml"): +DEFAULT_CONFIG = """# Full Example: https://github.com/geekan/MetaGPT/blob/main/config/config2.example.yaml +# Reflected Code: https://github.com/geekan/MetaGPT/blob/main/metagpt/config2.py +llm: + api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options + model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview + base_url: "https://api.openai.com/v1" # or forward url / other llm url + api_key: "YOUR_API_KEY" +""" + + +def copy_config_to(): """Initialize the configuration file for MetaGPT.""" target_path = CONFIG_ROOT / "config2.yaml" @@ -136,7 +145,7 @@ def copy_config_to(config_path=METAGPT_ROOT / "config" / "config2.yaml"): print(f"Existing configuration file backed up at {backup_path}") # 复制文件 - shutil.copy(str(config_path), target_path) + target_path.write_text(DEFAULT_CONFIG, encoding="utf-8") print(f"Configuration file initialized at {target_path}") From 0a6dc8f7e7d888f7556a0c009c422d61933c0406 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Fri, 23 Feb 2024 10:15:49 +0800 Subject: [PATCH 02/21] update version to 0.7.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ed3d5f78f..f5d880ac9 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.2", + version="0.7.3", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", From 08bf756ae8cc81bfa483d5fcfbb9a45b49decca6 Mon Sep 17 00:00:00 2001 From: yzlin Date: Fri, 1 Mar 2024 18:33:49 +0800 Subject: [PATCH 03/21] rename interpreter --- README.md | 4 +++- examples/di/README.md | 18 ++++++++++++++++++ examples/{mi => di}/crawl_webpage.py | 6 +++--- examples/{mi => di}/data_visualization.py | 6 +++--- examples/{mi => di}/email_summary.py | 6 +++--- examples/{mi => di}/imitate_webpage.py | 6 +++--- examples/{mi => di}/machine_learning.py | 6 +++--- examples/{mi => di}/ml_engineer_with_tools.py | 2 +- examples/{mi => di}/ocr_receipt.py | 6 +++--- examples/{mi => di}/rm_image_background.py | 6 +++--- examples/{mi => di}/sd_tool_usage.py | 6 +++--- examples/{mi => di}/solve_math_problems.py | 6 +++--- examples/mi/README.md | 18 ------------------ metagpt/actions/__init__.py | 6 +++--- metagpt/actions/{mi => di}/__init__.py | 0 metagpt/actions/{mi => di}/ask_review.py | 0 metagpt/actions/{mi => di}/debug_code.py | 2 +- metagpt/actions/{mi => di}/execute_nb_code.py | 0 metagpt/actions/{mi => di}/ml_action.py | 6 +++--- .../actions/{mi => di}/write_analysis_code.py | 2 +- metagpt/actions/{mi => di}/write_plan.py | 2 +- metagpt/prompts/{mi => di}/__init__.py | 0 metagpt/prompts/{mi => di}/ml_action.py | 0 .../prompts/{mi => di}/write_analysis_code.py | 0 metagpt/roles/{mi => di}/__init__.py | 0 .../interpreter.py => di/data_interpreter.py} | 12 ++++++------ metagpt/roles/{mi => di}/ml_engineer.py | 10 +++++----- metagpt/strategy/planner.py | 4 ++-- metagpt/strategy/solver.py | 4 ++-- setup.py | 2 +- .../actions/{mi => di}/test_ask_review.py | 2 +- .../actions/{mi => di}/test_debug_code.py | 2 +- .../actions/{mi => di}/test_execute_nb_code.py | 2 +- .../actions/{mi => di}/test_ml_action.py | 2 +- .../{mi => di}/test_write_analysis_code.py | 4 ++-- .../actions/{mi => di}/test_write_plan.py | 2 +- .../test_data_interpreter.py} | 10 +++++----- .../roles/{mi => di}/test_ml_engineer.py | 6 +++--- tests/metagpt/utils/test_save_code.py | 2 +- 39 files changed, 90 insertions(+), 88 deletions(-) create mode 100644 examples/di/README.md rename examples/{mi => di}/crawl_webpage.py (78%) rename examples/{mi => di}/data_visualization.py (59%) rename examples/{mi => di}/email_summary.py (90%) rename examples/{mi => di}/imitate_webpage.py (81%) rename examples/{mi => di}/machine_learning.py (67%) rename examples/{mi => di}/ml_engineer_with_tools.py (94%) rename examples/{mi => di}/ocr_receipt.py (81%) rename examples/{mi => di}/rm_image_background.py (75%) rename examples/{mi => di}/sd_tool_usage.py (73%) rename examples/{mi => di}/solve_math_problems.py (70%) delete mode 100644 examples/mi/README.md rename metagpt/actions/{mi => di}/__init__.py (100%) rename metagpt/actions/{mi => di}/ask_review.py (100%) rename metagpt/actions/{mi => di}/debug_code.py (98%) rename metagpt/actions/{mi => di}/execute_nb_code.py (100%) rename metagpt/actions/{mi => di}/ml_action.py (93%) rename metagpt/actions/{mi => di}/write_analysis_code.py (99%) rename metagpt/actions/{mi => di}/write_plan.py (98%) rename metagpt/prompts/{mi => di}/__init__.py (100%) rename metagpt/prompts/{mi => di}/ml_action.py (100%) rename metagpt/prompts/{mi => di}/write_analysis_code.py (100%) rename metagpt/roles/{mi => di}/__init__.py (100%) rename metagpt/roles/{mi/interpreter.py => di/data_interpreter.py} (91%) rename metagpt/roles/{mi => di}/ml_engineer.py (89%) rename tests/metagpt/actions/{mi => di}/test_ask_review.py (84%) rename tests/metagpt/actions/{mi => di}/test_debug_code.py (96%) rename tests/metagpt/actions/{mi => di}/test_execute_nb_code.py (98%) rename tests/metagpt/actions/{mi => di}/test_ml_action.py (95%) rename tests/metagpt/actions/{mi => di}/test_write_analysis_code.py (99%) rename tests/metagpt/actions/{mi => di}/test_write_plan.py (95%) rename tests/metagpt/roles/{mi/test_interpreter.py => di/test_data_interpreter.py} (65%) rename tests/metagpt/roles/{mi => di}/test_ml_engineer.py (94%) diff --git a/README.md b/README.md index a1aa5ded8..197c387fd 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,9 @@ # MetaGPT: The Multi-Agent Framework

## News -🚀 Feb. 08, 2024: [v0.7.0](https://github.com/geekan/MetaGPT/releases/tag/v0.7.0) released, supporting assigning different LLMs to different Roles. We also introduced [Interpreter](https://github.com/geekan/MetaGPT/blob/main/examples/mi/README.md), a powerful agent capable of solving a wide range of real-world problems. +🚀 March. 01, 2024: Our Data Interpreter paper is on arxiv. Find all design and benchmark details [here](https://arxiv.org/abs/2402.18679)! + +🚀 Feb. 08, 2024: [v0.7.0](https://github.com/geekan/MetaGPT/releases/tag/v0.7.0) released, supporting assigning different LLMs to different Roles. We also introduced [Data Interpreter](https://github.com/geekan/MetaGPT/blob/main/examples/di/README.md), a powerful agent capable of solving a wide range of real-world problems. 🚀 Jan. 16, 2024: Our paper [MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework ](https://arxiv.org/abs/2308.00352) accepted for oral presentation **(top 1.2%)** at ICLR 2024, **ranking #1** in the LLM-based Agent category. diff --git a/examples/di/README.md b/examples/di/README.md new file mode 100644 index 000000000..fb4e8044b --- /dev/null +++ b/examples/di/README.md @@ -0,0 +1,18 @@ +# Data Interpreter (DI) + +## What is Data Interpreter +Data Interpreter is an agent who solves problems through codes. It understands user requirements, makes plans, writes codes for execution, and uses tools if necessary. These capabilities enable it to tackle a wide range of scenarios, please check out the examples below. + +## Example List +- Data visualization +- Machine learning modeling +- Image background removal +- Solve math problems +- Receipt OCR +- Tool usage: web page imitation +- Tool usage: web crawling +- Tool usage: text2image +- Tool usage: email summarization and response +- More on the way! + +Please see [here](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/intro.html) for detailed explanation. \ No newline at end of file diff --git a/examples/mi/crawl_webpage.py b/examples/di/crawl_webpage.py similarity index 78% rename from examples/mi/crawl_webpage.py rename to examples/di/crawl_webpage.py index b5d2fb3d0..f06b85d9b 100644 --- a/examples/mi/crawl_webpage.py +++ b/examples/di/crawl_webpage.py @@ -5,15 +5,15 @@ @File : crawl_webpage.py """ -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): prompt = """Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/, and save it to a csv file. paper title must include `multiagent` or `large language model`. *notice: print key variables*""" - mi = Interpreter(use_tools=True) + di = DataInterpreter(use_tools=True) - await mi.run(prompt) + await di.run(prompt) if __name__ == "__main__": diff --git a/examples/mi/data_visualization.py b/examples/di/data_visualization.py similarity index 59% rename from examples/mi/data_visualization.py rename to examples/di/data_visualization.py index 2e4acc9b4..9af72dc42 100644 --- a/examples/mi/data_visualization.py +++ b/examples/di/data_visualization.py @@ -1,11 +1,11 @@ import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=False) - await mi.run(requirement) + di = DataInterpreter(use_tools=False) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/email_summary.py b/examples/di/email_summary.py similarity index 90% rename from examples/mi/email_summary.py rename to examples/di/email_summary.py index e1511c5b0..af081fee2 100644 --- a/examples/mi/email_summary.py +++ b/examples/di/email_summary.py @@ -6,7 +6,7 @@ """ import os -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): @@ -22,9 +22,9 @@ async def main(): Firstly, Please help me fetch the latest 5 senders and full letter contents. Then, summarize each of the 5 emails into one sentence (you can do this by yourself, no need to import other models to do this) and output them in a markdown format.""" - mi = Interpreter(use_tools=True) + di = DataInterpreter(use_tools=True) - await mi.run(prompt) + await di.run(prompt) if __name__ == "__main__": diff --git a/examples/mi/imitate_webpage.py b/examples/di/imitate_webpage.py similarity index 81% rename from examples/mi/imitate_webpage.py rename to examples/di/imitate_webpage.py index 0e9ca731d..e2c99b874 100644 --- a/examples/mi/imitate_webpage.py +++ b/examples/di/imitate_webpage.py @@ -5,7 +5,7 @@ @Author : mannaandpoem @File : imitate_webpage.py """ -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): @@ -15,9 +15,9 @@ Firstly, utilize Selenium and WebDriver for rendering. Secondly, convert image to a webpage including HTML, CSS and JS in one go. Finally, save webpage in a text file. Note: All required dependencies and environments have been fully installed and configured.""" - mi = Interpreter(use_tools=True) + di = DataInterpreter(use_tools=True) - await mi.run(prompt) + await di.run(prompt) if __name__ == "__main__": diff --git a/examples/mi/machine_learning.py b/examples/di/machine_learning.py similarity index 67% rename from examples/mi/machine_learning.py rename to examples/di/machine_learning.py index a8ab5051e..a58735831 100644 --- a/examples/mi/machine_learning.py +++ b/examples/di/machine_learning.py @@ -1,12 +1,12 @@ import fire -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(auto_run: bool = True): requirement = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy." - mi = Interpreter(auto_run=auto_run) - await mi.run(requirement) + di = DataInterpreter(auto_run=auto_run) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/ml_engineer_with_tools.py b/examples/di/ml_engineer_with_tools.py similarity index 94% rename from examples/mi/ml_engineer_with_tools.py rename to examples/di/ml_engineer_with_tools.py index 9d0e7f951..6119ad843 100644 --- a/examples/mi/ml_engineer_with_tools.py +++ b/examples/di/ml_engineer_with_tools.py @@ -1,6 +1,6 @@ import asyncio -from metagpt.roles.mi.ml_engineer import MLEngineer +from metagpt.roles.di.ml_engineer import MLEngineer async def main(requirement: str): diff --git a/examples/mi/ocr_receipt.py b/examples/di/ocr_receipt.py similarity index 81% rename from examples/mi/ocr_receipt.py rename to examples/di/ocr_receipt.py index ffa5cff05..8b48be4f1 100644 --- a/examples/mi/ocr_receipt.py +++ b/examples/di/ocr_receipt.py @@ -1,4 +1,4 @@ -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): @@ -8,9 +8,9 @@ async def main(): requirement = f"""This is a {language} receipt image. Your goal is to perform OCR on images using PaddleOCR, then extract the total amount from ocr text results, and finally save as table. Image path: {image_path}. NOTE: The environments for Paddle and PaddleOCR are all ready and has been fully installed.""" - mi = Interpreter() + di = DataInterpreter() - await mi.run(requirement) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/rm_image_background.py b/examples/di/rm_image_background.py similarity index 75% rename from examples/mi/rm_image_background.py rename to examples/di/rm_image_background.py index 57e89b103..b74a79eeb 100644 --- a/examples/mi/rm_image_background.py +++ b/examples/di/rm_image_background.py @@ -1,11 +1,11 @@ import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=False) - await mi.run(requirement) + di = DataInterpreter(use_tools=False) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/sd_tool_usage.py b/examples/di/sd_tool_usage.py similarity index 73% rename from examples/mi/sd_tool_usage.py rename to examples/di/sd_tool_usage.py index f8507a411..69c7df5bd 100644 --- a/examples/mi/sd_tool_usage.py +++ b/examples/di/sd_tool_usage.py @@ -4,12 +4,12 @@ # @Desc : import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=True, goal=requirement) - await mi.run(requirement) + di = DataInterpreter(use_tools=True, goal=requirement) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/solve_math_problems.py b/examples/di/solve_math_problems.py similarity index 70% rename from examples/mi/solve_math_problems.py rename to examples/di/solve_math_problems.py index dce2edb00..35a69c953 100644 --- a/examples/mi/solve_math_problems.py +++ b/examples/di/solve_math_problems.py @@ -1,11 +1,11 @@ import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=False) - await mi.run(requirement) + di = DataInterpreter(use_tools=False) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/README.md b/examples/mi/README.md deleted file mode 100644 index 1734ba388..000000000 --- a/examples/mi/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# MetaGPT Interpreter (MI) - -## What is Interpreter -Interpreter is an agent who solves problems through codes. It understands user requirements, makes plans, writes codes for execution, and uses tools if necessary. These capabilities enable it to tackle a wide range of scenarios, please check out the examples below. - -## Example List -- Data visualization -- Machine learning modeling -- Image background removal -- Solve math problems -- Receipt OCR -- Tool usage: web page imitation -- Tool usage: web crawling -- Tool usage: text2image -- Tool usage: email summarization and response -- More on the way! - -Please see [here](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/mi_intro.html) for detailed explanation. \ No newline at end of file diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index 19a7c10d5..29af1dad1 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -22,9 +22,9 @@ from metagpt.actions.write_code_review import WriteCodeReview from metagpt.actions.write_prd import WritePRD from metagpt.actions.write_prd_review import WritePRDReview from metagpt.actions.write_test import WriteTest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.write_analysis_code import WriteCodeWithoutTools, WriteCodeWithTools -from metagpt.actions.mi.write_plan import WritePlan +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.write_analysis_code import WriteCodeWithoutTools, WriteCodeWithTools +from metagpt.actions.di.write_plan import WritePlan class ActionType(Enum): diff --git a/metagpt/actions/mi/__init__.py b/metagpt/actions/di/__init__.py similarity index 100% rename from metagpt/actions/mi/__init__.py rename to metagpt/actions/di/__init__.py diff --git a/metagpt/actions/mi/ask_review.py b/metagpt/actions/di/ask_review.py similarity index 100% rename from metagpt/actions/mi/ask_review.py rename to metagpt/actions/di/ask_review.py diff --git a/metagpt/actions/mi/debug_code.py b/metagpt/actions/di/debug_code.py similarity index 98% rename from metagpt/actions/mi/debug_code.py rename to metagpt/actions/di/debug_code.py index db3b7a9db..f589436f5 100644 --- a/metagpt/actions/mi/debug_code.py +++ b/metagpt/actions/di/debug_code.py @@ -1,6 +1,6 @@ from __future__ import annotations -from metagpt.actions.mi.write_analysis_code import BaseWriteAnalysisCode +from metagpt.actions.di.write_analysis_code import BaseWriteAnalysisCode from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import create_func_call_config diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/di/execute_nb_code.py similarity index 100% rename from metagpt/actions/mi/execute_nb_code.py rename to metagpt/actions/di/execute_nb_code.py diff --git a/metagpt/actions/mi/ml_action.py b/metagpt/actions/di/ml_action.py similarity index 93% rename from metagpt/actions/mi/ml_action.py rename to metagpt/actions/di/ml_action.py index 60b2fb547..d49b7b67d 100644 --- a/metagpt/actions/mi/ml_action.py +++ b/metagpt/actions/di/ml_action.py @@ -3,14 +3,14 @@ from __future__ import annotations from typing import Tuple from metagpt.actions import Action -from metagpt.actions.mi.write_analysis_code import WriteCodeWithTools -from metagpt.prompts.mi.ml_action import ( +from metagpt.actions.di.write_analysis_code import WriteCodeWithTools +from metagpt.prompts.di.ml_action import ( ML_GENERATE_CODE_PROMPT, ML_TOOL_USAGE_PROMPT, PRINT_DATA_COLUMNS, UPDATE_DATA_COLUMNS, ) -from metagpt.prompts.mi.write_analysis_code import CODE_GENERATOR_WITH_TOOLS +from metagpt.prompts.di.write_analysis_code import CODE_GENERATOR_WITH_TOOLS from metagpt.schema import Message, Plan from metagpt.utils.common import create_func_call_config, remove_comments diff --git a/metagpt/actions/mi/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py similarity index 99% rename from metagpt/actions/mi/write_analysis_code.py rename to metagpt/actions/di/write_analysis_code.py index b3d0632b6..0c4980c2b 100644 --- a/metagpt/actions/mi/write_analysis_code.py +++ b/metagpt/actions/di/write_analysis_code.py @@ -10,7 +10,7 @@ from typing import Tuple from metagpt.actions import Action from metagpt.logs import logger -from metagpt.prompts.mi.write_analysis_code import ( +from metagpt.prompts.di.write_analysis_code import ( CODE_GENERATOR_WITH_TOOLS, SELECT_FUNCTION_TOOLS, TOOL_RECOMMENDATION_PROMPT, diff --git a/metagpt/actions/mi/write_plan.py b/metagpt/actions/di/write_plan.py similarity index 98% rename from metagpt/actions/mi/write_plan.py rename to metagpt/actions/di/write_plan.py index 8067d7b87..518dfb6c6 100644 --- a/metagpt/actions/mi/write_plan.py +++ b/metagpt/actions/di/write_plan.py @@ -12,7 +12,7 @@ from typing import Tuple from metagpt.actions import Action from metagpt.logs import logger -from metagpt.prompts.mi.write_analysis_code import ( +from metagpt.prompts.di.write_analysis_code import ( ASSIGN_TASK_TYPE_CONFIG, ASSIGN_TASK_TYPE_PROMPT, ) diff --git a/metagpt/prompts/mi/__init__.py b/metagpt/prompts/di/__init__.py similarity index 100% rename from metagpt/prompts/mi/__init__.py rename to metagpt/prompts/di/__init__.py diff --git a/metagpt/prompts/mi/ml_action.py b/metagpt/prompts/di/ml_action.py similarity index 100% rename from metagpt/prompts/mi/ml_action.py rename to metagpt/prompts/di/ml_action.py diff --git a/metagpt/prompts/mi/write_analysis_code.py b/metagpt/prompts/di/write_analysis_code.py similarity index 100% rename from metagpt/prompts/mi/write_analysis_code.py rename to metagpt/prompts/di/write_analysis_code.py diff --git a/metagpt/roles/mi/__init__.py b/metagpt/roles/di/__init__.py similarity index 100% rename from metagpt/roles/mi/__init__.py rename to metagpt/roles/di/__init__.py diff --git a/metagpt/roles/mi/interpreter.py b/metagpt/roles/di/data_interpreter.py similarity index 91% rename from metagpt/roles/mi/interpreter.py rename to metagpt/roles/di/data_interpreter.py index fa50098e9..b3a2e789f 100644 --- a/metagpt/roles/mi/interpreter.py +++ b/metagpt/roles/di/data_interpreter.py @@ -2,9 +2,9 @@ from __future__ import annotations from pydantic import Field -from metagpt.actions.mi.ask_review import ReviewConst -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.write_analysis_code import ( +from metagpt.actions.di.ask_review import ReviewConst +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.write_analysis_code import ( WriteCodeWithoutTools, WriteCodeWithTools, ) @@ -13,9 +13,9 @@ from metagpt.roles import Role from metagpt.schema import Message, Task, TaskResult -class Interpreter(Role): - name: str = "Ivy" - profile: str = "Interpreter" +class DataInterpreter(Role): + name: str = "David" + profile: str = "DataInterpreter" auto_run: bool = True use_tools: bool = False execute_code: ExecuteNbCode = Field(default_factory=ExecuteNbCode, exclude=True) diff --git a/metagpt/roles/mi/ml_engineer.py b/metagpt/roles/di/ml_engineer.py similarity index 89% rename from metagpt/roles/mi/ml_engineer.py rename to metagpt/roles/di/ml_engineer.py index 78d605d3e..b33b166cf 100644 --- a/metagpt/roles/mi/ml_engineer.py +++ b/metagpt/roles/di/ml_engineer.py @@ -1,13 +1,13 @@ -from metagpt.actions.mi.debug_code import DebugCode -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.ml_action import UpdateDataColumns, WriteCodeWithToolsML +from metagpt.actions.di.debug_code import DebugCode +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.ml_action import UpdateDataColumns, WriteCodeWithToolsML from metagpt.logs import logger -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter from metagpt.tools.tool_type import ToolType from metagpt.utils.common import any_to_str -class MLEngineer(Interpreter): +class MLEngineer(DataInterpreter): name: str = "Mark" profile: str = "MLEngineer" debug_context: list = [] diff --git a/metagpt/strategy/planner.py b/metagpt/strategy/planner.py index 99d16f78b..44294be00 100644 --- a/metagpt/strategy/planner.py +++ b/metagpt/strategy/planner.py @@ -4,8 +4,8 @@ import json from pydantic import BaseModel, Field -from metagpt.actions.mi.ask_review import AskReview, ReviewConst -from metagpt.actions.mi.write_plan import ( +from metagpt.actions.di.ask_review import AskReview, ReviewConst +from metagpt.actions.di.write_plan import ( WritePlan, precheck_update_plan_from_rsp, update_plan_from_rsp, diff --git a/metagpt/strategy/solver.py b/metagpt/strategy/solver.py index ab16d82bd..e532f736b 100644 --- a/metagpt/strategy/solver.py +++ b/metagpt/strategy/solver.py @@ -49,8 +49,8 @@ class TOTSolver(BaseSolver): raise NotImplementedError -class InterpreterSolver(BaseSolver): - """InterpreterSolver: Write&Run code in the graph""" +class DataInterpreterSolver(BaseSolver): + """DataInterpreterSolver: Write&Run code in the graph""" async def solve(self): raise NotImplementedError diff --git a/setup.py b/setup.py index f5d880ac9..2044e577a 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.3", + version="0.7.4", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/metagpt/actions/mi/test_ask_review.py b/tests/metagpt/actions/di/test_ask_review.py similarity index 84% rename from tests/metagpt/actions/mi/test_ask_review.py rename to tests/metagpt/actions/di/test_ask_review.py index 92e8bd046..6bb1accf5 100644 --- a/tests/metagpt/actions/mi/test_ask_review.py +++ b/tests/metagpt/actions/di/test_ask_review.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.ask_review import AskReview +from metagpt.actions.di.ask_review import AskReview @pytest.mark.asyncio diff --git a/tests/metagpt/actions/mi/test_debug_code.py b/tests/metagpt/actions/di/test_debug_code.py similarity index 96% rename from tests/metagpt/actions/mi/test_debug_code.py rename to tests/metagpt/actions/di/test_debug_code.py index 24cfcef10..67f72ad63 100644 --- a/tests/metagpt/actions/mi/test_debug_code.py +++ b/tests/metagpt/actions/di/test_debug_code.py @@ -5,7 +5,7 @@ import pytest -from metagpt.actions.mi.debug_code import DebugCode +from metagpt.actions.di.debug_code import DebugCode from metagpt.schema import Message ErrorStr = """Tested passed: diff --git a/tests/metagpt/actions/mi/test_execute_nb_code.py b/tests/metagpt/actions/di/test_execute_nb_code.py similarity index 98% rename from tests/metagpt/actions/mi/test_execute_nb_code.py rename to tests/metagpt/actions/di/test_execute_nb_code.py index 59a814054..b491dd212 100644 --- a/tests/metagpt/actions/mi/test_execute_nb_code.py +++ b/tests/metagpt/actions/di/test_execute_nb_code.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode, truncate +from metagpt.actions.di.execute_nb_code import ExecuteNbCode, truncate @pytest.mark.asyncio diff --git a/tests/metagpt/actions/mi/test_ml_action.py b/tests/metagpt/actions/di/test_ml_action.py similarity index 95% rename from tests/metagpt/actions/mi/test_ml_action.py rename to tests/metagpt/actions/di/test_ml_action.py index 27d47b0e3..826a7fcf2 100644 --- a/tests/metagpt/actions/mi/test_ml_action.py +++ b/tests/metagpt/actions/di/test_ml_action.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.ml_action import WriteCodeWithToolsML +from metagpt.actions.di.ml_action import WriteCodeWithToolsML from metagpt.schema import Plan, Task diff --git a/tests/metagpt/actions/mi/test_write_analysis_code.py b/tests/metagpt/actions/di/test_write_analysis_code.py similarity index 99% rename from tests/metagpt/actions/mi/test_write_analysis_code.py rename to tests/metagpt/actions/di/test_write_analysis_code.py index 6c2228222..d6bbae1c6 100644 --- a/tests/metagpt/actions/mi/test_write_analysis_code.py +++ b/tests/metagpt/actions/di/test_write_analysis_code.py @@ -2,8 +2,8 @@ import asyncio import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.write_analysis_code import ( +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.write_analysis_code import ( WriteCodeWithoutTools, WriteCodeWithTools, ) diff --git a/tests/metagpt/actions/mi/test_write_plan.py b/tests/metagpt/actions/di/test_write_plan.py similarity index 95% rename from tests/metagpt/actions/mi/test_write_plan.py rename to tests/metagpt/actions/di/test_write_plan.py index 97632ea44..80b3399b8 100644 --- a/tests/metagpt/actions/mi/test_write_plan.py +++ b/tests/metagpt/actions/di/test_write_plan.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.write_plan import ( +from metagpt.actions.di.write_plan import ( Plan, Task, WritePlan, diff --git a/tests/metagpt/roles/mi/test_interpreter.py b/tests/metagpt/roles/di/test_data_interpreter.py similarity index 65% rename from tests/metagpt/roles/mi/test_interpreter.py rename to tests/metagpt/roles/di/test_data_interpreter.py index 3bae4a1ac..ba50f473b 100644 --- a/tests/metagpt/roles/mi/test_interpreter.py +++ b/tests/metagpt/roles/di/test_data_interpreter.py @@ -1,23 +1,23 @@ import pytest from metagpt.logs import logger -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter @pytest.mark.asyncio @pytest.mark.parametrize("auto_run", [(True), (False)]) async def test_interpreter(mocker, auto_run): - mocker.patch("metagpt.actions.mi.execute_nb_code.ExecuteNbCode.run", return_value=("a successful run", True)) + mocker.patch("metagpt.actions.di.execute_nb_code.ExecuteNbCode.run", return_value=("a successful run", True)) mocker.patch("builtins.input", return_value="confirm") requirement = "Run data analysis on sklearn Iris dataset, include a plot" tools = [] - mi = Interpreter(auto_run=auto_run, use_tools=True, tools=tools) - rsp = await mi.run(requirement) + di = DataInterpreter(auto_run=auto_run, use_tools=True, tools=tools) + rsp = await di.run(requirement) logger.info(rsp) assert len(rsp.content) > 0 - finished_tasks = mi.planner.plan.get_finished_tasks() + finished_tasks = di.planner.plan.get_finished_tasks() assert len(finished_tasks) > 0 assert len(finished_tasks[0].code) > 0 # check one task to see if code is recorded diff --git a/tests/metagpt/roles/mi/test_ml_engineer.py b/tests/metagpt/roles/di/test_ml_engineer.py similarity index 94% rename from tests/metagpt/roles/mi/test_ml_engineer.py rename to tests/metagpt/roles/di/test_ml_engineer.py index 921ac8822..08b92cd27 100644 --- a/tests/metagpt/roles/mi/test_ml_engineer.py +++ b/tests/metagpt/roles/di/test_ml_engineer.py @@ -1,11 +1,11 @@ import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.execute_nb_code import ExecuteNbCode from metagpt.logs import logger -from metagpt.roles.mi.ml_engineer import MLEngineer +from metagpt.roles.di.ml_engineer import MLEngineer from metagpt.schema import Message, Plan, Task from metagpt.tools.tool_type import ToolType -from tests.metagpt.actions.mi.test_debug_code import CODE, DebugContext, ErrorStr +from tests.metagpt.actions.di.test_debug_code import CODE, DebugContext, ErrorStr def test_mle_init(): diff --git a/tests/metagpt/utils/test_save_code.py b/tests/metagpt/utils/test_save_code.py index 9df2650f3..aceecec3b 100644 --- a/tests/metagpt/utils/test_save_code.py +++ b/tests/metagpt/utils/test_save_code.py @@ -6,7 +6,7 @@ import nbformat import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.execute_nb_code import ExecuteNbCode from metagpt.utils.common import read_json_file from metagpt.utils.save_code import DATA_PATH, save_code_file From 97bf135da6d78eed0a740beb8a87ca6a6fadc58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Mar 2024 12:20:58 +0800 Subject: [PATCH 04/21] feat: gemini + proxy --- metagpt/provider/google_gemini_api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 09e554205..8f8f5f707 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # @Desc : Google Gemini LLM from https://ai.google.dev/tutorials/python_quickstart +import os from typing import Optional, Union import google.generativeai as genai @@ -15,7 +16,7 @@ from google.generativeai.types.generation_types import ( ) from metagpt.configs.llm_config import LLMConfig, LLMType -from metagpt.logs import log_llm_stream +from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -51,6 +52,10 @@ class GeminiLLM(BaseLLM): self.llm = GeminiGenerativeModel(model_name=self.model) def __init_gemini(self, config: LLMConfig): + if config.proxy: + logger.info(f"Use proxy: {config.proxy}") + os.environ["HTTP_PROXY"] = config.proxy + os.environ["HTTP_PROXYS"] = config.proxy genai.configure(api_key=config.api_key) def _user_msg(self, msg: str, images: Optional[Union[str, list[str]]] = None) -> dict[str, str]: From f6260ec084443d9ae96dd6aec02afc595c709ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Mar 2024 12:27:28 +0800 Subject: [PATCH 05/21] feat: gemini + proxy --- metagpt/provider/google_gemini_api.py | 6 +++++- setup.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 2647ab16b..bdbf7acd6 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # @Desc : Google Gemini LLM from https://ai.google.dev/tutorials/python_quickstart - +import os from typing import Optional, Union import google.generativeai as genai @@ -58,6 +58,10 @@ class GeminiLLM(BaseLLM): self.llm = GeminiGenerativeModel(model_name=self.model) def __init_gemini(self, config: LLMConfig): + if config.proxy: + logger.info(f"Use proxy: {config.proxy}") + os.environ["HTTP_PROXY"] = config.proxy + os.environ["HTTP_PROXYS"] = config.proxy genai.configure(api_key=config.api_key) def _user_msg(self, msg: str, images: Optional[Union[str, list[str]]] = None) -> dict[str, str]: diff --git a/setup.py b/setup.py index 2044e577a..072237e8c 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.4", + version="0.7.5", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", From 0867dad4d7b66ad098934dd9d39b3cdb261c0c00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Mar 2024 17:04:44 +0800 Subject: [PATCH 06/21] fixbug: gbk UnicodeEncodeError --- metagpt/learn/skill_loader.py | 5 ++--- metagpt/utils/common.py | 16 +++++++++++---- metagpt/utils/dependency_file.py | 7 ++----- metagpt/utils/file_repository.py | 7 ++----- metagpt/utils/mermaid.py | 8 ++------ setup.py | 2 +- .../metagpt/roles/test_tutorial_assistant.py | 7 +++---- tests/metagpt/utils/test_common.py | 20 ++++++++++++++----- tests/metagpt/utils/test_git_repository.py | 5 ++--- tests/metagpt/utils/test_s3.py | 8 ++------ 10 files changed, 43 insertions(+), 42 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index bcf28bb87..e98f73cf9 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -9,11 +9,11 @@ from pathlib import Path from typing import Dict, List, Optional -import aiofiles import yaml from pydantic import BaseModel, Field from metagpt.context import Context +from metagpt.utils.common import aread class Example(BaseModel): @@ -68,8 +68,7 @@ class SkillsDeclaration(BaseModel): async def load(skill_yaml_file_name: Path = None) -> "SkillsDeclaration": if not skill_yaml_file_name: skill_yaml_file_name = Path(__file__).parent.parent.parent / "docs/.well-known/skills.yaml" - async with aiofiles.open(str(skill_yaml_file_name), mode="r") as reader: - data = await reader.read(-1) + data = await aread(filename=skill_yaml_file_name) skill_data = yaml.safe_load(data) return SkillsDeclaration(**skill_data) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 015902c3d..aba75fbec 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -29,6 +29,7 @@ from pathlib import Path from typing import Any, Callable, List, Tuple, Union import aiofiles +import chardet import loguru import requests from PIL import Image @@ -587,14 +588,21 @@ def role_raise_decorator(func): @handle_exception -async def aread(filename: str | Path, encoding=None) -> str: +async def aread(filename: str | Path, encoding="utf-8") -> str: """Read file asynchronously.""" - async with aiofiles.open(str(filename), mode="r", encoding=encoding) as reader: - content = await reader.read() + try: + async with aiofiles.open(str(filename), mode="r", encoding=encoding) as reader: + content = await reader.read() + except UnicodeDecodeError: + async with aiofiles.open(str(filename), mode="rb") as reader: + raw = await reader.read() + result = chardet.detect(raw) + detected_encoding = result["encoding"] + content = raw.decode(detected_encoding) return content -async def awrite(filename: str | Path, data: str, encoding=None): +async def awrite(filename: str | Path, data: str, encoding="utf-8"): """Write file asynchronously.""" pathname = Path(filename) pathname.parent.mkdir(parents=True, exist_ok=True) diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py index d3add1171..0a375051c 100644 --- a/metagpt/utils/dependency_file.py +++ b/metagpt/utils/dependency_file.py @@ -13,9 +13,7 @@ import re from pathlib import Path from typing import Set -import aiofiles - -from metagpt.utils.common import aread +from metagpt.utils.common import aread, awrite from metagpt.utils.exceptions import handle_exception @@ -45,8 +43,7 @@ class DependencyFile: async def save(self): """Save dependencies to the file asynchronously.""" data = json.dumps(self._dependencies) - async with aiofiles.open(str(self._filename), mode="w") as writer: - await writer.write(data) + await awrite(filename=self._filename, data=data) async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True): """Update dependencies for a file asynchronously. diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index d2a06963a..d19f2b705 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -14,11 +14,9 @@ from datetime import datetime from pathlib import Path from typing import Dict, List, Set -import aiofiles - from metagpt.logs import logger from metagpt.schema import Document -from metagpt.utils.common import aread +from metagpt.utils.common import aread, awrite from metagpt.utils.json_to_markdown import json_to_markdown @@ -55,8 +53,7 @@ class FileRepository: pathname = self.workdir / filename pathname.parent.mkdir(parents=True, exist_ok=True) content = content if content else "" # avoid `argument must be str, not None` to make it continue - async with aiofiles.open(str(pathname), mode="w") as writer: - await writer.write(content) + await awrite(filename=str(pathname), data=content) logger.info(f"save to: {str(pathname)}") if dependencies is not None: diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index ae3c5118f..e1d140e84 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -9,11 +9,9 @@ import asyncio import os from pathlib import Path -import aiofiles - from metagpt.config2 import config from metagpt.logs import logger -from metagpt.utils.common import check_cmd_exists +from metagpt.utils.common import awrite, check_cmd_exists async def mermaid_to_file(engine, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: @@ -30,9 +28,7 @@ async def mermaid_to_file(engine, mermaid_code, output_file_without_suffix, widt if dir_name and not os.path.exists(dir_name): os.makedirs(dir_name) tmp = Path(f"{output_file_without_suffix}.mmd") - async with aiofiles.open(tmp, "w", encoding="utf-8") as f: - await f.write(mermaid_code) - # tmp.write_text(mermaid_code, encoding="utf-8") + await awrite(filename=tmp, data=mermaid_code) if engine == "nodejs": if check_cmd_exists(config.mermaid.path) != 0: diff --git a/setup.py b/setup.py index 072237e8c..813d2410c 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.5", + version="0.7.6", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/metagpt/roles/test_tutorial_assistant.py b/tests/metagpt/roles/test_tutorial_assistant.py index c12c2b26e..732f346fd 100644 --- a/tests/metagpt/roles/test_tutorial_assistant.py +++ b/tests/metagpt/roles/test_tutorial_assistant.py @@ -6,11 +6,11 @@ @File : test_tutorial_assistant.py """ -import aiofiles import pytest from metagpt.const import TUTORIAL_PATH from metagpt.roles.tutorial_assistant import TutorialAssistant +from metagpt.utils.common import aread @pytest.mark.asyncio @@ -20,9 +20,8 @@ async def test_tutorial_assistant(language: str, topic: str, context): msg = await role.run(topic) assert TUTORIAL_PATH.exists() filename = msg.content - async with aiofiles.open(filename, mode="r", encoding="utf-8") as reader: - content = await reader.read() - assert "pip" in content + content = await aread(filename=filename) + assert "pip" in content if __name__ == "__main__": diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index 9b1fa878e..7c59b8072 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -13,7 +13,6 @@ import uuid from pathlib import Path from typing import Any, Set -import aiofiles import pytest from pydantic import BaseModel @@ -125,9 +124,7 @@ class TestGetProjectRoot: async def test_parse_data_exception(self, filename, want): pathname = Path(__file__).parent.parent.parent / "data/output_parser" / filename assert pathname.exists() - async with aiofiles.open(str(pathname), mode="r") as reader: - data = await reader.read() - + data = await aread(filename=pathname) result = OutputParser.parse_data(data=data) assert want in result @@ -198,12 +195,25 @@ class TestGetProjectRoot: @pytest.mark.asyncio async def test_read_write(self): - pathname = Path(__file__).parent / uuid.uuid4().hex / "test.tmp" + pathname = Path(__file__).parent / f"../../../workspace/unittest/{uuid.uuid4().hex}" / "test.tmp" await awrite(pathname, "ABC") data = await aread(pathname) assert data == "ABC" pathname.unlink(missing_ok=True) + @pytest.mark.asyncio + async def test_read_write_error_charset(self): + pathname = Path(__file__).parent / f"../../../workspace/unittest/{uuid.uuid4().hex}" / "test.txt" + content = "中国abc123\u27f6" + await awrite(filename=pathname, data=content) + data = await aread(filename=pathname) + assert data == content + + content = "GB18030 是中国国家标准局发布的新一代中文字符集标准，是 GBK 的升级版，支持更广泛的字符范围。" + await awrite(filename=pathname, data=content, encoding="gb2312") + data = await aread(filename=pathname, encoding="utf-8") + assert data == content + if __name__ == "__main__": pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index ea28b8f0b..480a22e24 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -10,15 +10,14 @@ import shutil from pathlib import Path -import aiofiles import pytest +from metagpt.utils.common import awrite from metagpt.utils.git_repository import GitRepository async def mock_file(filename, content=""): - async with aiofiles.open(str(filename), mode="w") as file: - await file.write(content) + await awrite(filename=filename, data=content) async def mock_repo(local_path) -> (GitRepository, Path): diff --git a/tests/metagpt/utils/test_s3.py b/tests/metagpt/utils/test_s3.py index b26ebe94d..c1a85f4ff 100644 --- a/tests/metagpt/utils/test_s3.py +++ b/tests/metagpt/utils/test_s3.py @@ -9,7 +9,6 @@ import uuid from pathlib import Path import aioboto3 -import aiofiles import pytest from metagpt.config2 import Config @@ -37,7 +36,7 @@ async def test_s3(mocker): conn = S3(s3) object_name = "unittest.bak" await conn.upload_file(bucket=s3.bucket, local_path=__file__, object_name=object_name) - pathname = (Path(__file__).parent / uuid.uuid4().hex).with_suffix(".bak") + pathname = (Path(__file__).parent / "../../../workspace/unittest" / uuid.uuid4().hex).with_suffix(".bak") pathname.unlink(missing_ok=True) await conn.download_file(bucket=s3.bucket, object_name=object_name, local_path=str(pathname)) assert pathname.exists() @@ -45,8 +44,7 @@ async def test_s3(mocker): assert url bin_data = await conn.get_object(bucket=s3.bucket, object_name=object_name) assert bin_data - async with aiofiles.open(__file__, mode="r", encoding="utf-8") as reader: - data = await reader.read() + data = await aread(filename=__file__) res = await conn.cache(data, ".bak", "script") assert "http" in res @@ -60,8 +58,6 @@ async def test_s3(mocker): except Exception: pass - await reader.close() - if __name__ == "__main__": pytest.main([__file__, "-s"]) From e39cafdd580ebcf1b587b6de40a51b5deae3fef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 11 Mar 2024 22:25:38 +0800 Subject: [PATCH 07/21] feat: + tree command --- metagpt/utils/tree.py | 99 ++++++++++++++++++++++++++++++++ tests/metagpt/utils/test_tree.py | 54 +++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 metagpt/utils/tree.py create mode 100644 tests/metagpt/utils/test_tree.py diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py new file mode 100644 index 000000000..49b5634c6 --- /dev/null +++ b/metagpt/utils/tree.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/3/11 +@Author : mashenquan +@File : tree.py +@Desc : Implement the same functionality as the `tree` command. +Example: + root + +-- dir1 + | +-- file1.txt + | +-- file2.txt + +-- dir2 + | +-- subdir1 + | | +-- file1.txt + | | +-- file2.txt + | +-- subdir2 + | +-- file1.txt + | +-- file2.txt + +-- file.txt +""" +from __future__ import annotations + +from pathlib import Path +from typing import Callable, Dict, List + +from anthropic import BaseModel +from pydantic import Field + + +class Tree(BaseModel): + """ + Represents a directory tree structure. + + Attributes: + root (str): The root directory of the tree. + tree (Dict[str, Dict]): The tree structure as a dictionary. + + Methods: + print: Print the directory tree structure. + + """ + + root: str + tree: Dict[str, Dict] = Field(default_factory=dict) + + def print(self, git_ignore_rules: Callable = None) -> str: + """ + Recursively traverses the directory structure and prints it out in a tree-like format. + + Args: + git_ignore_rules (Callable): Optional. A function to filter files to ignore. + + Returns: + str: A string representation of the directory tree. + + """ + root = Path(self.root).resolve() + self.tree[root.name] = self._list_children(root=root, git_ignore_rules=git_ignore_rules) + v = self._print_tree(self.tree) + return "\n".join(v) + + @staticmethod + def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: + tree = {} + for i in root.iterdir(): + if git_ignore_rules and git_ignore_rules(str(i)): + continue + if i.is_file(): + tree[i.name] = {} + else: + tree[i.name] = Tree._list_children(root=i, git_ignore_rules=git_ignore_rules) + return tree + + @staticmethod + def _print_tree(tree: Dict[str:Dict], indent: int = 0) -> List[str]: + ret = [] + for name, children in tree.items(): + ret.append(name) + if not children: + continue + lines = Tree._print_tree(tree=children, indent=indent + 1) + for j, v in enumerate(lines): + if v[0] not in ["+", " ", "|"]: + ret = Tree._add_line(ret) + row = f"+-- {v}" + else: + row = f" {v}" + ret.append(row) + return ret + + @staticmethod + def _add_line(rows: List[str]) -> List[str]: + for i in range(len(rows) - 1, -1, -1): + v = rows[i] + if v[0] != " ": + return rows + rows[i] = "|" + v[1:] + return rows diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py new file mode 100644 index 000000000..0d48f7ce3 --- /dev/null +++ b/tests/metagpt/utils/test_tree.py @@ -0,0 +1,54 @@ +from pathlib import Path +from typing import List + +import pytest +from gitignore_parser import parse_gitignore + +from metagpt.utils.tree import Tree + + +@pytest.mark.parametrize( + ("root", "rules"), + [ + (str(Path(__file__).parent / "../.."), None), + (str(Path(__file__).parent / "../.."), str(Path(__file__).parent / "../../../.gitignore")), + ], +) +def test_tree(root: str, rules: str): + gitignore_rules = parse_gitignore(full_path=rules) if rules else None + tree = Tree(root=root).print(git_ignore_rules=gitignore_rules) + assert tree + + +@pytest.mark.parametrize( + ("tree", "want"), + [ + ({"a": {"b": {}, "c": {}}}, ["a", "+-- b", "+-- c"]), + ({"a": {"b": {}, "c": {"d": {}}}}, ["a", "+-- b", "+-- c", " +-- d"]), + ( + {"a": {"b": {"e": {"f": {}, "g": {}}}, "c": {"d": {}}}}, + ["a", "+-- b", "| +-- e", "| +-- f", "| +-- g", "+-- c", " +-- d"], + ), + ( + {"h": {"a": {"b": {"e": {"f": {}, "g": {}}}, "c": {"d": {}}}, "i": {}}}, + [ + "h", + "+-- a", + "| +-- b", + "| | +-- e", + "| | +-- f", + "| | +-- g", + "| +-- c", + "| +-- d", + "+-- i", + ], + ), + ], +) +def test__print_tree(tree: dict, want: List[str]): + v = Tree._print_tree(tree) + assert v == want + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 7d32f9efe6958bf9b76b4309d426c590e3268e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 12 Mar 2024 11:10:09 +0800 Subject: [PATCH 08/21] refactor: Replace Tree class with tree() --- metagpt/utils/tree.py | 218 ++++++++++++++++++++----------- tests/metagpt/utils/test_tree.py | 8 +- 2 files changed, 149 insertions(+), 77 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index 49b5634c6..ad3373f5f 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -6,94 +6,166 @@ @File : tree.py @Desc : Implement the same functionality as the `tree` command. Example: - root - +-- dir1 - | +-- file1.txt - | +-- file2.txt - +-- dir2 - | +-- subdir1 - | | +-- file1.txt - | | +-- file2.txt - | +-- subdir2 - | +-- file1.txt - | +-- file2.txt - +-- file.txt + Usage: + >>> print_tree(".") + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- dependency_file.py + +-- index.html + +-- make_sk_kernel.py + +-- token_counter.py + +-- embedding.py + +-- repair_llm_raw_output.py + +-- mermaid.py + +-- parse_html.py + +-- visual_graph_repo.py + +-- special_tokens.py + +-- ahttp_client.py + +-- __init__.py + +-- mmdc_ink.py + +-- di_graph_repository.py + +-- yaml_model.py + +-- cost_manager.py + +-- __pycache__ + | +-- __init__.cpython-39.pyc + | +-- redis.cpython-39.pyc + | +-- singleton.cpython-39.pyc + | +-- mmdc_ink.cpython-39.pyc + | +-- read_document.cpython-39.pyc + | +-- mermaid.cpython-39.pyc + | +-- parse_html.cpython-39.pyc + | +-- human_interaction.cpython-39.pyc + | +-- cost_manager.cpython-39.pyc + | +-- json_to_markdown.cpython-39.pyc + | +-- graph_repository.cpython-39.pyc + | +-- ahttp_client.cpython-39.pyc + | +-- visual_graph_repo.cpython-39.pyc + | +-- file.cpython-39.pyc + | +-- di_graph_repository.cpython-39.pyc + | +-- pycst.cpython-39.pyc + | +-- save_code.cpython-39.pyc + | +-- dependency_file.cpython-39.pyc + | +-- text.cpython-39.pyc + | +-- token_counter.cpython-39.pyc + | +-- project_repo.cpython-39.pyc + | +-- yaml_model.cpython-39.pyc + | +-- serialize.cpython-39.pyc + | +-- git_repository.cpython-39.pyc + | +-- custom_decoder.cpython-39.pyc + | +-- parse_docstring.cpython-39.pyc + | +-- common.cpython-39.pyc + | +-- exceptions.cpython-39.pyc + | +-- repair_llm_raw_output.cpython-39.pyc + | +-- s3.cpython-39.pyc + | +-- embedding.cpython-39.pyc + | +-- make_sk_kernel.cpython-39.pyc + | +-- file_repository.cpython-39.pyc + +-- file.py + +-- save_code.py + +-- common.py + +-- redis.py + +-- text.py + +-- graph_repository.py + +-- singleton.py + +-- recovery_util.py + +-- file_repository.py + +-- pycst.py + +-- exceptions.py + +-- human_interaction.py + +-- highlight.py + +-- mmdc_pyppeteer.py + +-- s3.py + +-- json_to_markdown.py + +-- custom_decoder.py + +-- git_repository.py + +-- read_document.py + +-- parse_docstring.py """ from __future__ import annotations from pathlib import Path from typing import Callable, Dict, List -from anthropic import BaseModel -from pydantic import Field - -class Tree(BaseModel): +def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: """ - Represents a directory tree structure. + Recursively traverses the directory structure and prints it out in a tree-like format. - Attributes: - root (str): The root directory of the tree. - tree (Dict[str, Dict]): The tree structure as a dictionary. + Args: + root (str or Path): The root directory from which to start traversing. + git_ignore_rules (Callable): Optional. A function to filter files to ignore. - Methods: - print: Print the directory tree structure. + Returns: + str: A string representation of the directory tree. + + Example: + >>> tree(".") + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- __pycache__ + | +-- __init__.cpython-39.pyc + | +-- redis.cpython-39.pyc + | +-- singleton.cpython-39.pyc + +-- parse_docstring.py + + >>> from gitignore_parser import parse_gitignore + >>> tree(".", git_ignore_rules=parse_gitignore(full_path="../../.gitignore")) + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- parse_docstring.py """ + root = Path(root).resolve() + dir_ = {root.name: _list_children(root=root, git_ignore_rules=git_ignore_rules)} + v = _print_tree(dir_) + return "\n".join(v) - root: str - tree: Dict[str, Dict] = Field(default_factory=dict) - def print(self, git_ignore_rules: Callable = None) -> str: - """ - Recursively traverses the directory structure and prints it out in a tree-like format. - - Args: - git_ignore_rules (Callable): Optional. A function to filter files to ignore. - - Returns: - str: A string representation of the directory tree. - - """ - root = Path(self.root).resolve() - self.tree[root.name] = self._list_children(root=root, git_ignore_rules=git_ignore_rules) - v = self._print_tree(self.tree) - return "\n".join(v) - - @staticmethod - def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: - tree = {} - for i in root.iterdir(): - if git_ignore_rules and git_ignore_rules(str(i)): - continue +def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: + dir_ = {} + for i in root.iterdir(): + if git_ignore_rules and git_ignore_rules(str(i)): + continue + try: if i.is_file(): - tree[i.name] = {} + dir_[i.name] = {} else: - tree[i.name] = Tree._list_children(root=i, git_ignore_rules=git_ignore_rules) - return tree + dir_[i.name] = _list_children(root=i, git_ignore_rules=git_ignore_rules) + except (FileNotFoundError, PermissionError, OSError): + dir_[i.name] = {} + return dir_ - @staticmethod - def _print_tree(tree: Dict[str:Dict], indent: int = 0) -> List[str]: - ret = [] - for name, children in tree.items(): - ret.append(name) - if not children: - continue - lines = Tree._print_tree(tree=children, indent=indent + 1) - for j, v in enumerate(lines): - if v[0] not in ["+", " ", "|"]: - ret = Tree._add_line(ret) - row = f"+-- {v}" - else: - row = f" {v}" - ret.append(row) - return ret - @staticmethod - def _add_line(rows: List[str]) -> List[str]: - for i in range(len(rows) - 1, -1, -1): - v = rows[i] - if v[0] != " ": - return rows - rows[i] = "|" + v[1:] - return rows +def _print_tree(dir_: Dict[str:Dict]) -> List[str]: + ret = [] + for name, children in dir_.items(): + ret.append(name) + if not children: + continue + lines = _print_tree(children) + for j, v in enumerate(lines): + if v[0] not in ["+", " ", "|"]: + ret = _add_line(ret) + row = f"+-- {v}" + else: + row = f" {v}" + ret.append(row) + return ret + + +def _add_line(rows: List[str]) -> List[str]: + for i in range(len(rows) - 1, -1, -1): + v = rows[i] + if v[0] != " ": + return rows + rows[i] = "|" + v[1:] + return rows diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py index 0d48f7ce3..34eae10cf 100644 --- a/tests/metagpt/utils/test_tree.py +++ b/tests/metagpt/utils/test_tree.py @@ -4,7 +4,7 @@ from typing import List import pytest from gitignore_parser import parse_gitignore -from metagpt.utils.tree import Tree +from metagpt.utils.tree import _print_tree, tree @pytest.mark.parametrize( @@ -16,8 +16,8 @@ from metagpt.utils.tree import Tree ) def test_tree(root: str, rules: str): gitignore_rules = parse_gitignore(full_path=rules) if rules else None - tree = Tree(root=root).print(git_ignore_rules=gitignore_rules) - assert tree + v = tree(root=root, git_ignore_rules=gitignore_rules) + assert v @pytest.mark.parametrize( @@ -46,7 +46,7 @@ def test_tree(root: str, rules: str): ], ) def test__print_tree(tree: dict, want: List[str]): - v = Tree._print_tree(tree) + v = _print_tree(tree) assert v == want From 6a8699cd4a8f0f2f6f3a8095c67bc72abe775b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 12 Mar 2024 11:14:35 +0800 Subject: [PATCH 09/21] refactor: Replace Tree class with tree() --- metagpt/utils/tree.py | 61 +------------------------------------------ 1 file changed, 1 insertion(+), 60 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index ad3373f5f..1c0060842 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -5,61 +5,18 @@ @Author : mashenquan @File : tree.py @Desc : Implement the same functionality as the `tree` command. -Example: - Usage: + Example: >>> print_tree(".") utils +-- serialize.py +-- project_repo.py +-- tree.py +-- mmdc_playwright.py - +-- dependency_file.py - +-- index.html - +-- make_sk_kernel.py - +-- token_counter.py - +-- embedding.py - +-- repair_llm_raw_output.py - +-- mermaid.py - +-- parse_html.py - +-- visual_graph_repo.py - +-- special_tokens.py - +-- ahttp_client.py - +-- __init__.py - +-- mmdc_ink.py - +-- di_graph_repository.py - +-- yaml_model.py +-- cost_manager.py +-- __pycache__ | +-- __init__.cpython-39.pyc | +-- redis.cpython-39.pyc | +-- singleton.cpython-39.pyc - | +-- mmdc_ink.cpython-39.pyc - | +-- read_document.cpython-39.pyc - | +-- mermaid.cpython-39.pyc - | +-- parse_html.cpython-39.pyc - | +-- human_interaction.cpython-39.pyc - | +-- cost_manager.cpython-39.pyc - | +-- json_to_markdown.cpython-39.pyc - | +-- graph_repository.cpython-39.pyc - | +-- ahttp_client.cpython-39.pyc - | +-- visual_graph_repo.cpython-39.pyc - | +-- file.cpython-39.pyc - | +-- di_graph_repository.cpython-39.pyc - | +-- pycst.cpython-39.pyc - | +-- save_code.cpython-39.pyc - | +-- dependency_file.cpython-39.pyc - | +-- text.cpython-39.pyc - | +-- token_counter.cpython-39.pyc - | +-- project_repo.cpython-39.pyc - | +-- yaml_model.cpython-39.pyc - | +-- serialize.cpython-39.pyc - | +-- git_repository.cpython-39.pyc - | +-- custom_decoder.cpython-39.pyc - | +-- parse_docstring.cpython-39.pyc - | +-- common.cpython-39.pyc - | +-- exceptions.cpython-39.pyc - | +-- repair_llm_raw_output.cpython-39.pyc - | +-- s3.cpython-39.pyc | +-- embedding.cpython-39.pyc | +-- make_sk_kernel.cpython-39.pyc | +-- file_repository.cpython-39.pyc @@ -67,22 +24,6 @@ Example: +-- save_code.py +-- common.py +-- redis.py - +-- text.py - +-- graph_repository.py - +-- singleton.py - +-- recovery_util.py - +-- file_repository.py - +-- pycst.py - +-- exceptions.py - +-- human_interaction.py - +-- highlight.py - +-- mmdc_pyppeteer.py - +-- s3.py - +-- json_to_markdown.py - +-- custom_decoder.py - +-- git_repository.py - +-- read_document.py - +-- parse_docstring.py """ from __future__ import annotations From 684730e94f7c5fe7f5bac8a31ed8fac1937b6d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 13 Mar 2024 15:06:40 +0800 Subject: [PATCH 10/21] feat: +`tree` command --- metagpt/utils/tree.py | 36 ++++++++++++++++++++++++++++---- tests/metagpt/utils/test_tree.py | 16 +++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index 1c0060842..c0386d822 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -27,17 +27,22 @@ """ from __future__ import annotations +import subprocess from pathlib import Path from typing import Callable, Dict, List +from gitignore_parser import parse_gitignore -def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: + +def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = True) -> str: """ Recursively traverses the directory structure and prints it out in a tree-like format. Args: root (str or Path): The root directory from which to start traversing. - git_ignore_rules (Callable): Optional. A function to filter files to ignore. + gitignore (str or Path): The filename of gitignore file. + run_command (bool): Whether to execute `tree` command. Execute the `tree` command and return the result if True, + otherwise execute python code instead. Returns: str: A string representation of the directory tree. @@ -55,8 +60,7 @@ def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: | +-- singleton.cpython-39.pyc +-- parse_docstring.py - >>> from gitignore_parser import parse_gitignore - >>> tree(".", git_ignore_rules=parse_gitignore(full_path="../../.gitignore")) + >>> tree(".", gitignore="../../.gitignore") utils +-- serialize.py +-- project_repo.py @@ -64,8 +68,21 @@ def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: +-- mmdc_playwright.py +-- parse_docstring.py + >>> tree(".", gitignore="../../.gitignore", run_command=True) + utils + ├── serialize.py + ├── project_repo.py + ├── tree.py + ├── mmdc_playwright.py + └── parse_docstring.py + + """ root = Path(root).resolve() + if run_command: + return _execute_tree(root, gitignore) + + git_ignore_rules = parse_gitignore(gitignore) if gitignore else None dir_ = {root.name: _list_children(root=root, git_ignore_rules=git_ignore_rules)} v = _print_tree(dir_) return "\n".join(v) @@ -110,3 +127,14 @@ def _add_line(rows: List[str]) -> List[str]: return rows rows[i] = "|" + v[1:] return rows + + +def _execute_tree(root: Path, gitignore: str | Path) -> str: + args = ["--gitignore", str(gitignore)] if gitignore else [] + try: + result = subprocess.run(["tree"] + args + [str(root)], capture_output=True, text=True, check=True) + if result.returncode != 0: + raise ValueError(f"tree exits with code {result.returncode}") + return result.stdout + except subprocess.CalledProcessError as e: + raise e diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py index 34eae10cf..03a2a5606 100644 --- a/tests/metagpt/utils/test_tree.py +++ b/tests/metagpt/utils/test_tree.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import List import pytest -from gitignore_parser import parse_gitignore from metagpt.utils.tree import _print_tree, tree @@ -15,8 +14,19 @@ from metagpt.utils.tree import _print_tree, tree ], ) def test_tree(root: str, rules: str): - gitignore_rules = parse_gitignore(full_path=rules) if rules else None - v = tree(root=root, git_ignore_rules=gitignore_rules) + v = tree(root=root, gitignore=rules) + assert v + + +@pytest.mark.parametrize( + ("root", "rules"), + [ + (str(Path(__file__).parent / "../.."), None), + (str(Path(__file__).parent / "../.."), str(Path(__file__).parent / "../../../.gitignore")), + ], +) +def test_tree_command(root: str, rules: str): + v = tree(root=root, gitignore=rules, run_command=True) assert v From 6f4d30825f84d90c35dab04e639fc1b483e1a823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 13 Mar 2024 15:10:13 +0800 Subject: [PATCH 11/21] feat: +`tree` command --- metagpt/utils/tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index c0386d822..fbf085e48 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -34,7 +34,7 @@ from typing import Callable, Dict, List from gitignore_parser import parse_gitignore -def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = True) -> str: +def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = False) -> str: """ Recursively traverses the directory structure and prints it out in a tree-like format. From af3a409ac4b7b1632512384ff86d46349a746145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 20 Mar 2024 21:24:41 +0800 Subject: [PATCH 12/21] fixbug: llm.timeout not working --- metagpt/actions/action_node.py | 6 +++--- metagpt/configs/llm_config.py | 2 +- metagpt/provider/anthropic_api.py | 8 ++++---- metagpt/provider/base_llm.py | 26 +++++++++++++++----------- metagpt/provider/dashscope_api.py | 8 ++++---- metagpt/provider/general_api_base.py | 2 +- metagpt/provider/google_gemini_api.py | 8 ++++---- metagpt/provider/human_provider.py | 14 +++++++------- metagpt/provider/ollama_api.py | 13 ++++++------- metagpt/provider/openai_api.py | 26 +++++++++++++------------- metagpt/provider/qianfan_api.py | 8 ++++---- metagpt/provider/spark_api.py | 8 ++++---- metagpt/provider/zhipuai_api.py | 10 +++++----- requirements.txt | 2 +- 14 files changed, 72 insertions(+), 69 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 09da4a988..63925a052 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -416,7 +416,7 @@ class ActionNode: images: Optional[Union[str, list[str]]] = None, system_msgs: Optional[list[str]] = None, schema="markdown", # compatible to original format - timeout=3, + timeout=0, ) -> (str, BaseModel): """Use ActionOutput to wrap the output of aask""" content = await self.llm.aask(prompt, system_msgs, images=images, timeout=timeout) @@ -448,7 +448,7 @@ class ActionNode: def set_context(self, context): self.set_recursive("context", context) - async def simple_fill(self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=3, exclude=None): + async def simple_fill(self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=0, exclude=None): prompt = self.compile(context=self.context, schema=schema, mode=mode, exclude=exclude) if schema != "raw": @@ -473,7 +473,7 @@ class ActionNode: mode="auto", strgy="simple", images: Optional[Union[str, list[str]]] = None, - timeout=3, + timeout=0, exclude=[], ): """Fill the node(s) with mode. diff --git a/metagpt/configs/llm_config.py b/metagpt/configs/llm_config.py index fa9bc0b1b..92b8e1512 100644 --- a/metagpt/configs/llm_config.py +++ b/metagpt/configs/llm_config.py @@ -74,7 +74,7 @@ class LLMConfig(YamlModel): stream: bool = False logprobs: Optional[bool] = None # https://cookbook.openai.com/examples/using_logprobs top_logprobs: Optional[int] = None - timeout: int = 60 + timeout: int = 600 # For Network proxy: Optional[str] = None diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 872f9b2c7..3125ffc22 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -41,15 +41,15 @@ class AnthropicLLM(BaseLLM): def get_choice_text(self, resp: Message) -> str: return resp.content[0].text - async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> Message: + async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> Message: resp: Message = await self.aclient.messages.create(**self._const_kwargs(messages)) self._update_costs(resp.usage, self.model) return resp - async def acompletion(self, messages: list[dict], timeout: int = 3) -> Message: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout: int = 0) -> Message: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: stream = await self.aclient.messages.create(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = Usage(input_tokens=0, output_tokens=0) diff --git a/metagpt/provider/base_llm.py b/metagpt/provider/base_llm.py index 71308930a..fa5119c67 100644 --- a/metagpt/provider/base_llm.py +++ b/metagpt/provider/base_llm.py @@ -23,6 +23,7 @@ from tenacity import ( ) from metagpt.configs.llm_config import LLMConfig +from metagpt.const import LLM_API_TIMEOUT from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import log_and_reraise @@ -108,7 +109,7 @@ class BaseLLM(ABC): system_msgs: Optional[list[str]] = None, format_msgs: Optional[list[dict[str, str]]] = None, images: Optional[Union[str, list[str]]] = None, - timeout=3, + timeout=0, stream=True, ) -> str: if system_msgs: @@ -124,31 +125,31 @@ class BaseLLM(ABC): else: message.extend(msg) logger.debug(message) - rsp = await self.acompletion_text(message, stream=stream, timeout=timeout) + rsp = await self.acompletion_text(message, stream=stream, timeout=self.get_timeout(timeout)) return rsp def _extract_assistant_rsp(self, context): return "\n".join([i["content"] for i in context if i["role"] == "assistant"]) - async def aask_batch(self, msgs: list, timeout=3) -> str: + async def aask_batch(self, msgs: list, timeout=0) -> str: """Sequential questioning""" context = [] for msg in msgs: umsg = self._user_msg(msg) context.append(umsg) - rsp_text = await self.acompletion_text(context, timeout=timeout) + rsp_text = await self.acompletion_text(context, timeout=self.get_timeout(timeout)) context.append(self._assistant_msg(rsp_text)) return self._extract_assistant_rsp(context) - async def aask_code(self, messages: Union[str, Message, list[dict]], timeout=3, **kwargs) -> dict: + async def aask_code(self, messages: Union[str, Message, list[dict]], timeout=0, **kwargs) -> dict: raise NotImplementedError @abstractmethod - async def _achat_completion(self, messages: list[dict], timeout=3): + async def _achat_completion(self, messages: list[dict], timeout=0): """_achat_completion implemented by inherited class""" @abstractmethod - async def acompletion(self, messages: list[dict], timeout=3): + async def acompletion(self, messages: list[dict], timeout=0): """Asynchronous version of completion All GPTAPIs are required to provide the standard OpenAI completion interface [ @@ -159,7 +160,7 @@ class BaseLLM(ABC): """ @abstractmethod - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: """_achat_completion_stream implemented by inherited class""" @retry( @@ -169,11 +170,11 @@ class BaseLLM(ABC): retry=retry_if_exception_type(ConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream: bool = False, timeout: int = 3) -> str: + async def acompletion_text(self, messages: list[dict], stream: bool = False, timeout: int = 0) -> str: """Asynchronous version of completion. Return str. Support stream-print""" if stream: - return await self._achat_completion_stream(messages, timeout=timeout) - resp = await self._achat_completion(messages, timeout=timeout) + return await self._achat_completion_stream(messages, timeout=self.get_timeout(timeout)) + resp = await self._achat_completion(messages, timeout=self.get_timeout(timeout)) return self.get_choice_text(resp) def get_choice_text(self, rsp: dict) -> str: @@ -236,3 +237,6 @@ class BaseLLM(ABC): """Set model and return self. For example, `with_model("gpt-3.5-turbo")`.""" self.config.model = model return self + + def get_timeout(self, timeout: int) -> int: + return timeout or self.config.timeout or LLM_API_TIMEOUT diff --git a/metagpt/provider/dashscope_api.py b/metagpt/provider/dashscope_api.py index 21f3ef351..a3efd5116 100644 --- a/metagpt/provider/dashscope_api.py +++ b/metagpt/provider/dashscope_api.py @@ -202,16 +202,16 @@ class DashScopeLLM(BaseLLM): self._update_costs(dict(resp.usage)) return resp.output - async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> GenerationOutput: + async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> GenerationOutput: resp: GenerationResponse = await self.aclient.acall(**self._const_kwargs(messages, stream=False)) self._check_response(resp) self._update_costs(dict(resp.usage)) return resp.output - async def acompletion(self, messages: list[dict], timeout=3) -> GenerationOutput: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout=0) -> GenerationOutput: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: resp = await self.aclient.acall(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = {} diff --git a/metagpt/provider/general_api_base.py b/metagpt/provider/general_api_base.py index 1b9149396..8e5da8f16 100644 --- a/metagpt/provider/general_api_base.py +++ b/metagpt/provider/general_api_base.py @@ -573,7 +573,7 @@ class APIRequestor: total=request_timeout[1], ) else: - timeout = aiohttp.ClientTimeout(total=request_timeout if request_timeout else TIMEOUT_SECS) + timeout = aiohttp.ClientTimeout(total=request_timeout or TIMEOUT_SECS) if files: # TODO: Use `aiohttp.MultipartWriter` to create the multipart form data here. diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 09e554205..2a00d4b5a 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -88,16 +88,16 @@ class GeminiLLM(BaseLLM): self._update_costs(usage) return resp - async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> "AsyncGenerateContentResponse": + async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> "AsyncGenerateContentResponse": resp: AsyncGenerateContentResponse = await self.llm.generate_content_async(**self._const_kwargs(messages)) usage = await self.aget_usage(messages, resp.text) self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=3) -> dict: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout=0) -> dict: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: resp: AsyncGenerateContentResponse = await self.llm.generate_content_async( **self._const_kwargs(messages, stream=True) ) diff --git a/metagpt/provider/human_provider.py b/metagpt/provider/human_provider.py index e5f37c5b9..df63a8bc9 100644 --- a/metagpt/provider/human_provider.py +++ b/metagpt/provider/human_provider.py @@ -18,7 +18,7 @@ class HumanProvider(BaseLLM): def __init__(self, config: LLMConfig): pass - def ask(self, msg: str, timeout=3) -> str: + def ask(self, msg: str, timeout=0) -> str: logger.info("It's your turn, please type in your response. You may also refer to the context below") rsp = input(msg) if rsp in ["exit", "quit"]: @@ -31,20 +31,20 @@ class HumanProvider(BaseLLM): system_msgs: Optional[list[str]] = None, format_msgs: Optional[list[dict[str, str]]] = None, generator: bool = False, - timeout=3, + timeout=0, ) -> str: - return self.ask(msg, timeout=timeout) + return self.ask(msg, timeout=self.get_timeout(timeout)) - async def _achat_completion(self, messages: list[dict], timeout=3): + async def _achat_completion(self, messages: list[dict], timeout=0): pass - async def acompletion(self, messages: list[dict], timeout=3): + async def acompletion(self, messages: list[dict], timeout=0): """dummy implementation of abstract method in base""" return [] - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: pass - async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout=0) -> str: """dummy implementation of abstract method in base""" return "" diff --git a/metagpt/provider/ollama_api.py b/metagpt/provider/ollama_api.py index f65d7e411..723abb574 100644 --- a/metagpt/provider/ollama_api.py +++ b/metagpt/provider/ollama_api.py @@ -5,7 +5,6 @@ import json from metagpt.configs.llm_config import LLMConfig, LLMType -from metagpt.const import LLM_API_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.general_api_requestor import GeneralAPIRequestor @@ -50,28 +49,28 @@ class OllamaLLM(BaseLLM): chunk = chunk.decode(encoding) return json.loads(chunk) - async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> dict: + async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> dict: resp, _, _ = await self.client.arequest( method=self.http_method, url=self.suffix_url, params=self._const_kwargs(messages), - request_timeout=LLM_API_TIMEOUT, + request_timeout=self.get_timeout(timeout), ) resp = self._decode_and_load(resp) usage = self.get_usage(resp) self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=3) -> dict: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout=0) -> dict: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: stream_resp, _, _ = await self.client.arequest( method=self.http_method, url=self.suffix_url, stream=True, params=self._const_kwargs(messages, stream=True), - request_timeout=LLM_API_TIMEOUT, + request_timeout=self.get_timeout(timeout), ) collected_content = [] diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index b4f99e69f..5b4abae61 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -79,9 +79,9 @@ class OpenAILLM(BaseLLM): return params - async def _achat_completion_stream(self, messages: list[dict], timeout=3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout=0) -> str: response: AsyncStream[ChatCompletionChunk] = await self.aclient.chat.completions.create( - **self._cons_kwargs(messages, timeout=timeout), stream=True + **self._cons_kwargs(messages, timeout=self.get_timeout(timeout)), stream=True ) usage = None collected_messages = [] @@ -109,7 +109,7 @@ class OpenAILLM(BaseLLM): self._update_costs(usage) return full_reply_content - def _cons_kwargs(self, messages: list[dict], timeout=3, **extra_kwargs) -> dict: + def _cons_kwargs(self, messages: list[dict], timeout=0, **extra_kwargs) -> dict: kwargs = { "messages": messages, "max_tokens": self._get_max_tokens(messages), @@ -117,20 +117,20 @@ class OpenAILLM(BaseLLM): # "stop": None, # default it's None and gpt4-v can't have this one "temperature": self.config.temperature, "model": self.model, - "timeout": max(self.config.timeout, timeout), + "timeout": self.get_timeout(timeout), } if extra_kwargs: kwargs.update(extra_kwargs) return kwargs - async def _achat_completion(self, messages: list[dict], timeout=3) -> ChatCompletion: - kwargs = self._cons_kwargs(messages, timeout=timeout) + async def _achat_completion(self, messages: list[dict], timeout=0) -> ChatCompletion: + kwargs = self._cons_kwargs(messages, timeout=self.get_timeout(timeout)) rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs) self._update_costs(rsp.usage) return rsp - async def acompletion(self, messages: list[dict], timeout=3) -> ChatCompletion: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout=0) -> ChatCompletion: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) @retry( wait=wait_random_exponential(min=1, max=60), @@ -139,24 +139,24 @@ class OpenAILLM(BaseLLM): retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout=0) -> str: """when streaming, print each token in place.""" if stream: return await self._achat_completion_stream(messages, timeout=timeout) - rsp = await self._achat_completion(messages, timeout=timeout) + rsp = await self._achat_completion(messages, timeout=self.get_timeout(timeout)) return self.get_choice_text(rsp) async def _achat_completion_function( - self, messages: list[dict], timeout: int = 3, **chat_configs + self, messages: list[dict], timeout: int = 0, **chat_configs ) -> ChatCompletion: messages = process_message(messages) - kwargs = self._cons_kwargs(messages=messages, timeout=timeout, **chat_configs) + kwargs = self._cons_kwargs(messages=messages, timeout=self.get_timeout(timeout), **chat_configs) rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs) self._update_costs(rsp.usage) return rsp - async def aask_code(self, messages: list[dict], timeout: int = 3, **kwargs) -> dict: + async def aask_code(self, messages: list[dict], timeout: int = 0, **kwargs) -> dict: """Use function of tools to ask a code. Note: Keep kwargs consistent with https://platform.openai.com/docs/api-reference/chat/create diff --git a/metagpt/provider/qianfan_api.py b/metagpt/provider/qianfan_api.py index 50916fa3e..7e0bf009e 100644 --- a/metagpt/provider/qianfan_api.py +++ b/metagpt/provider/qianfan_api.py @@ -107,15 +107,15 @@ class QianFanLLM(BaseLLM): self._update_costs(resp.body.get("usage", {})) return resp.body - async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> JsonBody: + async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> JsonBody: resp = await self.aclient.ado(**self._const_kwargs(messages=messages, stream=False)) self._update_costs(resp.body.get("usage", {})) return resp.body - async def acompletion(self, messages: list[dict], timeout: int = 3) -> JsonBody: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout: int = 0) -> JsonBody: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: resp = await self.aclient.ado(**self._const_kwargs(messages=messages, stream=True)) collected_content = [] usage = {} diff --git a/metagpt/provider/spark_api.py b/metagpt/provider/spark_api.py index 882c6ce85..0f450ccf5 100644 --- a/metagpt/provider/spark_api.py +++ b/metagpt/provider/spark_api.py @@ -31,19 +31,19 @@ class SparkLLM(BaseLLM): def get_choice_text(self, rsp: dict) -> str: return rsp["payload"]["choices"]["text"][-1]["content"] - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: pass - async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 3) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 0) -> str: # 不支持 # logger.warning("当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。") w = GetMessageFromWeb(messages, self.config) return w.run() - async def _achat_completion(self, messages: list[dict], timeout=3): + async def _achat_completion(self, messages: list[dict], timeout=0): pass - async def acompletion(self, messages: list[dict], timeout=3): + async def acompletion(self, messages: list[dict], timeout=0): # 不支持异步 w = GetMessageFromWeb(messages, self.config) return w.run() diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 14ad1a36b..8c5284770 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -45,22 +45,22 @@ class ZhiPuAILLM(BaseLLM): kwargs = {"model": self.model, "messages": messages, "stream": stream, "temperature": 0.3} return kwargs - def completion(self, messages: list[dict], timeout=3) -> dict: + def completion(self, messages: list[dict], timeout=0) -> dict: resp: Completion = self.llm.chat.completions.create(**self._const_kwargs(messages)) usage = resp.usage.model_dump() self._update_costs(usage) return resp.model_dump() - async def _achat_completion(self, messages: list[dict], timeout=3) -> dict: + async def _achat_completion(self, messages: list[dict], timeout=0) -> dict: resp = await self.llm.acreate(**self._const_kwargs(messages)) usage = resp.get("usage", {}) self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=3) -> dict: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout=0) -> dict: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout=3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout=0) -> str: response = await self.llm.acreate_stream(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = {} diff --git a/requirements.txt b/requirements.txt index 83565278b..6b23e47b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ PyYAML==6.0.1 # sentence_transformers==2.2.2 setuptools==65.6.3 tenacity==8.2.3 -tiktoken==0.5.2 +tiktoken==0.6.0 tqdm==4.66.2 #unstructured[local-inference] # selenium>4 From 7c8f57e46c16aaa5a3acfeae6f3659f47b8952d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 20 Mar 2024 21:37:47 +0800 Subject: [PATCH 13/21] feat: + timeout --- config/config2.example.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config2.example.yaml b/config/config2.example.yaml index 3a5cc3585..46059e7e1 100644 --- a/config/config2.example.yaml +++ b/config/config2.example.yaml @@ -4,6 +4,7 @@ llm: api_key: "YOUR_API_KEY" model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview proxy: "YOUR_PROXY" # for LLM API requests + # timeout: 600 # Optional. pricing_plan: "" # Optional. If invalid, it will be automatically filled in with the value of the `model`. # Azure-exclusive pricing plan mappings： # - gpt-3.5-turbo 4k: "gpt-3.5-turbo-1106" From 9350e214b4280049387d50bf8af2ae6bf5205029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 12:59:44 +0800 Subject: [PATCH 14/21] feat: + repo to markdown --- .gitignore | 2 +- metagpt/utils/common.py | 19 +++++ metagpt/utils/repo_to_markdown.py | 80 ++++++++++++++++++++ metagpt/utils/tree.py | 2 +- tests/metagpt/utils/test_repo_to_markdown.py | 25 ++++++ 5 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 metagpt/utils/repo_to_markdown.py create mode 100644 tests/metagpt/utils/test_repo_to_markdown.py diff --git a/.gitignore b/.gitignore index 922116d12..aa5edd74a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ ### Python template # Byte-compiled / optimized / DLL files -__pycache__/ +__pycache__ *.py[cod] *$py.class diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index e9cef69a4..cc40e3762 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -18,6 +18,7 @@ import csv import importlib import inspect import json +import mimetypes import os import platform import re @@ -834,3 +835,21 @@ See FAQ 5.8 """ ) raise retry_state.outcome.exception() + + +def get_markdown_codeblock_type(filename: str) -> str: + """Return the markdown code-block type corresponding to the file extension.""" + mime_type, _ = mimetypes.guess_type(filename) + mappings = { + "text/x-shellscript": "bash", + "text/x-c++src": "cpp", + "text/css": "css", + "text/html": "html", + "text/x-java": "java", + "application/javascript": "javascript", + "application/json": "json", + "text/x-python": "python", + "text/x-ruby": "ruby", + "application/sql": "sql", + } + return mappings.get(mime_type, "text") diff --git a/metagpt/utils/repo_to_markdown.py b/metagpt/utils/repo_to_markdown.py new file mode 100644 index 000000000..76dfe1b82 --- /dev/null +++ b/metagpt/utils/repo_to_markdown.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +This file provides functionality to convert a local repository into a markdown representation. +""" +from __future__ import annotations + +import mimetypes +from pathlib import Path + +from gitignore_parser import parse_gitignore + +from metagpt.logs import logger +from metagpt.utils.common import aread, awrite, get_markdown_codeblock_type, list_files +from metagpt.utils.tree import tree + + +async def repo_to_markdown(repo_path: str | Path, output: str | Path = None, gitignore: str | Path = None) -> str: + """ + Convert a local repository into a markdown representation. + + This function takes a path to a local repository and generates a markdown representation of the repository structure, + including directory trees and file listings. + + Args: + repo_path (str | Path): The path to the local repository. + output (str | Path, optional): The path to save the generated markdown file. Defaults to None. + gitignore (str | Path, optional): The path to the .gitignore file. Defaults to None. + + Returns: + str: The markdown representation of the repository. + """ + repo_path = Path(repo_path) + gitignore = Path(gitignore or Path(__file__).parent / "../../.gitignore").resolve() + + markdown = await _write_dir_tree(repo_path=repo_path, gitignore=gitignore) + + gitignore_rules = parse_gitignore(full_path=str(gitignore)) + markdown += await _write_files(repo_path=repo_path, gitignore_rules=gitignore_rules) + + if output: + await awrite(filename=str(output), data=markdown, encoding="utf-8") + return markdown + + +async def _write_dir_tree(repo_path: Path, gitignore: Path) -> str: + try: + content = tree(repo_path, gitignore, run_command=True) + except Exception as e: + logger.info(f"{e}, using safe mode.") + content = tree(repo_path, gitignore, run_command=False) + + doc = f"## Directory Tree\n```text\n{content}\n```\n---\n\n" + return doc + + +async def _write_files(repo_path, gitignore_rules) -> str: + filenames = list_files(repo_path) + markdown = "" + for filename in filenames: + if gitignore_rules(str(filename)): + continue + markdown += await _write_file(filename=filename, repo_path=repo_path) + return markdown + + +async def _write_file(filename: Path, repo_path: Path) -> str: + relative_path = filename.relative_to(repo_path) + markdown = f"## {relative_path}\n" + + mime_type, _ = mimetypes.guess_type(filename.name) + if "text/" not in mime_type: + logger.info(f"Ignore content: {filename}") + markdown += "\n---\n\n" + return markdown + content = await aread(filename, encoding="utf-8") + content = content.replace("```", "\\`\\`\\`").replace("---", "\\-\\-\\-") + code_block_type = get_markdown_codeblock_type(filename.name) + markdown += f"```{code_block_type}\n{content}\n```\n---\n\n" + return markdown diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index fbf085e48..bd7922290 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -130,7 +130,7 @@ def _add_line(rows: List[str]) -> List[str]: def _execute_tree(root: Path, gitignore: str | Path) -> str: - args = ["--gitignore", str(gitignore)] if gitignore else [] + args = ["--gitfile", str(gitignore)] if gitignore else [] try: result = subprocess.run(["tree"] + args + [str(root)], capture_output=True, text=True, check=True) if result.returncode != 0: diff --git a/tests/metagpt/utils/test_repo_to_markdown.py b/tests/metagpt/utils/test_repo_to_markdown.py new file mode 100644 index 000000000..914c50dd7 --- /dev/null +++ b/tests/metagpt/utils/test_repo_to_markdown.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import uuid +from pathlib import Path + +import pytest + +from metagpt.utils.repo_to_markdown import repo_to_markdown + + +@pytest.mark.parametrize( + ["repo_path", "output"], + [(Path(__file__).parent.parent, Path(__file__).parent.parent.parent / f"workspace/unittest/{uuid.uuid4().hex}.md")], +) +@pytest.mark.asyncio +async def test_repo_to_markdown(repo_path: Path, output: Path): + markdown = await repo_to_markdown(repo_path=repo_path, output=output) + assert output.exists() + assert markdown + + output.unlink(missing_ok=True) + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From b42cf5cbd6d1cc7b0ffb22e893efb754022da8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 13:21:24 +0800 Subject: [PATCH 15/21] refactor: timeout 0 --- config/config2.example.yaml | 2 +- metagpt/actions/action_node.py | 9 ++++++--- metagpt/configs/llm_config.py | 6 ++++++ metagpt/const.py | 5 ++++- metagpt/provider/anthropic_api.py | 7 ++++--- metagpt/provider/base_llm.py | 18 ++++++++++-------- metagpt/provider/dashscope_api.py | 7 ++++--- metagpt/provider/google_gemini_api.py | 9 ++++++--- metagpt/provider/human_provider.py | 13 +++++++------ metagpt/provider/ollama_api.py | 7 ++++--- metagpt/provider/openai_api.py | 15 ++++++++------- metagpt/provider/qianfan_api.py | 7 ++++--- metagpt/provider/spark_api.py | 9 +++++---- metagpt/provider/zhipuai_api.py | 9 +++++---- 14 files changed, 74 insertions(+), 49 deletions(-) diff --git a/config/config2.example.yaml b/config/config2.example.yaml index 46059e7e1..c5454ec32 100644 --- a/config/config2.example.yaml +++ b/config/config2.example.yaml @@ -4,7 +4,7 @@ llm: api_key: "YOUR_API_KEY" model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview proxy: "YOUR_PROXY" # for LLM API requests - # timeout: 600 # Optional. + # timeout: 600 # Optional. If set to 0, default value is 300. pricing_plan: "" # Optional. If invalid, it will be automatically filled in with the value of the `model`. # Azure-exclusive pricing plan mappings： # - gpt-3.5-turbo 4k: "gpt-3.5-turbo-1106" diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 63925a052..3f822568e 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -17,6 +17,7 @@ from pydantic import BaseModel, Field, create_model, model_validator from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action_outcls_registry import register_action_outcls +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.llm import BaseLLM from metagpt.logs import logger from metagpt.provider.postprocess.llm_output_postprocess import llm_output_postprocess @@ -416,7 +417,7 @@ class ActionNode: images: Optional[Union[str, list[str]]] = None, system_msgs: Optional[list[str]] = None, schema="markdown", # compatible to original format - timeout=0, + timeout=USE_CONFIG_TIMEOUT, ) -> (str, BaseModel): """Use ActionOutput to wrap the output of aask""" content = await self.llm.aask(prompt, system_msgs, images=images, timeout=timeout) @@ -448,7 +449,9 @@ class ActionNode: def set_context(self, context): self.set_recursive("context", context) - async def simple_fill(self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=0, exclude=None): + async def simple_fill( + self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=USE_CONFIG_TIMEOUT, exclude=None + ): prompt = self.compile(context=self.context, schema=schema, mode=mode, exclude=exclude) if schema != "raw": @@ -473,7 +476,7 @@ class ActionNode: mode="auto", strgy="simple", images: Optional[Union[str, list[str]]] = None, - timeout=0, + timeout=USE_CONFIG_TIMEOUT, exclude=[], ): """Fill the node(s) with mode. diff --git a/metagpt/configs/llm_config.py b/metagpt/configs/llm_config.py index 92b8e1512..af8f56372 100644 --- a/metagpt/configs/llm_config.py +++ b/metagpt/configs/llm_config.py @@ -10,6 +10,7 @@ from typing import Optional from pydantic import field_validator +from metagpt.const import LLM_API_TIMEOUT from metagpt.utils.yaml_model import YamlModel @@ -88,3 +89,8 @@ class LLMConfig(YamlModel): if v in ["", None, "YOUR_API_KEY"]: raise ValueError("Please set your API key in config2.yaml") return v + + @field_validator("timeout") + @classmethod + def check_timeout(cls, v): + return v or LLM_API_TIMEOUT diff --git a/metagpt/const.py b/metagpt/const.py index 6dbbfe0c1..e4cebfd96 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -123,7 +123,6 @@ BASE64_FORMAT = "base64" # REDIS REDIS_KEY = "REDIS_KEY" -LLM_API_TIMEOUT = 300 # Message id IGNORED_MESSAGE_ID = "0" @@ -132,3 +131,7 @@ IGNORED_MESSAGE_ID = "0" GENERALIZATION = "Generalize" COMPOSITION = "Composite" AGGREGATION = "Aggregate" + +# Timeout +USE_CONFIG_TIMEOUT = 0 # Using llm.timeout configuration. +LLM_API_TIMEOUT = 300 diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 3125ffc22..1aeacbe83 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -5,6 +5,7 @@ from anthropic import AsyncAnthropic from anthropic.types import Message, Usage from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -41,15 +42,15 @@ class AnthropicLLM(BaseLLM): def get_choice_text(self, resp: Message) -> str: return resp.content[0].text - async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> Message: + async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> Message: resp: Message = await self.aclient.messages.create(**self._const_kwargs(messages)) self._update_costs(resp.usage, self.model) return resp - async def acompletion(self, messages: list[dict], timeout: int = 0) -> Message: + async def acompletion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> Message: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: stream = await self.aclient.messages.create(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = Usage(input_tokens=0, output_tokens=0) diff --git a/metagpt/provider/base_llm.py b/metagpt/provider/base_llm.py index fa5119c67..e085d0187 100644 --- a/metagpt/provider/base_llm.py +++ b/metagpt/provider/base_llm.py @@ -23,7 +23,7 @@ from tenacity import ( ) from metagpt.configs.llm_config import LLMConfig -from metagpt.const import LLM_API_TIMEOUT +from metagpt.const import LLM_API_TIMEOUT, USE_CONFIG_TIMEOUT from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import log_and_reraise @@ -109,7 +109,7 @@ class BaseLLM(ABC): system_msgs: Optional[list[str]] = None, format_msgs: Optional[list[dict[str, str]]] = None, images: Optional[Union[str, list[str]]] = None, - timeout=0, + timeout=USE_CONFIG_TIMEOUT, stream=True, ) -> str: if system_msgs: @@ -131,7 +131,7 @@ class BaseLLM(ABC): def _extract_assistant_rsp(self, context): return "\n".join([i["content"] for i in context if i["role"] == "assistant"]) - async def aask_batch(self, msgs: list, timeout=0) -> str: + async def aask_batch(self, msgs: list, timeout=USE_CONFIG_TIMEOUT) -> str: """Sequential questioning""" context = [] for msg in msgs: @@ -141,15 +141,15 @@ class BaseLLM(ABC): context.append(self._assistant_msg(rsp_text)) return self._extract_assistant_rsp(context) - async def aask_code(self, messages: Union[str, Message, list[dict]], timeout=0, **kwargs) -> dict: + async def aask_code(self, messages: Union[str, Message, list[dict]], timeout=USE_CONFIG_TIMEOUT, **kwargs) -> dict: raise NotImplementedError @abstractmethod - async def _achat_completion(self, messages: list[dict], timeout=0): + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): """_achat_completion implemented by inherited class""" @abstractmethod - async def acompletion(self, messages: list[dict], timeout=0): + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): """Asynchronous version of completion All GPTAPIs are required to provide the standard OpenAI completion interface [ @@ -160,7 +160,7 @@ class BaseLLM(ABC): """ @abstractmethod - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: """_achat_completion_stream implemented by inherited class""" @retry( @@ -170,7 +170,9 @@ class BaseLLM(ABC): retry=retry_if_exception_type(ConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream: bool = False, timeout: int = 0) -> str: + async def acompletion_text( + self, messages: list[dict], stream: bool = False, timeout: int = USE_CONFIG_TIMEOUT + ) -> str: """Asynchronous version of completion. Return str. Support stream-print""" if stream: return await self._achat_completion_stream(messages, timeout=self.get_timeout(timeout)) diff --git a/metagpt/provider/dashscope_api.py b/metagpt/provider/dashscope_api.py index a3efd5116..82224e893 100644 --- a/metagpt/provider/dashscope_api.py +++ b/metagpt/provider/dashscope_api.py @@ -25,6 +25,7 @@ from dashscope.common.error import ( UnsupportedApiProtocol, ) +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM, LLMConfig from metagpt.provider.llm_provider_registry import LLMType, register_provider @@ -202,16 +203,16 @@ class DashScopeLLM(BaseLLM): self._update_costs(dict(resp.usage)) return resp.output - async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> GenerationOutput: + async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> GenerationOutput: resp: GenerationResponse = await self.aclient.acall(**self._const_kwargs(messages, stream=False)) self._check_response(resp) self._update_costs(dict(resp.usage)) return resp.output - async def acompletion(self, messages: list[dict], timeout=0) -> GenerationOutput: + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> GenerationOutput: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: resp = await self.aclient.acall(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = {} diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 2a00d4b5a..e041f4c87 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -15,6 +15,7 @@ from google.generativeai.types.generation_types import ( ) from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -88,16 +89,18 @@ class GeminiLLM(BaseLLM): self._update_costs(usage) return resp - async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> "AsyncGenerateContentResponse": + async def _achat_completion( + self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT + ) -> "AsyncGenerateContentResponse": resp: AsyncGenerateContentResponse = await self.llm.generate_content_async(**self._const_kwargs(messages)) usage = await self.aget_usage(messages, resp.text) self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=0) -> dict: + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: resp: AsyncGenerateContentResponse = await self.llm.generate_content_async( **self._const_kwargs(messages, stream=True) ) diff --git a/metagpt/provider/human_provider.py b/metagpt/provider/human_provider.py index df63a8bc9..f205ecd1f 100644 --- a/metagpt/provider/human_provider.py +++ b/metagpt/provider/human_provider.py @@ -6,6 +6,7 @@ Author: garylin2099 from typing import Optional from metagpt.configs.llm_config import LLMConfig +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import logger from metagpt.provider.base_llm import BaseLLM @@ -18,7 +19,7 @@ class HumanProvider(BaseLLM): def __init__(self, config: LLMConfig): pass - def ask(self, msg: str, timeout=0) -> str: + def ask(self, msg: str, timeout=USE_CONFIG_TIMEOUT) -> str: logger.info("It's your turn, please type in your response. You may also refer to the context below") rsp = input(msg) if rsp in ["exit", "quit"]: @@ -31,20 +32,20 @@ class HumanProvider(BaseLLM): system_msgs: Optional[list[str]] = None, format_msgs: Optional[list[dict[str, str]]] = None, generator: bool = False, - timeout=0, + timeout=USE_CONFIG_TIMEOUT, ) -> str: return self.ask(msg, timeout=self.get_timeout(timeout)) - async def _achat_completion(self, messages: list[dict], timeout=0): + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): pass - async def acompletion(self, messages: list[dict], timeout=0): + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): """dummy implementation of abstract method in base""" return [] - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: pass - async def acompletion_text(self, messages: list[dict], stream=False, timeout=0) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout=USE_CONFIG_TIMEOUT) -> str: """dummy implementation of abstract method in base""" return "" diff --git a/metagpt/provider/ollama_api.py b/metagpt/provider/ollama_api.py index 723abb574..2913eb1dd 100644 --- a/metagpt/provider/ollama_api.py +++ b/metagpt/provider/ollama_api.py @@ -5,6 +5,7 @@ import json from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.general_api_requestor import GeneralAPIRequestor @@ -49,7 +50,7 @@ class OllamaLLM(BaseLLM): chunk = chunk.decode(encoding) return json.loads(chunk) - async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> dict: + async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> dict: resp, _, _ = await self.client.arequest( method=self.http_method, url=self.suffix_url, @@ -61,10 +62,10 @@ class OllamaLLM(BaseLLM): self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=0) -> dict: + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: stream_resp, _, _ = await self.client.arequest( method=self.http_method, url=self.suffix_url, diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 5b4abae61..10b7749d6 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -25,6 +25,7 @@ from tenacity import ( ) from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_llm import BaseLLM from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA @@ -79,7 +80,7 @@ class OpenAILLM(BaseLLM): return params - async def _achat_completion_stream(self, messages: list[dict], timeout=0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> str: response: AsyncStream[ChatCompletionChunk] = await self.aclient.chat.completions.create( **self._cons_kwargs(messages, timeout=self.get_timeout(timeout)), stream=True ) @@ -109,7 +110,7 @@ class OpenAILLM(BaseLLM): self._update_costs(usage) return full_reply_content - def _cons_kwargs(self, messages: list[dict], timeout=0, **extra_kwargs) -> dict: + def _cons_kwargs(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT, **extra_kwargs) -> dict: kwargs = { "messages": messages, "max_tokens": self._get_max_tokens(messages), @@ -123,13 +124,13 @@ class OpenAILLM(BaseLLM): kwargs.update(extra_kwargs) return kwargs - async def _achat_completion(self, messages: list[dict], timeout=0) -> ChatCompletion: + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> ChatCompletion: kwargs = self._cons_kwargs(messages, timeout=self.get_timeout(timeout)) rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs) self._update_costs(rsp.usage) return rsp - async def acompletion(self, messages: list[dict], timeout=0) -> ChatCompletion: + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> ChatCompletion: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) @retry( @@ -139,7 +140,7 @@ class OpenAILLM(BaseLLM): retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream=False, timeout=0) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout=USE_CONFIG_TIMEOUT) -> str: """when streaming, print each token in place.""" if stream: return await self._achat_completion_stream(messages, timeout=timeout) @@ -148,7 +149,7 @@ class OpenAILLM(BaseLLM): return self.get_choice_text(rsp) async def _achat_completion_function( - self, messages: list[dict], timeout: int = 0, **chat_configs + self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT, **chat_configs ) -> ChatCompletion: messages = process_message(messages) kwargs = self._cons_kwargs(messages=messages, timeout=self.get_timeout(timeout), **chat_configs) @@ -156,7 +157,7 @@ class OpenAILLM(BaseLLM): self._update_costs(rsp.usage) return rsp - async def aask_code(self, messages: list[dict], timeout: int = 0, **kwargs) -> dict: + async def aask_code(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT, **kwargs) -> dict: """Use function of tools to ask a code. Note: Keep kwargs consistent with https://platform.openai.com/docs/api-reference/chat/create diff --git a/metagpt/provider/qianfan_api.py b/metagpt/provider/qianfan_api.py index 7e0bf009e..3d78c8bfc 100644 --- a/metagpt/provider/qianfan_api.py +++ b/metagpt/provider/qianfan_api.py @@ -9,6 +9,7 @@ from qianfan import ChatCompletion from qianfan.resources.typing import JsonBody from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -107,15 +108,15 @@ class QianFanLLM(BaseLLM): self._update_costs(resp.body.get("usage", {})) return resp.body - async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> JsonBody: + async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> JsonBody: resp = await self.aclient.ado(**self._const_kwargs(messages=messages, stream=False)) self._update_costs(resp.body.get("usage", {})) return resp.body - async def acompletion(self, messages: list[dict], timeout: int = 0) -> JsonBody: + async def acompletion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> JsonBody: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: resp = await self.aclient.ado(**self._const_kwargs(messages=messages, stream=True)) collected_content = [] usage = {} diff --git a/metagpt/provider/spark_api.py b/metagpt/provider/spark_api.py index 0f450ccf5..594267259 100644 --- a/metagpt/provider/spark_api.py +++ b/metagpt/provider/spark_api.py @@ -17,6 +17,7 @@ from wsgiref.handlers import format_date_time import websocket # 使用websocket_client from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import logger from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -31,19 +32,19 @@ class SparkLLM(BaseLLM): def get_choice_text(self, rsp: dict) -> str: return rsp["payload"]["choices"]["text"][-1]["content"] - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: pass - async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 0) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = USE_CONFIG_TIMEOUT) -> str: # 不支持 # logger.warning("当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。") w = GetMessageFromWeb(messages, self.config) return w.run() - async def _achat_completion(self, messages: list[dict], timeout=0): + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): pass - async def acompletion(self, messages: list[dict], timeout=0): + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): # 不支持异步 w = GetMessageFromWeb(messages, self.config) return w.run() diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 8c5284770..2db441991 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -8,6 +8,7 @@ from typing import Optional from zhipuai.types.chat.chat_completion import Completion from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -45,22 +46,22 @@ class ZhiPuAILLM(BaseLLM): kwargs = {"model": self.model, "messages": messages, "stream": stream, "temperature": 0.3} return kwargs - def completion(self, messages: list[dict], timeout=0) -> dict: + def completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict: resp: Completion = self.llm.chat.completions.create(**self._const_kwargs(messages)) usage = resp.usage.model_dump() self._update_costs(usage) return resp.model_dump() - async def _achat_completion(self, messages: list[dict], timeout=0) -> dict: + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict: resp = await self.llm.acreate(**self._const_kwargs(messages)) usage = resp.get("usage", {}) self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=0) -> dict: + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout=0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> str: response = await self.llm.acreate_stream(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = {} From 8cda8c6f1b0f35e90ecca9467a3ee346b441860b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 14:54:14 +0800 Subject: [PATCH 16/21] fixbug: reset ocr_receipt.py --- examples/di/ocr_receipt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/di/ocr_receipt.py b/examples/di/ocr_receipt.py index 8b48be4f1..6b969ee99 100644 --- a/examples/di/ocr_receipt.py +++ b/examples/di/ocr_receipt.py @@ -6,7 +6,9 @@ async def main(): image_path = "image.jpg" language = "English" requirement = f"""This is a {language} receipt image. - Your goal is to perform OCR on images using PaddleOCR, then extract the total amount from ocr text results, and finally save as table. Image path: {image_path}. + Your goal is to perform OCR on images using PaddleOCR, output text content from the OCR results and discard + coordinates and confidence levels, then recognize the total amount from ocr text content, and finally save as table. + Image path: {image_path}. NOTE: The environments for Paddle and PaddleOCR are all ready and has been fully installed.""" di = DataInterpreter() From 587c13489320c62c8958f57bea54842e5a993614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 14:55:00 +0800 Subject: [PATCH 17/21] fixbug: reset ocr_receipt.py --- examples/di/ocr_receipt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/di/ocr_receipt.py b/examples/di/ocr_receipt.py index 6b969ee99..af54d519b 100644 --- a/examples/di/ocr_receipt.py +++ b/examples/di/ocr_receipt.py @@ -8,7 +8,7 @@ async def main(): requirement = f"""This is a {language} receipt image. Your goal is to perform OCR on images using PaddleOCR, output text content from the OCR results and discard coordinates and confidence levels, then recognize the total amount from ocr text content, and finally save as table. - Image path: {image_path}. + Image path: {image_path}. NOTE: The environments for Paddle and PaddleOCR are all ready and has been fully installed.""" di = DataInterpreter() From 9cfcfb1ea8bae086ed4fc5fe9d8365038c95e89e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 15:14:14 +0800 Subject: [PATCH 18/21] feat: use --gitfile --- .gitignore | 2 +- metagpt/utils/tree.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6bc67fa61..1542bbb98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ ### Python template # Byte-compiled / optimized / DLL files -__pycache__/ +__pycache__ *.py[cod] *$py.class diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index fbf085e48..bd7922290 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -130,7 +130,7 @@ def _add_line(rows: List[str]) -> List[str]: def _execute_tree(root: Path, gitignore: str | Path) -> str: - args = ["--gitignore", str(gitignore)] if gitignore else [] + args = ["--gitfile", str(gitignore)] if gitignore else [] try: result = subprocess.run(["tree"] + args + [str(root)], capture_output=True, text=True, check=True) if result.returncode != 0: From 12551ab60cc9ee0e59cf31a1a29c24e13d9e7abd Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 21 Mar 2024 21:57:59 +0800 Subject: [PATCH 19/21] fix state value extract for https://github.com/geekan/MetaGPT/issues/1067 --- metagpt/utils/repair_llm_raw_output.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index b8756e8c6..17e095c5f 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -340,7 +340,9 @@ def extract_state_value_from_output(content: str) -> str: content (str): llm's output from `Role._think` """ content = content.strip() # deal the output cases like " 0", "0\n" and so on. - pattern = r"([0-9])" # TODO find the number using a more proper method not just extract from content using pattern + pattern = ( + r"(? 0 else "-1" From e88aa63bf0f49e3e90a33bf818a91f45ba950752 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Fri, 22 Mar 2024 10:28:11 +0800 Subject: [PATCH 20/21] make the rag module optional --- requirements.txt | 8 -------- setup.py | 11 ++++++++++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 83565278b..a447eef13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,14 +10,6 @@ typer==0.9.0 # godot==0.1.1 # google_api_python_client==2.93.0 # Used by search_engine.py lancedb==0.4.0 -llama-index-core==0.10.15 -llama-index-embeddings-azure-openai==0.1.6 -llama-index-embeddings-openai==0.1.5 -llama-index-llms-azure-openai==0.1.4 -llama-index-readers-file==0.1.4 -llama-index-retrievers-bm25==0.1.3 -llama-index-vector-stores-faiss==0.1.1 -chromadb==0.4.23 loguru==0.6.0 meilisearch==0.21.0 numpy==1.24.3 diff --git a/setup.py b/setup.py index df9bedc9b..f834b4c44 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,16 @@ extras_require = { "search-google": ["google-api-python-client==2.94.0"], "search-ddg": ["duckduckgo-search~=4.1.1"], "ocr": ["paddlepaddle==2.4.2", "paddleocr>=2.0.1", "tabulate==0.9.0"], + "rag": [ + "llama-index-core==0.10.15", + "llama-index-embeddings-azure-openai==0.1.6", + "llama-index-embeddings-openai==0.1.5", + "llama-index-llms-azure-openai==0.1.4", + "llama-index-readers-file==0.1.4", + "llama-index-retrievers-bm25==0.1.3", + "llama-index-vector-stores-faiss==0.1.1", + "chromadb==0.4.23", + ], } extras_require["test"] = [ @@ -42,7 +52,6 @@ extras_require["test"] = [ "connexion[uvicorn]~=3.0.5", "azure-cognitiveservices-speech~=1.31.0", "aioboto3~=11.3.0", - "chromadb==0.4.23", "gradio==3.0.0", "grpcio-status==1.48.2", "pylint==3.0.3", From aea06852bad093aa0bb1f68dacba7d5384d9c429 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Fri, 22 Mar 2024 11:15:31 +0800 Subject: [PATCH 21/21] make the rag module optional --- metagpt/environment/__init__.py | 3 +-- metagpt/roles/__init__.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/metagpt/environment/__init__.py b/metagpt/environment/__init__.py index 692672fa7..28981f2f8 100644 --- a/metagpt/environment/__init__.py +++ b/metagpt/environment/__init__.py @@ -4,10 +4,9 @@ from metagpt.environment.base_env import Environment from metagpt.environment.android_env.android_env import AndroidEnv -from metagpt.environment.mincraft_env.mincraft_env import MincraftExtEnv from metagpt.environment.werewolf_env.werewolf_env import WerewolfEnv from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv from metagpt.environment.software_env.software_env import SoftwareEnv -__all__ = ["AndroidEnv", "MincraftExtEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"] +__all__ = ["AndroidEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"] diff --git a/metagpt/roles/__init__.py b/metagpt/roles/__init__.py index f033a5dfa..08a0406b3 100644 --- a/metagpt/roles/__init__.py +++ b/metagpt/roles/__init__.py @@ -14,7 +14,6 @@ from metagpt.roles.engineer import Engineer from metagpt.roles.qa_engineer import QaEngineer from metagpt.roles.searcher import Searcher from metagpt.roles.sales import Sales -from metagpt.roles.customer_service import CustomerService __all__ = [ @@ -26,5 +25,4 @@ __all__ = [ "QaEngineer", "Searcher", "Sales", - "CustomerService", ]