mirror of
https://github.com/FoundationAgents/MetaGPT.git
synced 2026-05-24 14:15:17 +02:00
Merge branch 'main' into feat_memory
This commit is contained in:
better629
GitHub
commit
d7968fdc20
160 changed files with 2194 additions and 2096 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -1,7 +1,7 @@
|
|||
### Python template
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
__pycache__
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
|
|
|
|||
86
README.md
86
README.md
|
|
@ -26,7 +26,7 @@ # MetaGPT: The Multi-Agent Framework
|
|||
</p>
|
||||
|
||||
## News
|
||||
🚀 March. 01, 2024: Our Data Interpreter paper is on arxiv. Find all design and benchmark details [here](https://arxiv.org/abs/2402.18679)!
|
||||
🚀 Mar. 14, 2024: Our **Data Interpreter** paper is on [arxiv](https://arxiv.org/abs/2402.18679). Check the [example](https://docs.deepwisdom.ai/main/en/DataInterpreter/) and [code](https://github.com/geekan/MetaGPT/tree/main/examples/di)!
|
||||
|
||||
🚀 Feb. 08, 2024: [v0.7.0](https://github.com/geekan/MetaGPT/releases/tag/v0.7.0) released, supporting assigning different LLMs to different Roles. We also introduced [Data Interpreter](https://github.com/geekan/MetaGPT/blob/main/examples/di/README.md), a powerful agent capable of solving a wide range of real-world problems.
|
||||
|
||||
|
|
@ -55,21 +55,30 @@ ## Software Company as Multi-Agent System
|
|||
|
||||
<p align="center">Software Company Multi-Agent Schematic (Gradually Implementing)</p>
|
||||
|
||||
## Install
|
||||
## Get Started
|
||||
|
||||
### Pip installation
|
||||
### Installation
|
||||
|
||||
> Ensure that Python 3.9+ is installed on your system. You can check this by using: `python --version`.
|
||||
> You can use conda like this: `conda create -n metagpt python=3.9 && conda activate metagpt`
|
||||
|
||||
```bash
|
||||
pip install metagpt
|
||||
# https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html
|
||||
metagpt --init-config # it will create ~/.metagpt/config2.yaml, just modify it to your needs
|
||||
pip install --upgrade metagpt
|
||||
# or `pip install --upgrade git+https://github.com/geekan/MetaGPT.git`
|
||||
# or `git clone https://github.com/geekan/MetaGPT && cd MetaGPT && pip install --upgrade -e .`
|
||||
```
|
||||
|
||||
For detailed installation guidance, please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version)
|
||||
or [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker)
|
||||
|
||||
### Configuration
|
||||
|
||||
You can init the config of MetaGPT by running the following command, or manually create `~/.metagpt/config2.yaml` file:
|
||||
```bash
|
||||
# Check https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html for more details
|
||||
metagpt --init-config # it will create ~/.metagpt/config2.yaml, just modify it to your needs
|
||||
```
|
||||
|
||||
You can configure `~/.metagpt/config2.yaml` according to the [example](https://github.com/geekan/MetaGPT/blob/main/config/config2.example.yaml) and [doc](https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html):
|
||||
|
||||
```yaml
|
||||
|
|
@ -82,13 +91,13 @@ ### Configuration
|
|||
|
||||
### Usage
|
||||
|
||||
After installation, you can use it as CLI
|
||||
After installation, you can use MetaGPT at CLI
|
||||
|
||||
```bash
|
||||
metagpt "Create a 2048 game" # this will create a repo in ./workspace
|
||||
```
|
||||
|
||||
or you can use it as library
|
||||
or use it as library
|
||||
|
||||
```python
|
||||
from metagpt.software_company import generate_repo, ProjectRepo
|
||||
|
|
@ -96,47 +105,19 @@ ### Usage
|
|||
print(repo) # it will print the repo structure with files
|
||||
```
|
||||
|
||||
detail installation please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version)
|
||||
or [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker)
|
||||
You can also use its [Data Interpreter](https://github.com/geekan/MetaGPT/tree/main/examples/di)
|
||||
|
||||
### Docker installation
|
||||
<details><summary><strong>⏬ Step 1: Download metagpt image and prepare config2.yaml </strong><i>:: click to expand ::</i></summary>
|
||||
<div>
|
||||
```python
|
||||
import asyncio
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
|
||||
```bash
|
||||
docker pull metagpt/metagpt:latest
|
||||
mkdir -p /opt/metagpt/{config,workspace}
|
||||
docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml
|
||||
vim /opt/metagpt/config/config2.yaml # Change the config
|
||||
async def main():
|
||||
di = DataInterpreter()
|
||||
await di.run("Run data analysis on sklearn Iris dataset, include a plot")
|
||||
|
||||
asyncio.run(main()) # or await main() in a jupyter notebook setting
|
||||
```
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details><summary><strong>⏬ Step 2: Run metagpt container </strong><i>:: click to expand ::</i></summary>
|
||||
<div>
|
||||
|
||||
```bash
|
||||
docker run --name metagpt -d \
|
||||
--privileged \
|
||||
-v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
|
||||
-v /opt/metagpt/workspace:/app/metagpt/workspace \
|
||||
metagpt/metagpt:latest
|
||||
```
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details><summary><strong>⏬ Step 3: Use metagpt </strong><i>:: click to expand ::</i></summary>
|
||||
<div>
|
||||
|
||||
```bash
|
||||
docker exec -it metagpt /bin/bash
|
||||
$ metagpt "Create a 2048 game" # this will create a repo in ./workspace
|
||||
```
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
### QuickStart & Demo Video
|
||||
- Try it on [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT)
|
||||
|
|
@ -156,6 +137,7 @@ ## Tutorial
|
|||
- 🧑💻 Contribution
|
||||
- [Develop Roadmap](docs/ROADMAP.md)
|
||||
- 🔖 Use Cases
|
||||
- [Data Interpreter](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/intro.html)
|
||||
- [Debate](https://docs.deepwisdom.ai/main/en/guide/use_cases/multi_agent/debate.html)
|
||||
- [Researcher](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/researcher.html)
|
||||
- [Recepit Assistant](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/receipt_assistant.html)
|
||||
|
|
@ -179,7 +161,9 @@ ### Contact Information
|
|||
|
||||
## Citation
|
||||
|
||||
For now, cite the [arXiv paper](https://arxiv.org/abs/2308.00352):
|
||||
To stay updated with the latest research and development, follow [@MetaGPT_](https://twitter.com/MetaGPT_) on Twitter.
|
||||
|
||||
To cite [MetaGPT](https://arxiv.org/abs/2308.00352) or [Data Interpreter](https://arxiv.org/abs/2402.18679) in publications, please use the following BibTeX entries.
|
||||
|
||||
```bibtex
|
||||
@misc{hong2023metagpt,
|
||||
|
|
@ -190,4 +174,14 @@ ## Citation
|
|||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
@misc{hong2024data,
|
||||
title={Data Interpreter: An LLM Agent For Data Science},
|
||||
author={Sirui Hong and Yizhang Lin and Bang Liu and Bangbang Liu and Binhao Wu and Danyang Li and Jiaqi Chen and Jiayi Zhang and Jinlin Wang and Li Zhang and Lingyao Zhang and Min Yang and Mingchen Zhuge and Taicheng Guo and Tuo Zhou and Wei Tao and Wenyi Wang and Xiangru Tang and Xiangtao Lu and Xiawu Zheng and Xinbing Liang and Yaying Fei and Yuheng Cheng and Zongze Xu and Chenglin Wu},
|
||||
year={2024},
|
||||
eprint={2402.18679},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ ## Supported Versions
|
|||
|
||||
| Version | Supported |
|
||||
|---------|--------------------|
|
||||
| 7.x | :x: |
|
||||
| 6.x | :x: |
|
||||
| < 6.x | :x: |
|
||||
| 0.7.x | :x: |
|
||||
| 0.6.x | :x: |
|
||||
| < 0.6.x | :x: |
|
||||
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ llm:
|
|||
api_key: "YOUR_API_KEY"
|
||||
model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
|
||||
proxy: "YOUR_PROXY" # for LLM API requests
|
||||
# timeout: 600 # Optional. If set to 0, default value is 300.
|
||||
pricing_plan: "" # Optional. If invalid, it will be automatically filled in with the value of the `model`.
|
||||
# Azure-exclusive pricing plan mappings：
|
||||
# - gpt-3.5-turbo 4k: "gpt-3.5-turbo-1106"
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ ### 联系信息
|
|||
|
||||
## 引用
|
||||
|
||||
引用 [arXiv paper](https://arxiv.org/abs/2308.00352):
|
||||
如果您在研究论文中使用 MetaGPT 或 Data Interpreter，请引用我们的工作：
|
||||
|
||||
```bibtex
|
||||
@misc{hong2023metagpt,
|
||||
|
|
@ -127,4 +127,12 @@ ## 引用
|
|||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
@misc{hong2024data,
|
||||
title={Data Interpreter: An LLM Agent For Data Science},
|
||||
author={Sirui Hong and Yizhang Lin and Bang Liu and Bangbang Liu and Binhao Wu and Danyang Li and Jiaqi Chen and Jiayi Zhang and Jinlin Wang and Li Zhang and Lingyao Zhang and Min Yang and Mingchen Zhuge and Taicheng Guo and Tuo Zhou and Wei Tao and Wenyi Wang and Xiangru Tang and Xiangtao Lu and Xiawu Zheng and Xinbing Liang and Yaying Fei and Yuheng Cheng and Zongze Xu and Chenglin Wu},
|
||||
year={2024},
|
||||
eprint={2402.18679},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ ## クイックスタート
|
|||
|
||||
## 引用
|
||||
|
||||
現時点では、[arXiv 論文](https://arxiv.org/abs/2308.00352)を引用してください:
|
||||
研究論文でMetaGPTやData Interpreterを使用する場合は、以下のように当社の作業を引用してください：
|
||||
|
||||
```bibtex
|
||||
@misc{hong2023metagpt,
|
||||
|
|
@ -306,6 +306,14 @@ ## 引用
|
|||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
@misc{hong2024data,
|
||||
title={Data Interpreter: An LLM Agent For Data Science},
|
||||
author={Sirui Hong and Yizhang Lin and Bang Liu and Bangbang Liu and Binhao Wu and Danyang Li and Jiaqi Chen and Jiayi Zhang and Jinlin Wang and Li Zhang and Lingyao Zhang and Min Yang and Mingchen Zhuge and Taicheng Guo and Tuo Zhou and Wei Tao and Wenyi Wang and Xiangru Tang and Xiangtao Lu and Xiawu Zheng and Xinbing Liang and Yaying Fei and Yuheng Cheng and Zongze Xu and Chenglin Wu},
|
||||
year={2024},
|
||||
eprint={2402.18679},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
```
|
||||
|
||||
## お問い合わせ先
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Author: garylin2099
|
|||
@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `send_to`
|
||||
value of the `Message` object; modify the argument type of `get_by_actions`.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import platform
|
||||
from typing import Any
|
||||
|
|
@ -105,4 +106,4 @@ def main(idea: str, investment: float = 3.0, n_round: int = 10):
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(main)
|
||||
fire.Fire(main) # run as python debate.py --idea="TOPIC" --investment=3.0 --n_round=5
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# Data Interpreter (DI)
|
||||
|
||||
## What is Data Interpreter
|
||||
Data Interpreter is an agent who solves problems through codes. It understands user requirements, makes plans, writes codes for execution, and uses tools if necessary. These capabilities enable it to tackle a wide range of scenarios, please check out the examples below.
|
||||
Data Interpreter is an agent who solves data-related problems through codes. It understands user requirements, makes plans, writes codes for execution, and uses tools if necessary. These capabilities enable it to tackle a wide range of scenarios, please check out the examples below. For overall design and technical details, please see our [paper](https://arxiv.org/abs/2402.18679).
|
||||
|
||||
## Example List
|
||||
- Data visualization
|
||||
|
|
@ -12,7 +12,9 @@ ## Example List
|
|||
- Tool usage: web page imitation
|
||||
- Tool usage: web crawling
|
||||
- Tool usage: text2image
|
||||
- Tool usage: email summarization and response
|
||||
- Tool usage: email summarization and response\
|
||||
- More on the way!
|
||||
|
||||
Please see [here](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/intro.html) for detailed explanation.
|
||||
Please see the [docs](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/intro.html) for more explanation.
|
||||
|
||||
We are continuously releasing codes, stay tuned!
|
||||
|
|
|
|||
21
examples/di/arxiv_reader.py
Normal file
21
examples/di/arxiv_reader.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
|
||||
|
||||
async def main():
|
||||
template = "https://arxiv.org/list/{tag}/pastweek?skip=0&show=300"
|
||||
tags = ["cs.ai", "cs.cl", "cs.lg", "cs.se"]
|
||||
urls = [template.format(tag=tag) for tag in tags]
|
||||
prompt = f"""This is a collection of arxiv urls: '{urls}' .
|
||||
Record each article, remove duplicates by title (they may have multiple tags), filter out papers related to
|
||||
large language model / agent / llm, print top 100 and visualize the word count of the titles"""
|
||||
di = DataInterpreter(react_mode="react", tools=["scrape_web_playwright"])
|
||||
|
||||
await di.run(prompt)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
asyncio.run(main())
|
||||
|
|
@ -7,13 +7,31 @@
|
|||
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
|
||||
PAPER_LIST_REQ = """"
|
||||
Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/,
|
||||
and save it to a csv file. paper title must include `multiagent` or `large language model`. *notice: print key variables*
|
||||
"""
|
||||
|
||||
ECOMMERCE_REQ = """
|
||||
Get products data from website https://scrapeme.live/shop/ and save it as a csv file.
|
||||
**Notice: Firstly parse the web page encoding and the text HTML structure;
|
||||
The first page product name, price, product URL, and image URL must be saved in the csv;**
|
||||
"""
|
||||
|
||||
NEWS_36KR_REQ = """从36kr创投平台https://pitchhub.36kr.com/financing-flash 所有初创企业融资的信息, **注意: 这是一个中文网站**;
|
||||
下面是一个大致流程, 你会根据每一步的运行结果对当前计划中的任务做出适当调整:
|
||||
1. 爬取并本地保存html结构;
|
||||
2. 直接打印第7个*`快讯`*关键词后2000个字符的html内容, 作为*快讯的html内容示例*;
|
||||
3. 反思*快讯的html内容示例*中的规律, 设计正则匹配表达式来获取*`快讯`*的标题、链接、时间;
|
||||
4. 筛选最近3天的初创企业融资*`快讯`*, 以list[dict]形式打印前5个。
|
||||
5. 将全部结果存在本地csv中
|
||||
"""
|
||||
|
||||
|
||||
async def main():
|
||||
prompt = """Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/,
|
||||
and save it to a csv file. paper title must include `multiagent` or `large language model`. *notice: print key variables*"""
|
||||
di = DataInterpreter(use_tools=True)
|
||||
di = DataInterpreter(tools=["scrape_web_playwright"])
|
||||
|
||||
await di.run(prompt)
|
||||
await di.run(ECOMMERCE_REQ)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
36
examples/di/custom_tool.py
Normal file
36
examples/di/custom_tool.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
@Time : 2024/3/22 10:54
|
||||
@Author : alexanderwu
|
||||
@File : custom_tool.py
|
||||
"""
|
||||
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
from metagpt.tools.tool_registry import register_tool
|
||||
|
||||
|
||||
@register_tool()
|
||||
def magic_function(arg1: str, arg2: int) -> dict:
|
||||
"""
|
||||
The magic function that does something.
|
||||
|
||||
Args:
|
||||
arg1 (str): ...
|
||||
arg2 (int): ...
|
||||
|
||||
Returns:
|
||||
dict: ...
|
||||
"""
|
||||
return {"arg1": arg1 * 3, "arg2": arg2 * 5}
|
||||
|
||||
|
||||
async def main():
|
||||
di = DataInterpreter(tools=["magic_function"])
|
||||
await di.run("Just call the magic function with arg1 'A' and arg2 2. Tell me the result.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
asyncio.run(main())
|
||||
|
|
@ -1,14 +1,17 @@
|
|||
import asyncio
|
||||
|
||||
from metagpt.logs import logger
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
from metagpt.utils.recovery_util import save_history
|
||||
|
||||
|
||||
async def main(requirement: str = ""):
|
||||
di = DataInterpreter(use_tools=False)
|
||||
await di.run(requirement)
|
||||
di = DataInterpreter()
|
||||
rsp = await di.run(requirement)
|
||||
logger.info(rsp)
|
||||
save_history(role=di)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
requirement = "Run data analysis on sklearn Iris dataset, include a plot"
|
||||
|
||||
asyncio.run(main(requirement))
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ async def main():
|
|||
Firstly, Please help me fetch the latest 5 senders and full letter contents.
|
||||
Then, summarize each of the 5 emails into one sentence (you can do this by yourself, no need to import other models to do this) and output them in a markdown format."""
|
||||
|
||||
di = DataInterpreter(use_tools=True)
|
||||
di = DataInterpreter()
|
||||
|
||||
await di.run(prompt)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,10 +12,9 @@ async def main():
|
|||
web_url = "https://pytorch.org/"
|
||||
prompt = f"""This is a URL of webpage: '{web_url}' .
|
||||
Firstly, utilize Selenium and WebDriver for rendering.
|
||||
Secondly, convert image to a webpage including HTML, CSS and JS in one go.
|
||||
Finally, save webpage in a text file.
|
||||
Secondly, convert image to a webpage including HTML, CSS and JS in one go.
|
||||
Note: All required dependencies and environments have been fully installed and configured."""
|
||||
di = DataInterpreter(use_tools=True)
|
||||
di = DataInterpreter(tools=["GPTvGenerator"])
|
||||
|
||||
await di.run(prompt)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,21 @@ import fire
|
|||
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
|
||||
WINE_REQ = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy."
|
||||
|
||||
async def main(auto_run: bool = True):
|
||||
requirement = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy."
|
||||
di = DataInterpreter(auto_run=auto_run)
|
||||
await di.run(requirement)
|
||||
DATA_DIR = "path/to/your/data"
|
||||
# sales_forecast data from https://www.kaggle.com/datasets/aslanahmedov/walmart-sales-forecast/data
|
||||
SALES_FORECAST_REQ = f"""Train a model to predict sales for each department in every store (split the last 40 weeks records as validation dataset, the others is train dataset), include plot total sales trends, print metric and plot scatter plots of
|
||||
groud truth and predictions on validation data. Dataset is {DATA_DIR}/train.csv, the metric is weighted mean absolute error (WMAE) for test data. Notice: *print* key variables to get more information for next task step.
|
||||
"""
|
||||
|
||||
REQUIREMENTS = {"wine": WINE_REQ, "sales_forecast": SALES_FORECAST_REQ}
|
||||
|
||||
|
||||
async def main(use_case: str = "wine"):
|
||||
mi = DataInterpreter()
|
||||
requirement = REQUIREMENTS[use_case]
|
||||
await mi.run(requirement)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
16
examples/di/machine_learning_with_tools.py
Normal file
16
examples/di/machine_learning_with_tools.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
import asyncio
|
||||
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
|
||||
|
||||
async def main(requirement: str):
|
||||
role = DataInterpreter(use_reflection=True, tools=["<all>"])
|
||||
await role.run(requirement)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
data_path = "your/path/to/titanic"
|
||||
train_path = f"{data_path}/split_train.csv"
|
||||
eval_path = f"{data_path}/split_eval.csv"
|
||||
requirement = f"This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '{train_path}', eval data path: '{eval_path}'."
|
||||
asyncio.run(main(requirement))
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
import asyncio
|
||||
|
||||
from metagpt.roles.di.ml_engineer import MLEngineer
|
||||
|
||||
|
||||
async def main(requirement: str):
|
||||
role = MLEngineer(auto_run=True, use_tools=True)
|
||||
await role.run(requirement)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
data_path = "your_path_to_icr/icr-identify-age-related-conditions"
|
||||
train_path = f"{data_path}/your_train_data.csv"
|
||||
eval_path = f"{data_path}/your_eval_data.csv"
|
||||
requirement = f"This is a medical dataset with over fifty anonymized health characteristics linked to three age-related conditions. Your goal is to predict whether a subject has or has not been diagnosed with one of these conditions.The target column is Class. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report f1 score on the eval data. Train data path: {train_path}, eval data path:{eval_path}."
|
||||
asyncio.run(main(requirement))
|
||||
|
|
@ -4,7 +4,7 @@ from metagpt.roles.di.data_interpreter import DataInterpreter
|
|||
|
||||
|
||||
async def main(requirement: str = ""):
|
||||
di = DataInterpreter(use_tools=False)
|
||||
di = DataInterpreter()
|
||||
await di.run(requirement)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from metagpt.roles.di.data_interpreter import DataInterpreter
|
|||
|
||||
|
||||
async def main(requirement: str = ""):
|
||||
di = DataInterpreter(use_tools=True, goal=requirement)
|
||||
di = DataInterpreter(tools=["SDEngine"])
|
||||
await di.run(requirement)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,11 @@ from metagpt.roles.di.data_interpreter import DataInterpreter
|
|||
|
||||
|
||||
async def main(requirement: str = ""):
|
||||
di = DataInterpreter(use_tools=False)
|
||||
di = DataInterpreter()
|
||||
await di.run(requirement)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
requirement = "Solve this math problem: The greatest common divisor of positive integers m and n is 6. The least common multiple of m and n is 126. What is the least possible value of m + n?"
|
||||
# answer: 60 (m = 18, n = 42)
|
||||
asyncio.run(main(requirement))
|
||||
|
|
|
|||
72
examples/reverse_engineering.py
Normal file
72
examples/reverse_engineering.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
|
||||
from metagpt.actions.rebuild_class_view import RebuildClassView
|
||||
from metagpt.actions.rebuild_sequence_view import RebuildSequenceView
|
||||
from metagpt.context import Context
|
||||
from metagpt.llm import LLM
|
||||
from metagpt.logs import logger
|
||||
from metagpt.utils.git_repository import GitRepository
|
||||
from metagpt.utils.project_repo import ProjectRepo
|
||||
|
||||
app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False)
|
||||
|
||||
|
||||
@app.command("", help="Python project reverse engineering.")
|
||||
def startup(
|
||||
project_root: str = typer.Argument(
|
||||
default="",
|
||||
help="Specify the root directory of the existing project for reverse engineering.",
|
||||
),
|
||||
output_dir: str = typer.Option(default="", help="Specify the output directory path for reverse engineering."),
|
||||
):
|
||||
package_root = Path(project_root)
|
||||
if not package_root.exists():
|
||||
raise FileNotFoundError(f"{project_root} not exists")
|
||||
if not _is_python_package_root(package_root):
|
||||
raise FileNotFoundError(f'There are no "*.py" files under "{project_root}".')
|
||||
init_file = package_root / "__init__.py" # used by pyreverse
|
||||
init_file_exists = init_file.exists()
|
||||
if not init_file_exists:
|
||||
init_file.touch()
|
||||
|
||||
if not output_dir:
|
||||
output_dir = package_root / "../reverse_engineering_output"
|
||||
logger.info(f"output dir:{output_dir}")
|
||||
try:
|
||||
asyncio.run(reverse_engineering(package_root, Path(output_dir)))
|
||||
finally:
|
||||
if not init_file_exists:
|
||||
init_file.unlink(missing_ok=True)
|
||||
tmp_dir = package_root / "__dot__"
|
||||
if tmp_dir.exists():
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def _is_python_package_root(package_root: Path) -> bool:
|
||||
for file_path in package_root.iterdir():
|
||||
if file_path.is_file():
|
||||
if file_path.suffix == ".py":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def reverse_engineering(package_root: Path, output_dir: Path):
|
||||
ctx = Context()
|
||||
ctx.git_repo = GitRepository(output_dir)
|
||||
ctx.repo = ProjectRepo(ctx.git_repo)
|
||||
action = RebuildClassView(name="ReverseEngineering", i_context=str(package_root), llm=LLM(), context=ctx)
|
||||
await action.run()
|
||||
|
||||
action = RebuildSequenceView(name="ReverseEngineering", llm=LLM(), context=ctx)
|
||||
await action.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
|
|
@ -4,21 +4,17 @@
|
|||
"""
|
||||
import asyncio
|
||||
|
||||
from metagpt.config2 import Config
|
||||
from metagpt.roles import Searcher
|
||||
from metagpt.tools.search_engine import SearchEngine, SearchEngineType
|
||||
from metagpt.tools.search_engine import SearchEngine
|
||||
|
||||
|
||||
async def main():
|
||||
question = "What are the most interesting human facts?"
|
||||
kwargs = {"api_key": "", "cse_id": "", "proxy": None}
|
||||
# Serper API
|
||||
# await Searcher(search_engine=SearchEngine(engine=SearchEngineType.SERPER_GOOGLE, **kwargs)).run(question)
|
||||
# SerpAPI
|
||||
# await Searcher(search_engine=SearchEngine(engine=SearchEngineType.SERPAPI_GOOGLE, **kwargs)).run(question)
|
||||
# Google API
|
||||
# await Searcher(search_engine=SearchEngine(engine=SearchEngineType.DIRECT_GOOGLE, **kwargs)).run(question)
|
||||
# DDG API
|
||||
await Searcher(search_engine=SearchEngine(engine=SearchEngineType.DUCK_DUCK_GO, **kwargs)).run(question)
|
||||
|
||||
search = Config.default().search
|
||||
kwargs = search.model_dump()
|
||||
await Searcher(search_engine=SearchEngine(engine=search.api_type, **kwargs)).run(question)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from metagpt.actions.write_prd import WritePRD
|
|||
from metagpt.actions.write_prd_review import WritePRDReview
|
||||
from metagpt.actions.write_test import WriteTest
|
||||
from metagpt.actions.di.execute_nb_code import ExecuteNbCode
|
||||
from metagpt.actions.di.write_analysis_code import WriteCodeWithoutTools, WriteCodeWithTools
|
||||
from metagpt.actions.di.write_analysis_code import WriteAnalysisCode
|
||||
from metagpt.actions.di.write_plan import WritePlan
|
||||
|
||||
|
||||
|
|
@ -46,8 +46,7 @@ class ActionType(Enum):
|
|||
WEB_BROWSE_AND_SUMMARIZE = WebBrowseAndSummarize
|
||||
CONDUCT_RESEARCH = ConductResearch
|
||||
EXECUTE_NB_CODE = ExecuteNbCode
|
||||
WRITE_CODE_WITHOUT_TOOLS = WriteCodeWithoutTools
|
||||
WRITE_CODE_WITH_TOOLS = WriteCodeWithTools
|
||||
WRITE_ANALYSIS_CODE = WriteAnalysisCode
|
||||
WRITE_PLAN = WritePlan
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from pydantic import BaseModel, Field, create_model, model_validator
|
|||
from tenacity import retry, stop_after_attempt, wait_random_exponential
|
||||
|
||||
from metagpt.actions.action_outcls_registry import register_action_outcls
|
||||
from metagpt.const import USE_CONFIG_TIMEOUT
|
||||
from metagpt.llm import BaseLLM
|
||||
from metagpt.logs import logger
|
||||
from metagpt.provider.postprocess.llm_output_postprocess import llm_output_postprocess
|
||||
|
|
@ -330,7 +331,7 @@ class ActionNode:
|
|||
|
||||
def compile_to(self, i: Dict, schema, kv_sep) -> str:
|
||||
if schema == "json":
|
||||
return json.dumps(i, indent=4)
|
||||
return json.dumps(i, indent=4, ensure_ascii=False)
|
||||
elif schema == "markdown":
|
||||
return dict_to_markdown(i, kv_sep=kv_sep)
|
||||
else:
|
||||
|
|
@ -339,10 +340,7 @@ class ActionNode:
|
|||
def tagging(self, text, schema, tag="") -> str:
|
||||
if not tag:
|
||||
return text
|
||||
if schema == "json":
|
||||
return f"[{tag}]\n" + text + f"\n[/{tag}]"
|
||||
else: # markdown
|
||||
return f"[{tag}]\n" + text + f"\n[/{tag}]"
|
||||
return f"[{tag}]\n{text}\n[/{tag}]"
|
||||
|
||||
def _compile_f(self, schema, mode, tag, format_func, kv_sep, exclude=None) -> str:
|
||||
nodes = self.to_dict(format_func=format_func, mode=mode, exclude=exclude)
|
||||
|
|
@ -374,7 +372,7 @@ class ActionNode:
|
|||
schema="markdown": 编译context, example(markdown), instruction(markdown), constraint, action
|
||||
"""
|
||||
if schema == "raw":
|
||||
return context + "\n\n## Actions\n" + LANGUAGE_CONSTRAINT + "\n" + self.instruction
|
||||
return f"{context}\n\n## Actions\n{LANGUAGE_CONSTRAINT}\n{self.instruction}"
|
||||
|
||||
### 直接使用 pydantic BaseModel 生成 instruction 与 example，仅限 JSON
|
||||
# child_class = self._create_children_class()
|
||||
|
|
@ -416,7 +414,7 @@ class ActionNode:
|
|||
images: Optional[Union[str, list[str]]] = None,
|
||||
system_msgs: Optional[list[str]] = None,
|
||||
schema="markdown", # compatible to original format
|
||||
timeout=3,
|
||||
timeout=USE_CONFIG_TIMEOUT,
|
||||
) -> (str, BaseModel):
|
||||
"""Use ActionOutput to wrap the output of aask"""
|
||||
content = await self.llm.aask(prompt, system_msgs, images=images, timeout=timeout)
|
||||
|
|
@ -448,7 +446,9 @@ class ActionNode:
|
|||
def set_context(self, context):
|
||||
self.set_recursive("context", context)
|
||||
|
||||
async def simple_fill(self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=3, exclude=None):
|
||||
async def simple_fill(
|
||||
self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=USE_CONFIG_TIMEOUT, exclude=None
|
||||
):
|
||||
prompt = self.compile(context=self.context, schema=schema, mode=mode, exclude=exclude)
|
||||
|
||||
if schema != "raw":
|
||||
|
|
@ -473,7 +473,7 @@ class ActionNode:
|
|||
mode="auto",
|
||||
strgy="simple",
|
||||
images: Optional[Union[str, list[str]]] = None,
|
||||
timeout=3,
|
||||
timeout=USE_CONFIG_TIMEOUT,
|
||||
exclude=[],
|
||||
):
|
||||
"""Fill the node(s) with mode.
|
||||
|
|
|
|||
|
|
@ -1,109 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from metagpt.actions.di.write_analysis_code import BaseWriteAnalysisCode
|
||||
from metagpt.logs import logger
|
||||
from metagpt.schema import Message
|
||||
from metagpt.utils.common import create_func_call_config
|
||||
|
||||
DEBUG_REFLECTION_EXAMPLE = '''
|
||||
Example 1:
|
||||
[previous impl]:
|
||||
```python
|
||||
def add(a: int, b: int) -> int:
|
||||
"""
|
||||
Given integers a and b, return the total value of a and b.
|
||||
"""
|
||||
return a - b
|
||||
```
|
||||
|
||||
[runtime Error]:
|
||||
Tested passed:
|
||||
|
||||
Tests failed:
|
||||
assert add(1, 2) == 3 # output: -1
|
||||
assert add(1, 2) == 4 # output: -1
|
||||
|
||||
[reflection on previous impl]:
|
||||
The implementation failed the test cases where the input integers are 1 and 2. The issue arises because the code does not add the two integers together, but instead subtracts the second integer from the first. To fix this issue, we should change the operator from `-` to `+` in the return statement. This will ensure that the function returns the correct output for the given input.
|
||||
|
||||
[improved impl]:
|
||||
```python
|
||||
def add(a: int, b: int) -> int:
|
||||
"""
|
||||
Given integers a and b, return the total value of a and b.
|
||||
"""
|
||||
return a + b
|
||||
```
|
||||
'''
|
||||
|
||||
REFLECTION_PROMPT = """
|
||||
Here is an example for you.
|
||||
{debug_example}
|
||||
[context]
|
||||
{context}
|
||||
|
||||
[previous impl]
|
||||
{code}
|
||||
[runtime Error]
|
||||
{runtime_result}
|
||||
|
||||
Analysis the error step by step, provide me improve method and code. Remember to follow [context] requirement. Don't forget write code for steps behind the error step.
|
||||
[reflection on previous impl]:
|
||||
xxx
|
||||
"""
|
||||
|
||||
CODE_REFLECTION = {
|
||||
"name": "execute_reflection_code",
|
||||
"description": "Execute reflection code.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"reflection": {
|
||||
"type": "string",
|
||||
"description": "Reflection on previous impl.",
|
||||
},
|
||||
"improved_impl": {
|
||||
"type": "string",
|
||||
"description": "Refined code after reflection.",
|
||||
},
|
||||
},
|
||||
"required": ["reflection", "improved_impl"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class DebugCode(BaseWriteAnalysisCode):
|
||||
async def run(
|
||||
self,
|
||||
context: list[Message] = None,
|
||||
code: str = "",
|
||||
runtime_result: str = "",
|
||||
) -> str:
|
||||
"""
|
||||
Execute the debugging process based on the provided context, code, and runtime_result.
|
||||
|
||||
Args:
|
||||
context (list[Message]): A list of Message objects representing the context.
|
||||
code (str): The code to be debugged.
|
||||
runtime_result (str): The result of the code execution.
|
||||
|
||||
Returns:
|
||||
str: The improved implementation based on the debugging process.
|
||||
"""
|
||||
|
||||
info = []
|
||||
reflection_prompt = REFLECTION_PROMPT.format(
|
||||
debug_example=DEBUG_REFLECTION_EXAMPLE,
|
||||
context=context,
|
||||
code=code,
|
||||
runtime_result=runtime_result,
|
||||
)
|
||||
system_prompt = "You are an AI Python assistant. You will be given your previous implementation code of a task, runtime error results, and a hint to change the implementation appropriately. Write your full implementation "
|
||||
info.append(Message(role="system", content=system_prompt))
|
||||
info.append(Message(role="user", content=reflection_prompt))
|
||||
|
||||
tool_config = create_func_call_config(CODE_REFLECTION)
|
||||
reflection = await self.llm.aask_code(messages=info, **tool_config)
|
||||
logger.info(f"reflection is {reflection}")
|
||||
|
||||
return {"code": reflection["improved_impl"]}
|
||||
|
|
@ -9,7 +9,6 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import base64
|
||||
import re
|
||||
import traceback
|
||||
from typing import Literal, Tuple
|
||||
|
||||
import nbformat
|
||||
|
|
@ -58,7 +57,23 @@ class ExecuteNbCode(Action):
|
|||
|
||||
async def terminate(self):
|
||||
"""kill NotebookClient"""
|
||||
await self.nb_client._async_cleanup_kernel()
|
||||
if self.nb_client.km is not None and await self.nb_client.km.is_alive():
|
||||
await self.nb_client.km.shutdown_kernel(now=True)
|
||||
await self.nb_client.km.cleanup_resources()
|
||||
|
||||
channels = [
|
||||
self.nb_client.kc.stdin_channel, # The channel for handling standard input to the kernel.
|
||||
self.nb_client.kc.hb_channel, # The channel for heartbeat communication between the kernel and client.
|
||||
self.nb_client.kc.control_channel, # The channel for controlling the kernel.
|
||||
]
|
||||
|
||||
# Stops all the running channels for this kernel
|
||||
for channel in channels:
|
||||
if channel.is_alive():
|
||||
channel.stop()
|
||||
|
||||
self.nb_client.kc = None
|
||||
self.nb_client.km = None
|
||||
|
||||
async def reset(self):
|
||||
"""reset NotebookClient"""
|
||||
|
|
@ -91,17 +106,17 @@ class ExecuteNbCode(Action):
|
|||
else:
|
||||
cell["outputs"].append(new_output(output_type="stream", name="stdout", text=str(output)))
|
||||
|
||||
def parse_outputs(self, outputs: list[str]) -> str:
|
||||
def parse_outputs(self, outputs: list[str], keep_len: int = 2000) -> Tuple[bool, str]:
|
||||
"""Parses the outputs received from notebook execution."""
|
||||
assert isinstance(outputs, list)
|
||||
parsed_output = ""
|
||||
|
||||
parsed_output, is_success = [], True
|
||||
for i, output in enumerate(outputs):
|
||||
output_text = ""
|
||||
if output["output_type"] == "stream" and not any(
|
||||
tag in output["text"]
|
||||
for tag in ["| INFO | metagpt", "| ERROR | metagpt", "| WARNING | metagpt", "DEBUG"]
|
||||
):
|
||||
parsed_output += output["text"]
|
||||
output_text = output["text"]
|
||||
elif output["output_type"] == "display_data":
|
||||
if "image/png" in output["data"]:
|
||||
self.show_bytes_figure(output["data"]["image/png"], self.interaction)
|
||||
|
|
@ -110,8 +125,22 @@ class ExecuteNbCode(Action):
|
|||
f"{i}th output['data'] from nbclient outputs dont have image/png, continue next output ..."
|
||||
)
|
||||
elif output["output_type"] == "execute_result":
|
||||
parsed_output += output["data"]["text/plain"]
|
||||
return parsed_output
|
||||
output_text = output["data"]["text/plain"]
|
||||
elif output["output_type"] == "error":
|
||||
output_text, is_success = "\n".join(output["traceback"]), False
|
||||
|
||||
# handle coroutines that are not executed asynchronously
|
||||
if output_text.strip().startswith("<coroutine object"):
|
||||
output_text = "Executed code failed, you need use key word 'await' to run a async code."
|
||||
is_success = False
|
||||
|
||||
output_text = remove_escape_and_color_codes(output_text)
|
||||
# The useful information of the exception is at the end,
|
||||
# the useful information of normal output is at the begining.
|
||||
output_text = output_text[:keep_len] if is_success else output_text[-keep_len:]
|
||||
|
||||
parsed_output.append(output_text)
|
||||
return is_success, ",".join(parsed_output)
|
||||
|
||||
def show_bytes_figure(self, image_base64: str, interaction_type: Literal["ipython", None]):
|
||||
image_bytes = base64.b64decode(image_base64)
|
||||
|
|
@ -145,7 +174,7 @@ class ExecuteNbCode(Action):
|
|||
"""
|
||||
try:
|
||||
await self.nb_client.async_execute_cell(cell, cell_index)
|
||||
return True, ""
|
||||
return self.parse_outputs(self.nb.cells[-1].outputs)
|
||||
except CellTimeoutError:
|
||||
assert self.nb_client.km is not None
|
||||
await self.nb_client.km.interrupt_kernel()
|
||||
|
|
@ -156,7 +185,7 @@ class ExecuteNbCode(Action):
|
|||
await self.reset()
|
||||
return False, "DeadKernelError"
|
||||
except Exception:
|
||||
return False, f"{traceback.format_exc()}"
|
||||
return self.parse_outputs(self.nb.cells[-1].outputs)
|
||||
|
||||
async def run(self, code: str, language: Literal["python", "markdown"] = "python") -> Tuple[str, bool]:
|
||||
"""
|
||||
|
|
@ -173,14 +202,7 @@ class ExecuteNbCode(Action):
|
|||
|
||||
# run code
|
||||
cell_index = len(self.nb.cells) - 1
|
||||
success, error_message = await self.run_cell(self.nb.cells[-1], cell_index)
|
||||
|
||||
if not success:
|
||||
return truncate(remove_escape_and_color_codes(error_message), is_success=success)
|
||||
|
||||
# code success
|
||||
outputs = self.parse_outputs(self.nb.cells[-1].outputs)
|
||||
outputs, success = truncate(remove_escape_and_color_codes(outputs), is_success=success)
|
||||
success, outputs = await self.run_cell(self.nb.cells[-1], cell_index)
|
||||
|
||||
if "!pip" in code:
|
||||
success = False
|
||||
|
|
@ -196,54 +218,39 @@ class ExecuteNbCode(Action):
|
|||
raise ValueError(f"Only support for language: python, markdown, but got {language}, ")
|
||||
|
||||
|
||||
def truncate(result: str, keep_len: int = 2000, is_success: bool = True):
|
||||
"""对于超出keep_len个字符的result: 执行失败的代码, 展示result后keep_len个字符; 执行成功的代码, 展示result前keep_len个字符。"""
|
||||
if is_success:
|
||||
desc = f"Executed code successfully. Truncated to show only first {keep_len} characters\n"
|
||||
else:
|
||||
desc = f"Executed code failed, please reflect the cause of bug and then debug. Truncated to show only last {keep_len} characters\n"
|
||||
|
||||
if result.strip().startswith("<coroutine object"):
|
||||
result = "Executed code failed, you need use key word 'await' to run a async code."
|
||||
return result, False
|
||||
|
||||
if len(result) > keep_len:
|
||||
result = result[-keep_len:] if not is_success else result[:keep_len]
|
||||
return desc + result, is_success
|
||||
|
||||
return result, is_success
|
||||
|
||||
|
||||
def remove_escape_and_color_codes(input_str: str):
|
||||
# 使用正则表达式去除转义字符和颜色代码
|
||||
# 使用正则表达式去除jupyter notebook输出结果中的转义字符和颜色代码
|
||||
# Use regular expressions to get rid of escape characters and color codes in jupyter notebook output.
|
||||
pattern = re.compile(r"\x1b\[[0-9;]*[mK]")
|
||||
result = pattern.sub("", input_str)
|
||||
return result
|
||||
|
||||
|
||||
def display_markdown(content: str):
|
||||
# 使用正则表达式逐个匹配代码块
|
||||
# Use regular expressions to match blocks of code one by one.
|
||||
matches = re.finditer(r"```(.+?)```", content, re.DOTALL)
|
||||
start_index = 0
|
||||
content_panels = []
|
||||
# 逐个打印匹配到的文本和代码
|
||||
# Set the text background color and text color.
|
||||
style = "black on white"
|
||||
# Print the matching text and code one by one.
|
||||
for match in matches:
|
||||
text_content = content[start_index : match.start()].strip()
|
||||
code_content = match.group(0).strip()[3:-3] # Remove triple backticks
|
||||
|
||||
if text_content:
|
||||
content_panels.append(Panel(Markdown(text_content), box=MINIMAL))
|
||||
content_panels.append(Panel(Markdown(text_content), style=style, box=MINIMAL))
|
||||
|
||||
if code_content:
|
||||
content_panels.append(Panel(Markdown(f"```{code_content}"), box=MINIMAL))
|
||||
content_panels.append(Panel(Markdown(f"```{code_content}"), style=style, box=MINIMAL))
|
||||
start_index = match.end()
|
||||
|
||||
# 打印剩余文本（如果有）
|
||||
# Print remaining text (if any).
|
||||
remaining_text = content[start_index:].strip()
|
||||
if remaining_text:
|
||||
content_panels.append(Panel(Markdown(remaining_text), box=MINIMAL))
|
||||
content_panels.append(Panel(Markdown(remaining_text), style=style, box=MINIMAL))
|
||||
|
||||
# 在Live模式中显示所有Panel
|
||||
# Display all panels in Live mode.
|
||||
with Live(auto_refresh=False, console=Console(), vertical_overflow="visible") as live:
|
||||
live.update(Group(*content_panels))
|
||||
live.refresh()
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Tuple
|
||||
|
||||
from metagpt.actions import Action
|
||||
from metagpt.actions.di.write_analysis_code import WriteCodeWithTools
|
||||
from metagpt.prompts.di.ml_action import (
|
||||
ML_GENERATE_CODE_PROMPT,
|
||||
ML_TOOL_USAGE_PROMPT,
|
||||
PRINT_DATA_COLUMNS,
|
||||
UPDATE_DATA_COLUMNS,
|
||||
)
|
||||
from metagpt.prompts.di.write_analysis_code import CODE_GENERATOR_WITH_TOOLS
|
||||
from metagpt.schema import Message, Plan
|
||||
from metagpt.utils.common import create_func_call_config, remove_comments
|
||||
|
||||
|
||||
class WriteCodeWithToolsML(WriteCodeWithTools):
|
||||
async def run(
|
||||
self,
|
||||
context: list[Message],
|
||||
plan: Plan = None,
|
||||
column_info: str = "",
|
||||
**kwargs,
|
||||
) -> Tuple[list[Message], str]:
|
||||
# prepare tool schemas and tool-type-specific instruction
|
||||
tool_schemas, tool_type_usage_prompt = await self._prepare_tools(plan=plan)
|
||||
|
||||
# ML-specific variables to be used in prompt
|
||||
finished_tasks = plan.get_finished_tasks()
|
||||
code_context = [remove_comments(task.code) for task in finished_tasks]
|
||||
code_context = "\n\n".join(code_context)
|
||||
|
||||
# prepare prompt depending on tool availability & LLM call
|
||||
if tool_schemas:
|
||||
prompt = ML_TOOL_USAGE_PROMPT.format(
|
||||
user_requirement=plan.goal,
|
||||
history_code=code_context,
|
||||
current_task=plan.current_task.instruction,
|
||||
column_info=column_info,
|
||||
tool_type_usage_prompt=tool_type_usage_prompt,
|
||||
tool_schemas=tool_schemas,
|
||||
)
|
||||
|
||||
else:
|
||||
prompt = ML_GENERATE_CODE_PROMPT.format(
|
||||
user_requirement=plan.goal,
|
||||
history_code=code_context,
|
||||
current_task=plan.current_task.instruction,
|
||||
column_info=column_info,
|
||||
tool_type_usage_prompt=tool_type_usage_prompt,
|
||||
)
|
||||
tool_config = create_func_call_config(CODE_GENERATOR_WITH_TOOLS)
|
||||
rsp = await self.llm.aask_code(prompt, **tool_config)
|
||||
|
||||
# Extra output to be used for potential debugging
|
||||
context = [Message(content=prompt, role="user")]
|
||||
|
||||
return context, rsp
|
||||
|
||||
|
||||
class UpdateDataColumns(Action):
|
||||
async def run(self, plan: Plan = None) -> dict:
|
||||
finished_tasks = plan.get_finished_tasks()
|
||||
code_context = [remove_comments(task.code) for task in finished_tasks]
|
||||
code_context = "\n\n".join(code_context)
|
||||
prompt = UPDATE_DATA_COLUMNS.format(history_code=code_context)
|
||||
tool_config = create_func_call_config(PRINT_DATA_COLUMNS)
|
||||
rsp = await self.llm.aask_code(prompt, **tool_config)
|
||||
return rsp
|
||||
|
|
@ -6,150 +6,68 @@
|
|||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Tuple
|
||||
import json
|
||||
|
||||
from metagpt.actions import Action
|
||||
from metagpt.logs import logger
|
||||
from metagpt.prompts.di.write_analysis_code import (
|
||||
CODE_GENERATOR_WITH_TOOLS,
|
||||
SELECT_FUNCTION_TOOLS,
|
||||
TOOL_RECOMMENDATION_PROMPT,
|
||||
TOOL_USAGE_PROMPT,
|
||||
CHECK_DATA_PROMPT,
|
||||
DEBUG_REFLECTION_EXAMPLE,
|
||||
INTERPRETER_SYSTEM_MSG,
|
||||
REFLECTION_PROMPT,
|
||||
REFLECTION_SYSTEM_MSG,
|
||||
STRUCTUAL_PROMPT,
|
||||
)
|
||||
from metagpt.schema import Message, Plan, SystemMessage
|
||||
from metagpt.tools import TOOL_REGISTRY
|
||||
from metagpt.tools.tool_registry import validate_tool_names
|
||||
from metagpt.utils.common import create_func_call_config
|
||||
from metagpt.schema import Message, Plan
|
||||
from metagpt.utils.common import CodeParser, remove_comments
|
||||
|
||||
|
||||
class BaseWriteAnalysisCode(Action):
|
||||
DEFAULT_SYSTEM_MSG: str = """You are Code Interpreter, a world-class programmer that can complete any goal by executing code. Strictly follow the plan and generate code step by step. Each step of the code will be executed on the user's machine, and the user will provide the code execution results to you.**Notice: The code for the next step depends on the code for the previous step. Must reuse variables in the lastest other code directly, dont creat it again, it is very import for you. Use !pip install in a standalone block to install missing packages.Usually the libraries you need are already installed.Dont check if packages already imported.**""" # prompt reference: https://github.com/KillianLucas/open-interpreter/blob/v0.1.4/interpreter/system_message.txt
|
||||
# REUSE_CODE_INSTRUCTION = """ATTENTION: DONT include codes from previous tasks in your current code block, include new codes only, DONT repeat codes!"""
|
||||
|
||||
def insert_system_message(self, context: list[Message], system_msg: str = None):
|
||||
system_msg = system_msg or self.DEFAULT_SYSTEM_MSG
|
||||
context.insert(0, SystemMessage(content=system_msg)) if context[0].role != "system" else None
|
||||
return context
|
||||
|
||||
async def run(self, context: list[Message], plan: Plan = None) -> dict:
|
||||
"""Run of a code writing action, used in data analysis or modeling
|
||||
|
||||
Args:
|
||||
context (list[Message]): Action output history, source action denoted by Message.cause_by
|
||||
plan (Plan, optional): Overall plan. Defaults to None.
|
||||
|
||||
Returns:
|
||||
dict: code result in the format of {"code": "print('hello world')", "language": "python"}
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class WriteCodeWithoutTools(BaseWriteAnalysisCode):
|
||||
"""Ask LLM to generate codes purely by itself without local user-defined tools"""
|
||||
|
||||
async def run(self, context: list[Message], plan: Plan = None, system_msg: str = None, **kwargs) -> dict:
|
||||
messages = self.insert_system_message(context, system_msg)
|
||||
rsp = await self.llm.aask_code(messages, **kwargs)
|
||||
return rsp
|
||||
|
||||
|
||||
class WriteCodeWithTools(BaseWriteAnalysisCode):
|
||||
"""Write code with help of local available tools. Choose tools first, then generate code to use the tools"""
|
||||
|
||||
# selected tools to choose from, listed by their names. An empty list means selection from all tools.
|
||||
selected_tools: list[str] = []
|
||||
|
||||
def _get_tools_by_type(self, tool_type: str) -> dict:
|
||||
"""
|
||||
Retreive tools by tool type from registry, but filtered by pre-selected tool list
|
||||
|
||||
Args:
|
||||
tool_type (str): Tool type to retrieve from the registry
|
||||
|
||||
Returns:
|
||||
dict: A dict of tool name to Tool object, representing available tools under the type
|
||||
"""
|
||||
candidate_tools = TOOL_REGISTRY.get_tools_by_type(tool_type)
|
||||
if self.selected_tools:
|
||||
candidate_tool_names = set(self.selected_tools) & candidate_tools.keys()
|
||||
candidate_tools = {tool_name: candidate_tools[tool_name] for tool_name in candidate_tool_names}
|
||||
return candidate_tools
|
||||
|
||||
async def _recommend_tool(
|
||||
self,
|
||||
task: str,
|
||||
available_tools: dict,
|
||||
) -> dict:
|
||||
"""
|
||||
Recommend tools for the specified task.
|
||||
|
||||
Args:
|
||||
task (str): the task to recommend tools for
|
||||
available_tools (dict): the available tools description
|
||||
|
||||
Returns:
|
||||
dict: schemas of recommended tools for the specified task
|
||||
"""
|
||||
prompt = TOOL_RECOMMENDATION_PROMPT.format(
|
||||
current_task=task,
|
||||
available_tools=available_tools,
|
||||
)
|
||||
tool_config = create_func_call_config(SELECT_FUNCTION_TOOLS)
|
||||
rsp = await self.llm.aask_code(prompt, **tool_config)
|
||||
recommend_tools = rsp["recommend_tools"]
|
||||
logger.info(f"Recommended tools: \n{recommend_tools}")
|
||||
|
||||
# Parses and validates the recommended tools, for LLM might hallucinate and recommend non-existing tools
|
||||
valid_tools = validate_tool_names(recommend_tools, return_tool_object=True)
|
||||
|
||||
tool_schemas = {tool.name: tool.schemas for tool in valid_tools}
|
||||
|
||||
return tool_schemas
|
||||
|
||||
async def _prepare_tools(self, plan: Plan) -> Tuple[dict, str]:
|
||||
"""Prepare tool schemas and usage instructions according to current task
|
||||
|
||||
Args:
|
||||
plan (Plan): The overall plan containing task information.
|
||||
|
||||
Returns:
|
||||
Tuple[dict, str]: A tool schemas ({tool_name: tool_schema_dict}) and a usage prompt for the type of tools selected
|
||||
"""
|
||||
# find tool type from task type through exact match, can extend to retrieval in the future
|
||||
tool_type = plan.current_task.task_type
|
||||
|
||||
# prepare tool-type-specific instruction
|
||||
tool_type_usage_prompt = (
|
||||
TOOL_REGISTRY.get_tool_type(tool_type).usage_prompt if TOOL_REGISTRY.has_tool_type(tool_type) else ""
|
||||
class WriteAnalysisCode(Action):
|
||||
async def _debug_with_reflection(self, context: list[Message], working_memory: list[Message]):
|
||||
reflection_prompt = REFLECTION_PROMPT.format(
|
||||
debug_example=DEBUG_REFLECTION_EXAMPLE,
|
||||
context=context,
|
||||
previous_impl=working_memory,
|
||||
)
|
||||
|
||||
# prepare schemas of available tools
|
||||
tool_schemas = {}
|
||||
available_tools = self._get_tools_by_type(tool_type)
|
||||
if available_tools:
|
||||
available_tools = {tool_name: tool.schemas["description"] for tool_name, tool in available_tools.items()}
|
||||
tool_schemas = await self._recommend_tool(plan.current_task.instruction, available_tools)
|
||||
rsp = await self._aask(reflection_prompt, system_msgs=[REFLECTION_SYSTEM_MSG])
|
||||
reflection = json.loads(CodeParser.parse_code(block=None, text=rsp))
|
||||
|
||||
return tool_schemas, tool_type_usage_prompt
|
||||
return reflection["improved_impl"]
|
||||
|
||||
async def run(
|
||||
self,
|
||||
context: list[Message],
|
||||
plan: Plan,
|
||||
user_requirement: str,
|
||||
plan_status: str = "",
|
||||
tool_info: str = "",
|
||||
working_memory: list[Message] = None,
|
||||
use_reflection: bool = False,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
# prepare tool schemas and tool-type-specific instruction
|
||||
tool_schemas, tool_type_usage_prompt = await self._prepare_tools(plan=plan)
|
||||
|
||||
# form a complete tool usage instruction and include it as a message in context
|
||||
tools_instruction = TOOL_USAGE_PROMPT.format(
|
||||
tool_schemas=tool_schemas, tool_type_usage_prompt=tool_type_usage_prompt
|
||||
structual_prompt = STRUCTUAL_PROMPT.format(
|
||||
user_requirement=user_requirement,
|
||||
plan_status=plan_status,
|
||||
tool_info=tool_info,
|
||||
)
|
||||
context.append(Message(content=tools_instruction, role="user"))
|
||||
|
||||
# prepare prompt & LLM call
|
||||
prompt = self.insert_system_message(context)
|
||||
tool_config = create_func_call_config(CODE_GENERATOR_WITH_TOOLS)
|
||||
rsp = await self.llm.aask_code(prompt, **tool_config)
|
||||
working_memory = working_memory or []
|
||||
context = self.llm.format_msg([Message(content=structual_prompt, role="user")] + working_memory)
|
||||
|
||||
return rsp
|
||||
# LLM call
|
||||
if use_reflection:
|
||||
code = await self._debug_with_reflection(context=context, working_memory=working_memory)
|
||||
else:
|
||||
rsp = await self.llm.aask(context, system_msgs=[INTERPRETER_SYSTEM_MSG], **kwargs)
|
||||
code = CodeParser.parse_code(block=None, text=rsp)
|
||||
|
||||
return code
|
||||
|
||||
|
||||
class CheckData(Action):
|
||||
async def run(self, plan: Plan) -> dict:
|
||||
finished_tasks = plan.get_finished_tasks()
|
||||
code_written = [remove_comments(task.code) for task in finished_tasks]
|
||||
code_written = "\n\n".join(code_written)
|
||||
prompt = CHECK_DATA_PROMPT.format(code_written=code_written)
|
||||
rsp = await self._aask(prompt)
|
||||
code = CodeParser.parse_code(block=None, text=rsp)
|
||||
return code
|
||||
|
|
|
|||
|
|
@ -12,81 +12,49 @@ from typing import Tuple
|
|||
|
||||
from metagpt.actions import Action
|
||||
from metagpt.logs import logger
|
||||
from metagpt.prompts.di.write_analysis_code import (
|
||||
ASSIGN_TASK_TYPE_CONFIG,
|
||||
ASSIGN_TASK_TYPE_PROMPT,
|
||||
)
|
||||
from metagpt.schema import Message, Plan, Task
|
||||
from metagpt.tools import TOOL_REGISTRY
|
||||
from metagpt.utils.common import CodeParser, create_func_call_config
|
||||
from metagpt.strategy.task_type import TaskType
|
||||
from metagpt.utils.common import CodeParser
|
||||
|
||||
|
||||
class WritePlan(Action):
|
||||
PROMPT_TEMPLATE: str = """
|
||||
# Context:
|
||||
__context__
|
||||
{context}
|
||||
# Available Task Types:
|
||||
{task_type_desc}
|
||||
# Task:
|
||||
Based on the context, write a plan or modify an existing plan of what you should do to achieve the goal. A plan consists of one to __max_tasks__ tasks.
|
||||
Based on the context, write a plan or modify an existing plan of what you should do to achieve the goal. A plan consists of one to {max_tasks} tasks.
|
||||
If you are modifying an existing plan, carefully follow the instruction, don't make unnecessary changes. Give the whole plan unless instructed to modify only one task of the plan.
|
||||
If you encounter errors on the current task, revise and output the current single task only.
|
||||
Output a list of jsons following the format:
|
||||
```json
|
||||
[
|
||||
{
|
||||
{{
|
||||
"task_id": str = "unique identifier for a task in plan, can be an ordinal",
|
||||
"dependent_task_ids": list[str] = "ids of tasks prerequisite to this task",
|
||||
"instruction": "what you should do in this task, one short phrase or sentence",
|
||||
},
|
||||
"task_type": "type of this task, should be one of Available Task Types",
|
||||
}},
|
||||
...
|
||||
]
|
||||
```
|
||||
"""
|
||||
|
||||
async def assign_task_type(self, tasks: list[dict]) -> str:
|
||||
"""Assign task type to each task in tasks
|
||||
|
||||
Args:
|
||||
tasks (list[dict]): tasks to be assigned task type
|
||||
|
||||
Returns:
|
||||
str: tasks with task type assigned in a json string
|
||||
"""
|
||||
task_info = "\n".join([f"Task {task['task_id']}: {task['instruction']}" for task in tasks])
|
||||
task_type_desc = "\n".join(
|
||||
[f"- **{tool_type.name}**: {tool_type.desc}" for tool_type in TOOL_REGISTRY.get_tool_types().values()]
|
||||
) # task type are binded with tool type now, should be improved in the future
|
||||
prompt = ASSIGN_TASK_TYPE_PROMPT.format(
|
||||
task_info=task_info, task_type_desc=task_type_desc
|
||||
) # task types are set to be the same as tool types, for now
|
||||
tool_config = create_func_call_config(ASSIGN_TASK_TYPE_CONFIG)
|
||||
rsp = await self.llm.aask_code(prompt, **tool_config)
|
||||
task_type_list = rsp["task_type"]
|
||||
logger.info(f"assigned task types: {task_type_list}")
|
||||
for task, task_type in zip(tasks, task_type_list):
|
||||
task["task_type"] = task_type
|
||||
return json.dumps(tasks)
|
||||
|
||||
async def run(self, context: list[Message], max_tasks: int = 5, use_tools: bool = False) -> str:
|
||||
prompt = (
|
||||
self.PROMPT_TEMPLATE.replace("__context__", "\n".join([str(ct) for ct in context]))
|
||||
# .replace("__current_plan__", current_plan)
|
||||
.replace("__max_tasks__", str(max_tasks))
|
||||
async def run(self, context: list[Message], max_tasks: int = 5) -> str:
|
||||
task_type_desc = "\n".join([f"- **{tt.type_name}**: {tt.value.desc}" for tt in TaskType])
|
||||
prompt = self.PROMPT_TEMPLATE.format(
|
||||
context="\n".join([str(ct) for ct in context]), max_tasks=max_tasks, task_type_desc=task_type_desc
|
||||
)
|
||||
rsp = await self._aask(prompt)
|
||||
rsp = CodeParser.parse_code(block=None, text=rsp)
|
||||
if use_tools:
|
||||
rsp = await self.assign_task_type(json.loads(rsp))
|
||||
return rsp
|
||||
|
||||
|
||||
def rsp_to_tasks(rsp: str) -> list[Task]:
|
||||
def update_plan_from_rsp(rsp: str, current_plan: Plan):
|
||||
rsp = json.loads(rsp)
|
||||
tasks = [Task(**task_config) for task_config in rsp]
|
||||
return tasks
|
||||
|
||||
|
||||
def update_plan_from_rsp(rsp: str, current_plan: Plan):
|
||||
tasks = rsp_to_tasks(rsp)
|
||||
if len(tasks) == 1 or tasks[0].dependent_task_ids:
|
||||
if tasks[0].dependent_task_ids and len(tasks) > 1:
|
||||
# tasks[0].dependent_task_ids means the generated tasks are not a complete plan
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ class RebuildClassView(Action):
|
|||
path = self.context.git_repo.workdir / DATA_API_DESIGN_FILE_REPO
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
pathname = path / self.context.git_repo.workdir.name
|
||||
filename = str(pathname.with_suffix(".mmd"))
|
||||
filename = str(pathname.with_suffix(".class_diagram.mmd"))
|
||||
async with aiofiles.open(filename, mode="w", encoding="utf-8") as writer:
|
||||
content = "classDiagram\n"
|
||||
logger.debug(content)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from __future__ import annotations
|
|||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Set
|
||||
|
||||
from pydantic import BaseModel
|
||||
from tenacity import retry, stop_after_attempt, wait_random_exponential
|
||||
|
|
@ -125,7 +125,7 @@ class RebuildSequenceView(Action):
|
|||
if prefix in r.subject:
|
||||
classes.append(r)
|
||||
await self._rebuild_use_case(r.subject)
|
||||
participants = set()
|
||||
participants = await self._search_participants(split_namespace(entry.subject)[0])
|
||||
class_details = []
|
||||
class_views = []
|
||||
for c in classes:
|
||||
|
|
@ -171,7 +171,8 @@ class RebuildSequenceView(Action):
|
|||
sequence_view = rsp.removeprefix("```mermaid").removesuffix("```")
|
||||
rows = await self.graph_db.select(subject=entry.subject, predicate=GraphKeyword.HAS_SEQUENCE_VIEW)
|
||||
for r in rows:
|
||||
await self.graph_db.delete(subject=r.subject, predicate=r.predicate, object_=r.object_)
|
||||
if r.predicate == GraphKeyword.HAS_SEQUENCE_VIEW:
|
||||
await self.graph_db.delete(subject=r.subject, predicate=r.predicate, object_=r.object_)
|
||||
await self.graph_db.insert(
|
||||
subject=entry.subject, predicate=GraphKeyword.HAS_SEQUENCE_VIEW, object_=sequence_view
|
||||
)
|
||||
|
|
@ -184,7 +185,7 @@ class RebuildSequenceView(Action):
|
|||
await self.graph_db.insert(
|
||||
subject=entry.subject, predicate=GraphKeyword.HAS_PARTICIPANT, object_=auto_namespace(c.subject)
|
||||
)
|
||||
await self.graph_db.save()
|
||||
await self._save_sequence_view(subject=entry.subject, content=sequence_view)
|
||||
|
||||
async def _merge_sequence_view(self, entry: SPO) -> bool:
|
||||
"""
|
||||
|
|
@ -267,38 +268,6 @@ class RebuildSequenceView(Action):
|
|||
prompt_blocks.append(block)
|
||||
prompt = "\n---\n".join(prompt_blocks)
|
||||
|
||||
# class _UseCase(BaseModel):
|
||||
# description: str = Field(default="...", description="Describes about what the use case to do")
|
||||
# inputs: List[str] = Field(default=["input name 1", "input name 2"],
|
||||
# description="Lists the input names of the use case from external sources")
|
||||
# outputs: List[str] = Field(default=["output name 1", "output name 2"],
|
||||
# description="Lists the output names of the use case to external sources")
|
||||
# actors: List[str] = Field(default=["actor name 1", "actor name 2"],
|
||||
# description="Lists the participant actors of the use case")
|
||||
# steps: List[str] = Field(default=["Step 1", "Step 2"],
|
||||
# description="Lists the steps about how the use case works step by step")
|
||||
# reason: str = Field(default="Because ...",
|
||||
# description="Explaining under what circumstances would the external system execute this use case.")
|
||||
#
|
||||
#
|
||||
# class _UseCaseList(BaseModel):
|
||||
# description: str = Field(default="...",
|
||||
# description="A summary explains what the whole source code want to do")
|
||||
# use_cases: List[_UseCase] = Field(default=[
|
||||
# {
|
||||
# "description": "Describes about what the use case to do",
|
||||
# "inputs": ["input name 1", "input name 2"],
|
||||
# "outputs": ["output name 1", "output name 2"],
|
||||
# "actors": ["actor name 1", "actor name 2"],
|
||||
# "steps": ["Step 1", "Step 2"],
|
||||
# "reason": "Because ..."
|
||||
# }
|
||||
# ], description="List all use cases.")
|
||||
# relationship: List[str] = Field(default=["use case 1 ..."],
|
||||
# description="Lists all the descriptions of relationship among these use cases")
|
||||
|
||||
# rsp = await ActionNode.from_pydantic(_UseCaseList).fill(context=prompt, llm=self.llm)
|
||||
|
||||
rsp = await self.llm.aask(
|
||||
msg=prompt,
|
||||
system_msgs=[
|
||||
|
|
@ -327,7 +296,6 @@ class RebuildSequenceView(Action):
|
|||
await self.graph_db.insert(
|
||||
subject=ns_class_name, predicate=GraphKeyword.HAS_CLASS_USE_CASE, object_=detail.model_dump_json()
|
||||
)
|
||||
await self.graph_db.save()
|
||||
|
||||
@retry(
|
||||
wait=wait_random_exponential(min=1, max=20),
|
||||
|
|
@ -347,7 +315,6 @@ class RebuildSequenceView(Action):
|
|||
use_case_markdown = await self._get_class_use_cases(ns_class_name)
|
||||
if not use_case_markdown: # external class
|
||||
await self.graph_db.insert(subject=ns_class_name, predicate=GraphKeyword.HAS_SEQUENCE_VIEW, object_="")
|
||||
await self.graph_db.save()
|
||||
return
|
||||
block = f"## Use Cases\n{use_case_markdown}"
|
||||
prompts_blocks.append(block)
|
||||
|
|
@ -382,7 +349,6 @@ class RebuildSequenceView(Action):
|
|||
await self.graph_db.insert(
|
||||
subject=ns_class_name, predicate=GraphKeyword.HAS_SEQUENCE_VIEW, object_=sequence_view
|
||||
)
|
||||
await self.graph_db.save()
|
||||
|
||||
async def _get_participants(self, ns_class_name: str) -> List[str]:
|
||||
"""
|
||||
|
|
@ -574,14 +540,12 @@ class RebuildSequenceView(Action):
|
|||
await self.graph_db.insert(
|
||||
subject=entry.subject, predicate=GraphKeyword.HAS_PARTICIPANT, object_=concat_namespace("?", class_name)
|
||||
)
|
||||
await self.graph_db.save()
|
||||
return
|
||||
if len(participants) > 1:
|
||||
for r in participants:
|
||||
await self.graph_db.insert(
|
||||
subject=entry.subject, predicate=GraphKeyword.HAS_PARTICIPANT, object_=auto_namespace(r.subject)
|
||||
)
|
||||
await self.graph_db.save()
|
||||
return
|
||||
|
||||
participant = participants[0]
|
||||
|
|
@ -619,4 +583,31 @@ class RebuildSequenceView(Action):
|
|||
await self.graph_db.insert(
|
||||
subject=entry.subject, predicate=GraphKeyword.HAS_PARTICIPANT, object_=auto_namespace(participant.subject)
|
||||
)
|
||||
await self.graph_db.save()
|
||||
await self._save_sequence_view(subject=entry.subject, content=sequence_view)
|
||||
|
||||
async def _save_sequence_view(self, subject: str, content: str):
|
||||
pattern = re.compile(r"[^a-zA-Z0-9]")
|
||||
name = re.sub(pattern, "_", subject)
|
||||
filename = Path(name).with_suffix(".sequence_diagram.mmd")
|
||||
await self.context.repo.resources.data_api_design.save(filename=str(filename), content=content)
|
||||
|
||||
async def _search_participants(self, filename: str) -> Set:
|
||||
content = await self._get_source_code(filename)
|
||||
|
||||
rsp = await self.llm.aask(
|
||||
msg=content,
|
||||
system_msgs=[
|
||||
"You are a tool for listing all class names used in a source file.",
|
||||
"Return a markdown JSON object with: "
|
||||
'- a "class_names" key containing the list of class names used in the file; '
|
||||
'- a "reasons" key lists all reason objects, each object containing a "class_name" key for class name, a "reference" key explaining the line where the class has been used.',
|
||||
],
|
||||
)
|
||||
|
||||
class _Data(BaseModel):
|
||||
class_names: List[str]
|
||||
reasons: List
|
||||
|
||||
json_blocks = parse_json_code_block(rsp)
|
||||
data = _Data.model_validate_json(json_blocks[0])
|
||||
return set(data.class_names)
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ class CollectLinks(Action):
|
|||
break
|
||||
|
||||
model_name = config.llm.model
|
||||
prompt = reduce_message_length(gen_msg(), model_name, system_text, 4096)
|
||||
prompt = reduce_message_length(gen_msg(), model_name, system_text, config.llm.max_token)
|
||||
logger.debug(prompt)
|
||||
queries = await self._aask(prompt, [system_text])
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ class Config(CLIParams, YamlModel):
|
|||
"""
|
||||
default_config_paths: List[Path] = [
|
||||
METAGPT_ROOT / "config/config2.yaml",
|
||||
Path.home() / ".metagpt/config2.yaml",
|
||||
CONFIG_ROOT / "config2.yaml",
|
||||
]
|
||||
|
||||
dicts = [dict(os.environ)]
|
||||
|
|
@ -100,6 +100,20 @@ class Config(CLIParams, YamlModel):
|
|||
final = merge_dict(dicts)
|
||||
return Config(**final)
|
||||
|
||||
@classmethod
|
||||
def from_llm_config(cls, llm_config: dict):
|
||||
"""user config llm
|
||||
example:
|
||||
llm_config = {"api_type": "xxx", "api_key": "xxx", "model": "xxx"}
|
||||
gpt4 = Config.from_llm_config(llm_config)
|
||||
A = Role(name="A", profile="Democratic candidate", goal="Win the election", actions=[a1], watch=[a2], config=gpt4)
|
||||
"""
|
||||
llm_config = LLMConfig.model_validate(llm_config)
|
||||
dicts = [dict(os.environ)]
|
||||
dicts += [{"llm": llm_config}]
|
||||
final = merge_dict(dicts)
|
||||
return Config(**final)
|
||||
|
||||
def update_via_cli(self, project_path, project_name, inc, reqa_file, max_auto_summarize_code):
|
||||
"""update config via cli"""
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from typing import Optional
|
|||
|
||||
from pydantic import field_validator
|
||||
|
||||
from metagpt.const import LLM_API_TIMEOUT
|
||||
from metagpt.utils.yaml_model import YamlModel
|
||||
|
||||
|
||||
|
|
@ -29,6 +30,7 @@ class LLMType(Enum):
|
|||
DASHSCOPE = "dashscope" # Aliyun LingJi DashScope
|
||||
MOONSHOT = "moonshot"
|
||||
MISTRAL = "mistral"
|
||||
YI = "yi" # lingyiwanwu
|
||||
|
||||
def __missing__(self, key):
|
||||
return self.OPENAI
|
||||
|
|
@ -73,7 +75,7 @@ class LLMConfig(YamlModel):
|
|||
stream: bool = False
|
||||
logprobs: Optional[bool] = None # https://cookbook.openai.com/examples/using_logprobs
|
||||
top_logprobs: Optional[int] = None
|
||||
timeout: int = 60
|
||||
timeout: int = 600
|
||||
|
||||
# For Network
|
||||
proxy: Optional[str] = None
|
||||
|
|
@ -87,3 +89,8 @@ class LLMConfig(YamlModel):
|
|||
if v in ["", None, "YOUR_API_KEY"]:
|
||||
raise ValueError("Please set your API key in config2.yaml")
|
||||
return v
|
||||
|
||||
@field_validator("timeout")
|
||||
@classmethod
|
||||
def check_timeout(cls, v):
|
||||
return v or LLM_API_TIMEOUT
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
"""
|
||||
from typing import Callable, Optional
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from metagpt.tools import SearchEngineType
|
||||
from metagpt.utils.yaml_model import YamlModel
|
||||
|
||||
|
|
@ -18,3 +20,11 @@ class SearchConfig(YamlModel):
|
|||
api_key: str = ""
|
||||
cse_id: str = "" # for google
|
||||
search_func: Optional[Callable] = None
|
||||
params: dict = Field(
|
||||
default_factory=lambda: {
|
||||
"engine": "google",
|
||||
"google_domain": "google.com",
|
||||
"gl": "us",
|
||||
"hl": "en",
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -123,7 +123,6 @@ BASE64_FORMAT = "base64"
|
|||
|
||||
# REDIS
|
||||
REDIS_KEY = "REDIS_KEY"
|
||||
LLM_API_TIMEOUT = 300
|
||||
|
||||
# Message id
|
||||
IGNORED_MESSAGE_ID = "0"
|
||||
|
|
@ -132,3 +131,7 @@ IGNORED_MESSAGE_ID = "0"
|
|||
GENERALIZATION = "Generalize"
|
||||
COMPOSITION = "Composite"
|
||||
AGGREGATION = "Aggregate"
|
||||
|
||||
# Timeout
|
||||
USE_CONFIG_TIMEOUT = 0 # Using llm.timeout configuration.
|
||||
LLM_API_TIMEOUT = 300
|
||||
|
|
|
|||
|
|
@ -34,5 +34,5 @@ # do a `tap` action on the screen
|
|||
## TODO
|
||||
- add android app operation assistant under `examples/android_assistant`
|
||||
- migrate roles/actions of werewolf game from old version into current version
|
||||
- migrate roles/actions of mincraft game from old version into current version
|
||||
- migrate roles/actions of minecraft game from old version into current version
|
||||
- migrate roles/actions of stanford_town game from old version into current version
|
||||
|
|
|
|||
|
|
@ -4,10 +4,9 @@
|
|||
|
||||
from metagpt.environment.base_env import Environment
|
||||
from metagpt.environment.android_env.android_env import AndroidEnv
|
||||
from metagpt.environment.mincraft_env.mincraft_env import MincraftExtEnv
|
||||
from metagpt.environment.werewolf_env.werewolf_env import WerewolfEnv
|
||||
from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv
|
||||
from metagpt.environment.software_env.software_env import SoftwareEnv
|
||||
|
||||
|
||||
__all__ = ["AndroidEnv", "MincraftExtEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"]
|
||||
__all__ = ["AndroidEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"]
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ class EnvType(Enum):
|
|||
ANDROID = "Android"
|
||||
GYM = "Gym"
|
||||
WEREWOLF = "Werewolf"
|
||||
MINCRAFT = "Mincraft"
|
||||
MINECRAFT = "Minecraft"
|
||||
STANFORDTOWN = "StanfordTown"
|
||||
|
||||
|
||||
|
|
@ -47,7 +47,7 @@ def mark_as_writeable(func):
|
|||
|
||||
|
||||
class ExtEnv(BaseModel):
|
||||
"""External Env to intergate actual game environment"""
|
||||
"""External Env to integrate actual game environment"""
|
||||
|
||||
def _check_api_exist(self, rw_api: Optional[str] = None):
|
||||
if not rw_api:
|
||||
|
|
@ -129,8 +129,8 @@ class Environment(ExtEnv):
|
|||
self.roles[role.profile] = role
|
||||
|
||||
for role in roles: # setup system message with roles
|
||||
role.set_env(self)
|
||||
role.context = self.context
|
||||
role.set_env(self)
|
||||
|
||||
def publish_message(self, message: Message, peekable: bool = True) -> bool:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@
|
|||
|
||||
from metagpt.const import METAGPT_ROOT
|
||||
|
||||
# For Mincraft Game Agent
|
||||
MC_CKPT_DIR = METAGPT_ROOT / "data/mincraft/ckpt"
|
||||
# For Minecraft Game Agent
|
||||
MC_CKPT_DIR = METAGPT_ROOT / "data/minecraft/ckpt"
|
||||
MC_LOG_DIR = METAGPT_ROOT / "logs"
|
||||
MC_DEFAULT_WARMUP = {
|
||||
"context": 15,
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Desc : MG Mincraft Env
|
||||
# @Desc : MG Minecraft Env
|
||||
# refs to `voyager voyager.py`
|
||||
|
||||
import json
|
||||
|
|
@ -12,15 +12,15 @@ from pydantic import ConfigDict, Field
|
|||
|
||||
from metagpt.config2 import config as CONFIG
|
||||
from metagpt.environment.base_env import Environment
|
||||
from metagpt.environment.mincraft_env.const import MC_CKPT_DIR
|
||||
from metagpt.environment.mincraft_env.mincraft_ext_env import MincraftExtEnv
|
||||
from metagpt.environment.minecraft_env.const import MC_CKPT_DIR
|
||||
from metagpt.environment.minecraft_env.minecraft_ext_env import MinecraftExtEnv
|
||||
from metagpt.logs import logger
|
||||
from metagpt.rag.vector_stores.chroma import ChromaVectorStore
|
||||
from metagpt.utils.common import load_mc_skills_code, read_json_file, write_json_file
|
||||
|
||||
|
||||
class MincraftEnv(Environment, MincraftExtEnv):
|
||||
"""MincraftEnv, including shared memory of cache and infomation between roles"""
|
||||
class MinecraftEnv(Environment, MinecraftExtEnv):
|
||||
"""MinecraftEnv, including shared memory of cache and information between roles"""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Desc : The Mincraft external environment to integrate with Mincraft game
|
||||
# @Desc : The Minecraft external environment to integrate with Minecraft game
|
||||
# refs to `voyager bridge.py`
|
||||
|
||||
import json
|
||||
|
|
@ -11,18 +11,18 @@ import requests
|
|||
from pydantic import ConfigDict, Field, model_validator
|
||||
|
||||
from metagpt.environment.base_env import ExtEnv, mark_as_writeable
|
||||
from metagpt.environment.mincraft_env.const import (
|
||||
from metagpt.environment.minecraft_env.const import (
|
||||
MC_CKPT_DIR,
|
||||
MC_CORE_INVENTORY_ITEMS,
|
||||
MC_CURRICULUM_OB,
|
||||
MC_DEFAULT_WARMUP,
|
||||
METAGPT_ROOT,
|
||||
)
|
||||
from metagpt.environment.mincraft_env.process_monitor import SubprocessMonitor
|
||||
from metagpt.environment.minecraft_env.process_monitor import SubprocessMonitor
|
||||
from metagpt.logs import logger
|
||||
|
||||
|
||||
class MincraftExtEnv(ExtEnv):
|
||||
class MinecraftExtEnv(ExtEnv):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
mc_port: Optional[int] = Field(default=None)
|
||||
|
|
@ -48,7 +48,7 @@ class MincraftExtEnv(ExtEnv):
|
|||
self.mineflayer = SubprocessMonitor(
|
||||
commands=[
|
||||
"node",
|
||||
METAGPT_ROOT.joinpath("metagpt", "environment", "mincraft_env", "mineflayer", "index.js"),
|
||||
METAGPT_ROOT.joinpath("metagpt", "environment", "minecraft_env", "mineflayer", "index.js"),
|
||||
str(self.server_port),
|
||||
],
|
||||
name="mineflayer",
|
||||
|
|
@ -9,11 +9,11 @@
|
|||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import aiofiles
|
||||
import yaml
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from metagpt.context import Context
|
||||
from metagpt.utils.common import aread
|
||||
|
||||
|
||||
class Example(BaseModel):
|
||||
|
|
@ -68,8 +68,7 @@ class SkillsDeclaration(BaseModel):
|
|||
async def load(skill_yaml_file_name: Path = None) -> "SkillsDeclaration":
|
||||
if not skill_yaml_file_name:
|
||||
skill_yaml_file_name = Path(__file__).parent.parent.parent / "docs/.well-known/skills.yaml"
|
||||
async with aiofiles.open(str(skill_yaml_file_name), mode="r") as reader:
|
||||
data = await reader.read(-1)
|
||||
data = await aread(filename=skill_yaml_file_name)
|
||||
skill_data = yaml.safe_load(data)
|
||||
return SkillsDeclaration(**skill_data)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,128 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2023/11/24 15:43
|
||||
# @Author : lidanyang
|
||||
# @File : ml_action
|
||||
# @Desc :
|
||||
UPDATE_DATA_COLUMNS = """
|
||||
# Background
|
||||
Keep dataset column information updated before model train.
|
||||
## Done Tasks
|
||||
```python
|
||||
{history_code}
|
||||
```end
|
||||
|
||||
# Task
|
||||
Update and print the dataset's column information only if the train or test data has changed. Use the following code:
|
||||
```python
|
||||
from metagpt.tools.libs.data_preprocess import get_column_info
|
||||
|
||||
column_info = get_column_info(df)
|
||||
print("column_info")
|
||||
print(column_info)
|
||||
```end
|
||||
|
||||
# Constraints:
|
||||
- Use the DataFrame variable from 'Done Tasks' in place of df.
|
||||
- Import `get_column_info` only if it's not already imported.
|
||||
"""
|
||||
|
||||
PRINT_DATA_COLUMNS = {
|
||||
"name": "print_column_info",
|
||||
"description": "Print the latest column information after 'Done Tasks' code if first read or data changed.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {
|
||||
"type": "string",
|
||||
"description": "The code to be added to a new cell in jupyter.",
|
||||
},
|
||||
},
|
||||
"required": ["code"],
|
||||
},
|
||||
}
|
||||
|
||||
ML_COMMON_PROMPT = """
|
||||
# Background
|
||||
As a data scientist, you need to help user to achieve their goal [{user_requirement}] step-by-step in an continuous Jupyter notebook.
|
||||
|
||||
## Done Tasks
|
||||
```python
|
||||
{history_code}
|
||||
```end
|
||||
|
||||
## Current Task
|
||||
{current_task}
|
||||
|
||||
# Latest Data Info
|
||||
Latest data info after previous tasks:
|
||||
{column_info}
|
||||
|
||||
# Task
|
||||
Write complete code for 'Current Task'. And avoid duplicating code from 'Done Tasks', such as repeated import of packages, reading data, etc.
|
||||
Specifically, {tool_type_usage_prompt}
|
||||
"""
|
||||
|
||||
USE_NO_TOOLS_EXAMPLE = """
|
||||
# Output Example:
|
||||
when current task is "train a lightgbm model on training data", the code can be like:
|
||||
```python
|
||||
# Step 1: check data type and convert to numeric
|
||||
obj_cols = train.select_dtypes(include='object').columns.tolist()
|
||||
|
||||
for col in obj_cols:
|
||||
encoder = LabelEncoder()
|
||||
train[col] = encoder.fit_transform(train[col].unique().tolist() + ['unknown'])
|
||||
test[col] = test[col].apply(lambda x: x if x in encoder.classes_ else 'unknown')
|
||||
test[col] = encoder.transform(test[col])
|
||||
|
||||
# Step 2: train lightgbm model
|
||||
model = LGBMClassifier()
|
||||
model.fit(train, y_train)
|
||||
```end
|
||||
|
||||
# Constraints:
|
||||
- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.
|
||||
"""
|
||||
|
||||
USE_TOOLS_EXAMPLE = """
|
||||
# Capabilities
|
||||
- You can utilize pre-defined tools in any code lines from 'Available Tools' in the form of Python Class.
|
||||
- You can freely combine the use of any other public packages, like sklearn, numpy, pandas, etc..
|
||||
|
||||
# Available Tools:
|
||||
Each Class tool is described in JSON format. When you call a tool, import the tool from its path first.
|
||||
{tool_schemas}
|
||||
|
||||
# Output Example:
|
||||
when current task is "do data preprocess, like fill missing value, handle outliers, etc.", the code can be like:
|
||||
```python
|
||||
# Step 1: fill missing value
|
||||
# Tools used: ['FillMissingValue']
|
||||
from metagpt.tools.libs.data_preprocess import FillMissingValue
|
||||
|
||||
train_processed = train.copy()
|
||||
test_processed = test.copy()
|
||||
num_cols = train_processed.select_dtypes(include='number').columns.tolist()
|
||||
if 'label' in num_cols:
|
||||
num_cols.remove('label')
|
||||
fill_missing_value = FillMissingValue(features=num_cols, strategy='mean')
|
||||
fill_missing_value.fit(train_processed)
|
||||
train_processed = fill_missing_value.transform(train_processed)
|
||||
test_processed = fill_missing_value.transform(test_processed)
|
||||
|
||||
# Step 2: handle outliers
|
||||
for col in num_cols:
|
||||
low, high = train_processed[col].quantile([0.01, 0.99])
|
||||
train_processed[col] = train_processed[col].clip(low, high)
|
||||
test_processed[col] = test_processed[col].clip(low, high)
|
||||
```end
|
||||
|
||||
# Constraints:
|
||||
- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.
|
||||
- Always prioritize using pre-defined tools for the same functionality.
|
||||
- Always copy the DataFrame before processing it and use the copy to process.
|
||||
"""
|
||||
|
||||
ML_GENERATE_CODE_PROMPT = ML_COMMON_PROMPT + USE_NO_TOOLS_EXAMPLE
|
||||
ML_TOOL_USAGE_PROMPT = ML_COMMON_PROMPT + USE_TOOLS_EXAMPLE
|
||||
|
|
@ -1,93 +1,112 @@
|
|||
ASSIGN_TASK_TYPE_PROMPT = """
|
||||
Please assign a task type to each task in the list below from the given categories:
|
||||
{task_info}
|
||||
INTERPRETER_SYSTEM_MSG = """As a data scientist, you need to help user to achieve their goal step by step in a continuous Jupyter notebook. Since it is a notebook environment, don't use asyncio.run. Instead, use await if you need to call an async function."""
|
||||
|
||||
## All Task Type:
|
||||
{task_type_desc}
|
||||
STRUCTUAL_PROMPT = """
|
||||
# User Requirement
|
||||
{user_requirement}
|
||||
|
||||
# Plan Status
|
||||
{plan_status}
|
||||
|
||||
# Tool Info
|
||||
{tool_info}
|
||||
|
||||
# Constraints
|
||||
- Take on Current Task if it is in Plan Status, otherwise, tackle User Requirement directly.
|
||||
- Ensure the output new code is executable in the same Jupyter notebook as the previous executed code.
|
||||
- Always prioritize using pre-defined tools for the same functionality.
|
||||
|
||||
# Output
|
||||
While some concise thoughts are helpful, code is absolutely required. Always output one and only one code block in your response. Output code in the following format:
|
||||
```python
|
||||
your code
|
||||
```
|
||||
"""
|
||||
|
||||
ASSIGN_TASK_TYPE_CONFIG = {
|
||||
"name": "assign_task_type",
|
||||
"description": "Assign task type to each task by order.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"task_type": {
|
||||
"type": "array",
|
||||
"description": "List of task type. The length should as long as task list",
|
||||
"items": {
|
||||
"type": "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["task_type"],
|
||||
},
|
||||
}
|
||||
REFLECTION_SYSTEM_MSG = """You are an AI Python assistant. You will be given your previous implementation code of a task, runtime error results, and a hint to change the implementation appropriately. Write your full implementation."""
|
||||
|
||||
TOOL_RECOMMENDATION_PROMPT = """
|
||||
## User Requirement:
|
||||
{current_task}
|
||||
DEBUG_REFLECTION_EXAMPLE = '''
|
||||
[previous impl]:
|
||||
assistant:
|
||||
```python
|
||||
def add(a: int, b: int) -> int:
|
||||
"""
|
||||
Given integers a and b, return the total value of a and b.
|
||||
"""
|
||||
return a - b
|
||||
```
|
||||
|
||||
## Task
|
||||
Recommend up to five tools from 'Available Tools' that can help solve the 'User Requirement'.
|
||||
user:
|
||||
Tests failed:
|
||||
assert add(1, 2) == 3 # output: -1
|
||||
assert add(1, 2) == 4 # output: -1
|
||||
|
||||
## Available Tools:
|
||||
{available_tools}
|
||||
[reflection on previous impl]:
|
||||
The implementation failed the test cases where the input integers are 1 and 2. The issue arises because the code does not add the two integers together, but instead subtracts the second integer from the first. To fix this issue, we should change the operator from `-` to `+` in the return statement. This will ensure that the function returns the correct output for the given input.
|
||||
|
||||
## Tool Selection and Instructions:
|
||||
- Select tools most relevant to completing the 'User Requirement'.
|
||||
- If you believe that no tools are suitable, indicate with an empty list.
|
||||
- Only list the names of the tools, not the full schema of each tool.
|
||||
- Ensure selected tools are listed in 'Available Tools'.
|
||||
[improved impl]:
|
||||
def add(a: int, b: int) -> int:
|
||||
"""
|
||||
Given integers a and b, return the total value of a and b.
|
||||
"""
|
||||
return a + b
|
||||
'''
|
||||
|
||||
REFLECTION_PROMPT = """
|
||||
[example]
|
||||
Here is an example of debugging with reflection.
|
||||
{debug_example}
|
||||
[/example]
|
||||
|
||||
[context]
|
||||
{context}
|
||||
|
||||
[previous impl]:
|
||||
{previous_impl}
|
||||
|
||||
[instruction]
|
||||
Analyze your previous code and error in [context] step by step, provide me with improved method and code. Remember to follow [context] requirement. Don't forget to write code for steps behind the error step.
|
||||
Output a json following the format:
|
||||
```json
|
||||
{{
|
||||
"reflection": str = "Reflection on previous implementation",
|
||||
"improved_impl": str = "Refined code after reflection.",
|
||||
}}
|
||||
```
|
||||
"""
|
||||
|
||||
SELECT_FUNCTION_TOOLS = {
|
||||
"name": "select_function_tools",
|
||||
"description": "For current task, select suitable tools for it.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"recommend_tools": {
|
||||
"type": "array",
|
||||
"description": "List of tool names. Empty list if no tool is suitable.",
|
||||
"items": {
|
||||
"type": "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["recommend_tools"],
|
||||
},
|
||||
}
|
||||
CHECK_DATA_PROMPT = """
|
||||
# Background
|
||||
Check latest data info to guide subsequent tasks.
|
||||
|
||||
CODE_GENERATOR_WITH_TOOLS = {
|
||||
"name": "add_subtask_code",
|
||||
"description": "Add new code cell of current task to the end of an active Jupyter notebook.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {
|
||||
"type": "string",
|
||||
"description": "The code to be added to a new cell in jupyter.",
|
||||
},
|
||||
},
|
||||
"required": ["code"],
|
||||
},
|
||||
}
|
||||
## Finished Tasks
|
||||
```python
|
||||
{code_written}
|
||||
```end
|
||||
|
||||
TOOL_USAGE_PROMPT = """
|
||||
# Instruction
|
||||
Write complete code for 'Current Task'. And avoid duplicating code from finished tasks, such as repeated import of packages, reading data, etc.
|
||||
Specifically, {tool_type_usage_prompt}
|
||||
# Task
|
||||
Check code in finished tasks, print key variables to guide your following actions.
|
||||
Specifically, if it is a data analysis or machine learning task, print the the latest column information using the following code, with DataFrame variable from 'Finished Tasks' in place of df:
|
||||
```python
|
||||
from metagpt.tools.libs.data_preprocess import get_column_info
|
||||
|
||||
# Capabilities
|
||||
- You can utilize pre-defined tools in any code lines from 'Available Tools' in the form of Python Class.
|
||||
- You can freely combine the use of any other public packages, like sklearn, numpy, pandas, etc..
|
||||
|
||||
# Available Tools (can be empty):
|
||||
Each Class tool is described in JSON format. When you call a tool, import the tool first.
|
||||
{tool_schemas}
|
||||
column_info = get_column_info(df)
|
||||
print("column_info")
|
||||
print(column_info)
|
||||
```end
|
||||
Otherwise, print out any key variables you see fit. Return an empty string if you think there is no important data to check.
|
||||
|
||||
# Constraints:
|
||||
- Ensure the output new code is executable in the same Jupyter notebook with previous tasks code have been executed.
|
||||
- Always prioritize using pre-defined tools for the same functionality.
|
||||
- Your code is to be added to a new cell in jupyter.
|
||||
|
||||
# Instruction
|
||||
Output code following the format:
|
||||
```python
|
||||
your code
|
||||
```
|
||||
"""
|
||||
|
||||
DATA_INFO = """
|
||||
# Latest Data Info
|
||||
Latest data info after previous tasks:
|
||||
{info}
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
# Prompt for using tools of "eda" type
|
||||
# Prompt for taking on "eda" tasks
|
||||
EDA_PROMPT = """
|
||||
The current task is about exploratory data analysis, please note the following:
|
||||
- Distinguish column types with `select_dtypes` for tailored analysis and visualization, such as correlation.
|
||||
- Remember to `import numpy as np` before using Numpy functions.
|
||||
"""
|
||||
|
||||
# Prompt for using tools of "data_preprocess" type
|
||||
# Prompt for taking on "data_preprocess" tasks
|
||||
DATA_PREPROCESS_PROMPT = """
|
||||
The current task is about data preprocessing, please note the following:
|
||||
- Monitor data types per column, applying appropriate methods.
|
||||
|
|
@ -15,9 +15,10 @@ The current task is about data preprocessing, please note the following:
|
|||
- Prefer alternatives to one-hot encoding for categorical data.
|
||||
- Only encode or scale necessary columns to allow for potential feature-specific engineering tasks (like time_extract, binning, extraction, etc.) later.
|
||||
- Each step do data preprocessing to train, must do same for test separately at the same time.
|
||||
- Always copy the DataFrame before processing it and use the copy to process.
|
||||
"""
|
||||
|
||||
# Prompt for using tools of "feature_engineering" type
|
||||
# Prompt for taking on "feature_engineering" tasks
|
||||
FEATURE_ENGINEERING_PROMPT = """
|
||||
The current task is about feature engineering. when performing it, please adhere to the following principles:
|
||||
- Generate as diverse features as possible to improve the model's performance step-by-step.
|
||||
|
|
@ -27,9 +28,10 @@ The current task is about feature engineering. when performing it, please adhere
|
|||
- Each feature engineering operation performed on the train set must also applies to the test separately at the same time.
|
||||
- Avoid using the label column to create features, except for cat encoding.
|
||||
- Use the data from previous task result if exist, do not mock or reload data yourself.
|
||||
- Always copy the DataFrame before processing it and use the copy to process.
|
||||
"""
|
||||
|
||||
# Prompt for using tools of "model_train" type
|
||||
# Prompt for taking on "model_train" tasks
|
||||
MODEL_TRAIN_PROMPT = """
|
||||
The current task is about training a model, please ensure high performance:
|
||||
- Keep in mind that your user prioritizes results and is highly focused on model performance. So, when needed, feel free to use models of any complexity to improve effectiveness, such as XGBoost, CatBoost, etc.
|
||||
|
|
@ -38,14 +40,14 @@ The current task is about training a model, please ensure high performance:
|
|||
- Set suitable hyperparameters for the model, make metrics as high as possible.
|
||||
"""
|
||||
|
||||
# Prompt for using tools of "model_evaluate" type
|
||||
# Prompt for taking on "model_evaluate" tasks
|
||||
MODEL_EVALUATE_PROMPT = """
|
||||
The current task is about evaluating a model, please note the following:
|
||||
- Ensure that the evaluated data is same processed as the training data. If not, remember use object in 'Done Tasks' to transform the data.
|
||||
- Use trained model from previous task result directly, do not mock or reload model yourself.
|
||||
"""
|
||||
|
||||
# Prompt for using tools of "vision" type
|
||||
# Prompt for taking on "image2webpage" tasks
|
||||
IMAGE2WEBPAGE_PROMPT = """
|
||||
The current task is about converting image into webpage code. please note the following:
|
||||
- Single-Step Code Generation: Execute the entire code generation process in a single step, encompassing HTML, CSS, and JavaScript. Avoid fragmenting the code generation into multiple separate steps to maintain consistency and simplify the development workflow.
|
||||
|
|
@ -5,6 +5,7 @@ from anthropic import AsyncAnthropic
|
|||
from anthropic.types import Message, Usage
|
||||
|
||||
from metagpt.configs.llm_config import LLMConfig, LLMType
|
||||
from metagpt.const import USE_CONFIG_TIMEOUT
|
||||
from metagpt.logs import log_llm_stream
|
||||
from metagpt.provider.base_llm import BaseLLM
|
||||
from metagpt.provider.llm_provider_registry import register_provider
|
||||
|
|
@ -41,15 +42,15 @@ class AnthropicLLM(BaseLLM):
|
|||
def get_choice_text(self, resp: Message) -> str:
|
||||
return resp.content[0].text
|
||||
|
||||
async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> Message:
|
||||
async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> Message:
|
||||
resp: Message = await self.aclient.messages.create(**self._const_kwargs(messages))
|
||||
self._update_costs(resp.usage, self.model)
|
||||
return resp
|
||||
|
||||
async def acompletion(self, messages: list[dict], timeout: int = 3) -> Message:
|
||||
return await self._achat_completion(messages, timeout=timeout)
|
||||
async def acompletion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> Message:
|
||||
return await self._achat_completion(messages, timeout=self.get_timeout(timeout))
|
||||
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str:
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str:
|
||||
stream = await self.aclient.messages.create(**self._const_kwargs(messages, stream=True))
|
||||
collected_content = []
|
||||
usage = Usage(input_tokens=0, output_tokens=0)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ class AzureOpenAILLM(OpenAILLM):
|
|||
# https://learn.microsoft.com/zh-cn/azure/ai-services/openai/how-to/migration?tabs=python-new%2Cdalle-fix
|
||||
self.aclient = AsyncAzureOpenAI(**kwargs)
|
||||
self.model = self.config.model # Used in _calc_usage & _cons_kwargs
|
||||
self.pricing_plan = self.config.pricing_plan
|
||||
self.pricing_plan = self.config.pricing_plan or self.model
|
||||
|
||||
def _make_client_kwargs(self) -> dict:
|
||||
kwargs = dict(
|
||||
|
|
|
|||
|
|
@ -10,10 +10,9 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, Optional, Union
|
||||
from typing import Optional, Union
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from openai.types import CompletionUsage
|
||||
from pydantic import BaseModel
|
||||
from tenacity import (
|
||||
after_log,
|
||||
|
|
@ -24,11 +23,11 @@ from tenacity import (
|
|||
)
|
||||
|
||||
from metagpt.configs.llm_config import LLMConfig
|
||||
from metagpt.const import LLM_API_TIMEOUT, USE_CONFIG_TIMEOUT
|
||||
from metagpt.logs import logger
|
||||
from metagpt.schema import Message
|
||||
from metagpt.utils.common import log_and_reraise
|
||||
from metagpt.utils.cost_manager import CostManager, Costs
|
||||
from metagpt.utils.exceptions import handle_exception
|
||||
|
||||
|
||||
class BaseLLM(ABC):
|
||||
|
|
@ -41,7 +40,7 @@ class BaseLLM(ABC):
|
|||
# OpenAI / Azure / Others
|
||||
aclient: Optional[Union[AsyncOpenAI]] = None
|
||||
cost_manager: Optional[CostManager] = None
|
||||
model: Optional[str] = None
|
||||
model: Optional[str] = None # deprecated
|
||||
pricing_plan: Optional[str] = None
|
||||
|
||||
@abstractmethod
|
||||
|
|
@ -75,6 +74,28 @@ class BaseLLM(ABC):
|
|||
def _system_msg(self, msg: str) -> dict[str, str]:
|
||||
return {"role": "system", "content": msg}
|
||||
|
||||
def format_msg(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]:
|
||||
"""convert messages to list[dict]."""
|
||||
from metagpt.schema import Message
|
||||
|
||||
if not isinstance(messages, list):
|
||||
messages = [messages]
|
||||
|
||||
processed_messages = []
|
||||
for msg in messages:
|
||||
if isinstance(msg, str):
|
||||
processed_messages.append({"role": "user", "content": msg})
|
||||
elif isinstance(msg, dict):
|
||||
assert set(msg.keys()) == set(["role", "content"])
|
||||
processed_messages.append(msg)
|
||||
elif isinstance(msg, Message):
|
||||
processed_messages.append(msg.to_dict())
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Only support message type are: str, Message, dict, but got {type(messages).__name__}!"
|
||||
)
|
||||
return processed_messages
|
||||
|
||||
def _system_msgs(self, msgs: list[str]) -> list[dict[str, str]]:
|
||||
return [self._system_msg(msg) for msg in msgs]
|
||||
|
||||
|
|
@ -88,6 +109,7 @@ class BaseLLM(ABC):
|
|||
local_calc_usage (bool): some models don't calculate usage, it will overwrite LLMConfig.calc_usage
|
||||
"""
|
||||
calc_usage = self.config.calc_usage and local_calc_usage
|
||||
model = model or self.pricing_plan
|
||||
model = model or self.model
|
||||
usage = usage.model_dump() if isinstance(usage, BaseModel) else usage
|
||||
if calc_usage and self.cost_manager:
|
||||
|
|
@ -105,11 +127,11 @@ class BaseLLM(ABC):
|
|||
|
||||
async def aask(
|
||||
self,
|
||||
msg: str,
|
||||
msg: Union[str, list[dict[str, str]]],
|
||||
system_msgs: Optional[list[str]] = None,
|
||||
format_msgs: Optional[list[dict[str, str]]] = None,
|
||||
images: Optional[Union[str, list[str]]] = None,
|
||||
timeout=3,
|
||||
timeout=USE_CONFIG_TIMEOUT,
|
||||
stream=True,
|
||||
) -> str:
|
||||
if system_msgs:
|
||||
|
|
@ -120,34 +142,36 @@ class BaseLLM(ABC):
|
|||
message = []
|
||||
if format_msgs:
|
||||
message.extend(format_msgs)
|
||||
message.append(self._user_msg(msg, images=images))
|
||||
if isinstance(msg, str):
|
||||
message.append(self._user_msg(msg, images=images))
|
||||
else:
|
||||
message.extend(msg)
|
||||
logger.debug(message)
|
||||
rsp = await self.acompletion_text(message, stream=stream, timeout=timeout)
|
||||
rsp = await self.acompletion_text(message, stream=stream, timeout=self.get_timeout(timeout))
|
||||
return rsp
|
||||
|
||||
def _extract_assistant_rsp(self, context):
|
||||
return "\n".join([i["content"] for i in context if i["role"] == "assistant"])
|
||||
|
||||
async def aask_batch(self, msgs: list, timeout=3) -> str:
|
||||
async def aask_batch(self, msgs: list, timeout=USE_CONFIG_TIMEOUT) -> str:
|
||||
"""Sequential questioning"""
|
||||
context = []
|
||||
for msg in msgs:
|
||||
umsg = self._user_msg(msg)
|
||||
context.append(umsg)
|
||||
rsp_text = await self.acompletion_text(context, timeout=timeout)
|
||||
rsp_text = await self.acompletion_text(context, timeout=self.get_timeout(timeout))
|
||||
context.append(self._assistant_msg(rsp_text))
|
||||
return self._extract_assistant_rsp(context)
|
||||
|
||||
async def aask_code(self, messages: Union[str, Message, list[dict]], timeout=3) -> dict:
|
||||
"""FIXME: No code segment filtering has been done here, and all results are actually displayed"""
|
||||
async def aask_code(self, messages: Union[str, Message, list[dict]], timeout=USE_CONFIG_TIMEOUT, **kwargs) -> dict:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def _achat_completion(self, messages: list[dict], timeout=3):
|
||||
async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT):
|
||||
"""_achat_completion implemented by inherited class"""
|
||||
|
||||
@abstractmethod
|
||||
async def acompletion(self, messages: list[dict], timeout=3):
|
||||
async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT):
|
||||
"""Asynchronous version of completion
|
||||
All GPTAPIs are required to provide the standard OpenAI completion interface
|
||||
[
|
||||
|
|
@ -158,7 +182,7 @@ class BaseLLM(ABC):
|
|||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str:
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str:
|
||||
"""_achat_completion_stream implemented by inherited class"""
|
||||
|
||||
@retry(
|
||||
|
|
@ -168,11 +192,13 @@ class BaseLLM(ABC):
|
|||
retry=retry_if_exception_type(ConnectionError),
|
||||
retry_error_callback=log_and_reraise,
|
||||
)
|
||||
async def acompletion_text(self, messages: list[dict], stream: bool = False, timeout: int = 3) -> str:
|
||||
async def acompletion_text(
|
||||
self, messages: list[dict], stream: bool = False, timeout: int = USE_CONFIG_TIMEOUT
|
||||
) -> str:
|
||||
"""Asynchronous version of completion. Return str. Support stream-print"""
|
||||
if stream:
|
||||
return await self._achat_completion_stream(messages, timeout=timeout)
|
||||
resp = await self._achat_completion(messages, timeout=timeout)
|
||||
return await self._achat_completion_stream(messages, timeout=self.get_timeout(timeout))
|
||||
resp = await self._achat_completion(messages, timeout=self.get_timeout(timeout))
|
||||
return self.get_choice_text(resp)
|
||||
|
||||
def get_choice_text(self, rsp: dict) -> str:
|
||||
|
|
@ -223,20 +249,6 @@ class BaseLLM(ABC):
|
|||
"""
|
||||
return json.loads(self.get_choice_function(rsp)["arguments"], strict=False)
|
||||
|
||||
@handle_exception
|
||||
def _update_costs(self, usage: CompletionUsage | Dict):
|
||||
"""
|
||||
Updates the costs based on the provided usage information.
|
||||
"""
|
||||
if self.config.calc_usage and usage and self.cost_manager:
|
||||
if isinstance(usage, Dict):
|
||||
prompt_tokens = int(usage.get("prompt_tokens", 0))
|
||||
completion_tokens = int(usage.get("completion_tokens", 0))
|
||||
else:
|
||||
prompt_tokens = usage.prompt_tokens
|
||||
completion_tokens = usage.completion_tokens
|
||||
self.cost_manager.update_cost(prompt_tokens, completion_tokens, self.pricing_plan)
|
||||
|
||||
def messages_to_prompt(self, messages: list[dict]):
|
||||
"""[{"role": "user", "content": msg}] to user: <msg> etc."""
|
||||
return "\n".join([f"{i['role']}: {i['content']}" for i in messages])
|
||||
|
|
@ -244,3 +256,11 @@ class BaseLLM(ABC):
|
|||
def messages_to_dict(self, messages):
|
||||
"""objects to [{"role": "user", "content": msg}] etc."""
|
||||
return [i.to_dict() for i in messages]
|
||||
|
||||
def with_model(self, model: str):
|
||||
"""Set model and return self. For example, `with_model("gpt-3.5-turbo")`."""
|
||||
self.config.model = model
|
||||
return self
|
||||
|
||||
def get_timeout(self, timeout: int) -> int:
|
||||
return timeout or self.config.timeout or LLM_API_TIMEOUT
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ GENERAL_FUNCTION_SCHEMA = {
|
|||
},
|
||||
}
|
||||
|
||||
|
||||
# tool_choice value for general_function_schema
|
||||
# https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice
|
||||
GENERAL_TOOL_CHOICE = {"type": "function", "function": {"name": "execute"}}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from dashscope.common.error import (
|
|||
UnsupportedApiProtocol,
|
||||
)
|
||||
|
||||
from metagpt.const import USE_CONFIG_TIMEOUT
|
||||
from metagpt.logs import log_llm_stream
|
||||
from metagpt.provider.base_llm import BaseLLM, LLMConfig
|
||||
from metagpt.provider.llm_provider_registry import LLMType, register_provider
|
||||
|
|
@ -202,16 +203,16 @@ class DashScopeLLM(BaseLLM):
|
|||
self._update_costs(dict(resp.usage))
|
||||
return resp.output
|
||||
|
||||
async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> GenerationOutput:
|
||||
async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> GenerationOutput:
|
||||
resp: GenerationResponse = await self.aclient.acall(**self._const_kwargs(messages, stream=False))
|
||||
self._check_response(resp)
|
||||
self._update_costs(dict(resp.usage))
|
||||
return resp.output
|
||||
|
||||
async def acompletion(self, messages: list[dict], timeout=3) -> GenerationOutput:
|
||||
return await self._achat_completion(messages, timeout=timeout)
|
||||
async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> GenerationOutput:
|
||||
return await self._achat_completion(messages, timeout=self.get_timeout(timeout))
|
||||
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str:
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str:
|
||||
resp = await self.aclient.acall(**self._const_kwargs(messages, stream=True))
|
||||
collected_content = []
|
||||
usage = {}
|
||||
|
|
|
|||
|
|
@ -573,7 +573,7 @@ class APIRequestor:
|
|||
total=request_timeout[1],
|
||||
)
|
||||
else:
|
||||
timeout = aiohttp.ClientTimeout(total=request_timeout if request_timeout else TIMEOUT_SECS)
|
||||
timeout = aiohttp.ClientTimeout(total=request_timeout or TIMEOUT_SECS)
|
||||
|
||||
if files:
|
||||
# TODO: Use `aiohttp.MultipartWriter` to create the multipart form data here.
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Desc : Google Gemini LLM from https://ai.google.dev/tutorials/python_quickstart
|
||||
|
||||
from typing import Optional, Union
|
||||
import json
|
||||
import os
|
||||
from dataclasses import asdict
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import google.generativeai as genai
|
||||
from google.ai import generativelanguage as glm
|
||||
|
|
@ -10,14 +12,17 @@ from google.generativeai.generative_models import GenerativeModel
|
|||
from google.generativeai.types import content_types
|
||||
from google.generativeai.types.generation_types import (
|
||||
AsyncGenerateContentResponse,
|
||||
BlockedPromptException,
|
||||
GenerateContentResponse,
|
||||
GenerationConfig,
|
||||
)
|
||||
|
||||
from metagpt.configs.llm_config import LLMConfig, LLMType
|
||||
from metagpt.logs import log_llm_stream
|
||||
from metagpt.const import USE_CONFIG_TIMEOUT
|
||||
from metagpt.logs import log_llm_stream, logger
|
||||
from metagpt.provider.base_llm import BaseLLM
|
||||
from metagpt.provider.llm_provider_registry import register_provider
|
||||
from metagpt.schema import Message
|
||||
|
||||
|
||||
class GeminiGenerativeModel(GenerativeModel):
|
||||
|
|
@ -51,6 +56,10 @@ class GeminiLLM(BaseLLM):
|
|||
self.llm = GeminiGenerativeModel(model_name=self.model)
|
||||
|
||||
def __init_gemini(self, config: LLMConfig):
|
||||
if config.proxy:
|
||||
logger.info(f"Use proxy: {config.proxy}")
|
||||
os.environ["http_proxy"] = config.proxy
|
||||
os.environ["https_proxy"] = config.proxy
|
||||
genai.configure(api_key=config.api_key)
|
||||
|
||||
def _user_msg(self, msg: str, images: Optional[Union[str, list[str]]] = None) -> dict[str, str]:
|
||||
|
|
@ -61,6 +70,35 @@ class GeminiLLM(BaseLLM):
|
|||
def _assistant_msg(self, msg: str) -> dict[str, str]:
|
||||
return {"role": "model", "parts": [msg]}
|
||||
|
||||
def _system_msg(self, msg: str) -> dict[str, str]:
|
||||
return {"role": "user", "parts": [msg]}
|
||||
|
||||
def format_msg(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]:
|
||||
"""convert messages to list[dict]."""
|
||||
from metagpt.schema import Message
|
||||
|
||||
if not isinstance(messages, list):
|
||||
messages = [messages]
|
||||
|
||||
# REF: https://ai.google.dev/tutorials/python_quickstart
|
||||
# As a dictionary, the message requires `role` and `parts` keys.
|
||||
# The role in a conversation can either be the `user`, which provides the prompts,
|
||||
# or `model`, which provides the responses.
|
||||
processed_messages = []
|
||||
for msg in messages:
|
||||
if isinstance(msg, str):
|
||||
processed_messages.append({"role": "user", "parts": [msg]})
|
||||
elif isinstance(msg, dict):
|
||||
assert set(msg.keys()) == set(["role", "parts"])
|
||||
processed_messages.append(msg)
|
||||
elif isinstance(msg, Message):
|
||||
processed_messages.append({"role": "user" if msg.role == "user" else "model", "parts": [msg.content]})
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Only support message type are: str, Message, dict, but got {type(messages).__name__}!"
|
||||
)
|
||||
return processed_messages
|
||||
|
||||
def _const_kwargs(self, messages: list[dict], stream: bool = False) -> dict:
|
||||
kwargs = {"contents": messages, "generation_config": GenerationConfig(temperature=0.3), "stream": stream}
|
||||
return kwargs
|
||||
|
|
@ -88,22 +126,28 @@ class GeminiLLM(BaseLLM):
|
|||
self._update_costs(usage)
|
||||
return resp
|
||||
|
||||
async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> "AsyncGenerateContentResponse":
|
||||
async def _achat_completion(
|
||||
self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT
|
||||
) -> "AsyncGenerateContentResponse":
|
||||
resp: AsyncGenerateContentResponse = await self.llm.generate_content_async(**self._const_kwargs(messages))
|
||||
usage = await self.aget_usage(messages, resp.text)
|
||||
self._update_costs(usage)
|
||||
return resp
|
||||
|
||||
async def acompletion(self, messages: list[dict], timeout=3) -> dict:
|
||||
return await self._achat_completion(messages, timeout=timeout)
|
||||
async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict:
|
||||
return await self._achat_completion(messages, timeout=self.get_timeout(timeout))
|
||||
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str:
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str:
|
||||
resp: AsyncGenerateContentResponse = await self.llm.generate_content_async(
|
||||
**self._const_kwargs(messages, stream=True)
|
||||
)
|
||||
collected_content = []
|
||||
async for chunk in resp:
|
||||
content = chunk.text
|
||||
try:
|
||||
content = chunk.text
|
||||
except Exception as e:
|
||||
logger.warning(f"messages: {messages}\nerrors: {e}\n{BlockedPromptException(str(chunk))}")
|
||||
raise BlockedPromptException(str(chunk))
|
||||
log_llm_stream(content)
|
||||
collected_content.append(content)
|
||||
log_llm_stream("\n")
|
||||
|
|
@ -112,3 +156,10 @@ class GeminiLLM(BaseLLM):
|
|||
usage = await self.aget_usage(messages, full_content)
|
||||
self._update_costs(usage)
|
||||
return full_content
|
||||
|
||||
def list_models(self) -> List:
|
||||
models = []
|
||||
for model in genai.list_models(page_size=100):
|
||||
models.append(asdict(model))
|
||||
logger.info(json.dumps(models))
|
||||
return models
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ Author: garylin2099
|
|||
from typing import Optional
|
||||
|
||||
from metagpt.configs.llm_config import LLMConfig
|
||||
from metagpt.const import LLM_API_TIMEOUT, USE_CONFIG_TIMEOUT
|
||||
from metagpt.logs import logger
|
||||
from metagpt.provider.base_llm import BaseLLM
|
||||
|
||||
|
|
@ -16,9 +17,9 @@ class HumanProvider(BaseLLM):
|
|||
"""
|
||||
|
||||
def __init__(self, config: LLMConfig):
|
||||
pass
|
||||
self.config = config
|
||||
|
||||
def ask(self, msg: str, timeout=3) -> str:
|
||||
def ask(self, msg: str, timeout=USE_CONFIG_TIMEOUT) -> str:
|
||||
logger.info("It's your turn, please type in your response. You may also refer to the context below")
|
||||
rsp = input(msg)
|
||||
if rsp in ["exit", "quit"]:
|
||||
|
|
@ -31,20 +32,23 @@ class HumanProvider(BaseLLM):
|
|||
system_msgs: Optional[list[str]] = None,
|
||||
format_msgs: Optional[list[dict[str, str]]] = None,
|
||||
generator: bool = False,
|
||||
timeout=3,
|
||||
timeout=USE_CONFIG_TIMEOUT,
|
||||
) -> str:
|
||||
return self.ask(msg, timeout=timeout)
|
||||
return self.ask(msg, timeout=self.get_timeout(timeout))
|
||||
|
||||
async def _achat_completion(self, messages: list[dict], timeout=3):
|
||||
async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT):
|
||||
pass
|
||||
|
||||
async def acompletion(self, messages: list[dict], timeout=3):
|
||||
async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT):
|
||||
"""dummy implementation of abstract method in base"""
|
||||
return []
|
||||
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str:
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str:
|
||||
pass
|
||||
|
||||
async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str:
|
||||
async def acompletion_text(self, messages: list[dict], stream=False, timeout=USE_CONFIG_TIMEOUT) -> str:
|
||||
"""dummy implementation of abstract method in base"""
|
||||
return ""
|
||||
|
||||
def get_timeout(self, timeout: int) -> int:
|
||||
return timeout or LLM_API_TIMEOUT
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
import json
|
||||
|
||||
from metagpt.configs.llm_config import LLMConfig, LLMType
|
||||
from metagpt.const import LLM_API_TIMEOUT
|
||||
from metagpt.const import USE_CONFIG_TIMEOUT
|
||||
from metagpt.logs import log_llm_stream
|
||||
from metagpt.provider.base_llm import BaseLLM
|
||||
from metagpt.provider.general_api_requestor import GeneralAPIRequestor
|
||||
|
|
@ -50,28 +50,28 @@ class OllamaLLM(BaseLLM):
|
|||
chunk = chunk.decode(encoding)
|
||||
return json.loads(chunk)
|
||||
|
||||
async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> dict:
|
||||
async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> dict:
|
||||
resp, _, _ = await self.client.arequest(
|
||||
method=self.http_method,
|
||||
url=self.suffix_url,
|
||||
params=self._const_kwargs(messages),
|
||||
request_timeout=LLM_API_TIMEOUT,
|
||||
request_timeout=self.get_timeout(timeout),
|
||||
)
|
||||
resp = self._decode_and_load(resp)
|
||||
usage = self.get_usage(resp)
|
||||
self._update_costs(usage)
|
||||
return resp
|
||||
|
||||
async def acompletion(self, messages: list[dict], timeout=3) -> dict:
|
||||
return await self._achat_completion(messages, timeout=timeout)
|
||||
async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict:
|
||||
return await self._achat_completion(messages, timeout=self.get_timeout(timeout))
|
||||
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str:
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str:
|
||||
stream_resp, _, _ = await self.client.arequest(
|
||||
method=self.http_method,
|
||||
url=self.suffix_url,
|
||||
stream=True,
|
||||
params=self._const_kwargs(messages, stream=True),
|
||||
request_timeout=LLM_API_TIMEOUT,
|
||||
request_timeout=self.get_timeout(timeout),
|
||||
)
|
||||
|
||||
collected_content = []
|
||||
|
|
|
|||
|
|
@ -25,11 +25,11 @@ from tenacity import (
|
|||
)
|
||||
|
||||
from metagpt.configs.llm_config import LLMConfig, LLMType
|
||||
from metagpt.const import USE_CONFIG_TIMEOUT
|
||||
from metagpt.logs import log_llm_stream, logger
|
||||
from metagpt.provider.base_llm import BaseLLM
|
||||
from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA
|
||||
from metagpt.provider.llm_provider_registry import register_provider
|
||||
from metagpt.schema import Message
|
||||
from metagpt.utils.common import CodeParser, decode_image, log_and_reraise
|
||||
from metagpt.utils.cost_manager import CostManager
|
||||
from metagpt.utils.exceptions import handle_exception
|
||||
|
|
@ -40,7 +40,7 @@ from metagpt.utils.token_counter import (
|
|||
)
|
||||
|
||||
|
||||
@register_provider([LLMType.OPENAI, LLMType.FIREWORKS, LLMType.OPEN_LLM, LLMType.MOONSHOT, LLMType.MISTRAL])
|
||||
@register_provider([LLMType.OPENAI, LLMType.FIREWORKS, LLMType.OPEN_LLM, LLMType.MOONSHOT, LLMType.MISTRAL, LLMType.YI])
|
||||
class OpenAILLM(BaseLLM):
|
||||
"""Check https://platform.openai.com/examples for examples"""
|
||||
|
||||
|
|
@ -75,15 +75,17 @@ class OpenAILLM(BaseLLM):
|
|||
|
||||
return params
|
||||
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout=3) -> str:
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> str:
|
||||
response: AsyncStream[ChatCompletionChunk] = await self.aclient.chat.completions.create(
|
||||
**self._cons_kwargs(messages, timeout=timeout), stream=True
|
||||
**self._cons_kwargs(messages, timeout=self.get_timeout(timeout)), stream=True
|
||||
)
|
||||
usage = None
|
||||
collected_messages = []
|
||||
async for chunk in response:
|
||||
chunk_message = chunk.choices[0].delta.content or "" if chunk.choices else "" # extract the message
|
||||
finish_reason = chunk.choices[0].finish_reason if hasattr(chunk.choices[0], "finish_reason") else None
|
||||
finish_reason = (
|
||||
chunk.choices[0].finish_reason if chunk.choices and hasattr(chunk.choices[0], "finish_reason") else None
|
||||
)
|
||||
log_llm_stream(chunk_message)
|
||||
collected_messages.append(chunk_message)
|
||||
if finish_reason:
|
||||
|
|
@ -103,7 +105,7 @@ class OpenAILLM(BaseLLM):
|
|||
self._update_costs(usage)
|
||||
return full_reply_content
|
||||
|
||||
def _cons_kwargs(self, messages: list[dict], timeout=3, **extra_kwargs) -> dict:
|
||||
def _cons_kwargs(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT, **extra_kwargs) -> dict:
|
||||
kwargs = {
|
||||
"messages": messages,
|
||||
"max_tokens": self._get_max_tokens(messages),
|
||||
|
|
@ -111,20 +113,20 @@ class OpenAILLM(BaseLLM):
|
|||
# "stop": None, # default it's None and gpt4-v can't have this one
|
||||
"temperature": self.config.temperature,
|
||||
"model": self.model,
|
||||
"timeout": max(self.config.timeout, timeout),
|
||||
"timeout": self.get_timeout(timeout),
|
||||
}
|
||||
if extra_kwargs:
|
||||
kwargs.update(extra_kwargs)
|
||||
return kwargs
|
||||
|
||||
async def _achat_completion(self, messages: list[dict], timeout=3) -> ChatCompletion:
|
||||
kwargs = self._cons_kwargs(messages, timeout=timeout)
|
||||
async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> ChatCompletion:
|
||||
kwargs = self._cons_kwargs(messages, timeout=self.get_timeout(timeout))
|
||||
rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs)
|
||||
self._update_costs(rsp.usage)
|
||||
return rsp
|
||||
|
||||
async def acompletion(self, messages: list[dict], timeout=3) -> ChatCompletion:
|
||||
return await self._achat_completion(messages, timeout=timeout)
|
||||
async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> ChatCompletion:
|
||||
return await self._achat_completion(messages, timeout=self.get_timeout(timeout))
|
||||
|
||||
@retry(
|
||||
wait=wait_random_exponential(min=1, max=60),
|
||||
|
|
@ -133,52 +135,24 @@ class OpenAILLM(BaseLLM):
|
|||
retry=retry_if_exception_type(APIConnectionError),
|
||||
retry_error_callback=log_and_reraise,
|
||||
)
|
||||
async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str:
|
||||
async def acompletion_text(self, messages: list[dict], stream=False, timeout=USE_CONFIG_TIMEOUT) -> str:
|
||||
"""when streaming, print each token in place."""
|
||||
if stream:
|
||||
await self._achat_completion_stream(messages, timeout=timeout)
|
||||
return await self._achat_completion_stream(messages, timeout=timeout)
|
||||
|
||||
rsp = await self._achat_completion(messages, timeout=timeout)
|
||||
rsp = await self._achat_completion(messages, timeout=self.get_timeout(timeout))
|
||||
return self.get_choice_text(rsp)
|
||||
|
||||
def _func_configs(self, messages: list[dict], timeout=3, **kwargs) -> dict:
|
||||
"""Note: Keep kwargs consistent with https://platform.openai.com/docs/api-reference/chat/create"""
|
||||
if "tools" not in kwargs:
|
||||
configs = {"tools": [{"type": "function", "function": GENERAL_FUNCTION_SCHEMA}]}
|
||||
kwargs.update(configs)
|
||||
|
||||
return self._cons_kwargs(messages=messages, timeout=timeout, **kwargs)
|
||||
|
||||
def _process_message(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]:
|
||||
"""convert messages to list[dict]."""
|
||||
# 全部转成list
|
||||
if not isinstance(messages, list):
|
||||
messages = [messages]
|
||||
|
||||
# 转成list[dict]
|
||||
processed_messages = []
|
||||
for msg in messages:
|
||||
if isinstance(msg, str):
|
||||
processed_messages.append({"role": "user", "content": msg})
|
||||
elif isinstance(msg, dict):
|
||||
assert set(msg.keys()) == set(["role", "content"])
|
||||
processed_messages.append(msg)
|
||||
elif isinstance(msg, Message):
|
||||
processed_messages.append(msg.to_dict())
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Only support message type are: str, Message, dict, but got {type(messages).__name__}!"
|
||||
)
|
||||
return processed_messages
|
||||
|
||||
async def _achat_completion_function(self, messages: list[dict], timeout=3, **chat_configs) -> ChatCompletion:
|
||||
messages = self._process_message(messages)
|
||||
kwargs = self._func_configs(messages=messages, timeout=timeout, **chat_configs)
|
||||
async def _achat_completion_function(
|
||||
self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT, **chat_configs
|
||||
) -> ChatCompletion:
|
||||
messages = self.format_msg(messages)
|
||||
kwargs = self._cons_kwargs(messages=messages, timeout=self.get_timeout(timeout), **chat_configs)
|
||||
rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs)
|
||||
self._update_costs(rsp.usage)
|
||||
return rsp
|
||||
|
||||
async def aask_code(self, messages: list[dict], **kwargs) -> dict:
|
||||
async def aask_code(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT, **kwargs) -> dict:
|
||||
"""Use function of tools to ask a code.
|
||||
Note: Keep kwargs consistent with https://platform.openai.com/docs/api-reference/chat/create
|
||||
|
||||
|
|
@ -188,12 +162,15 @@ class OpenAILLM(BaseLLM):
|
|||
>>> rsp = await llm.aask_code(msg)
|
||||
# -> {'language': 'python', 'code': "print('Hello, World!')"}
|
||||
"""
|
||||
if "tools" not in kwargs:
|
||||
configs = {"tools": [{"type": "function", "function": GENERAL_FUNCTION_SCHEMA}]}
|
||||
kwargs.update(configs)
|
||||
rsp = await self._achat_completion_function(messages, **kwargs)
|
||||
return self.get_choice_function_arguments(rsp)
|
||||
|
||||
def _parse_arguments(self, arguments: str) -> dict:
|
||||
"""parse arguments in openai function call"""
|
||||
if "langugae" not in arguments and "code" not in arguments:
|
||||
if "language" not in arguments and "code" not in arguments:
|
||||
logger.warning(f"Not found `code`, `language`, We assume it is pure code:\n {arguments}\n. ")
|
||||
return {"language": "python", "code": arguments}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from qianfan import ChatCompletion
|
|||
from qianfan.resources.typing import JsonBody
|
||||
|
||||
from metagpt.configs.llm_config import LLMConfig, LLMType
|
||||
from metagpt.const import USE_CONFIG_TIMEOUT
|
||||
from metagpt.logs import log_llm_stream
|
||||
from metagpt.provider.base_llm import BaseLLM
|
||||
from metagpt.provider.llm_provider_registry import register_provider
|
||||
|
|
@ -107,15 +108,15 @@ class QianFanLLM(BaseLLM):
|
|||
self._update_costs(resp.body.get("usage", {}))
|
||||
return resp.body
|
||||
|
||||
async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> JsonBody:
|
||||
async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> JsonBody:
|
||||
resp = await self.aclient.ado(**self._const_kwargs(messages=messages, stream=False))
|
||||
self._update_costs(resp.body.get("usage", {}))
|
||||
return resp.body
|
||||
|
||||
async def acompletion(self, messages: list[dict], timeout: int = 3) -> JsonBody:
|
||||
return await self._achat_completion(messages, timeout=timeout)
|
||||
async def acompletion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> JsonBody:
|
||||
return await self._achat_completion(messages, timeout=self.get_timeout(timeout))
|
||||
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str:
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str:
|
||||
resp = await self.aclient.ado(**self._const_kwargs(messages=messages, stream=True))
|
||||
collected_content = []
|
||||
usage = {}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from wsgiref.handlers import format_date_time
|
|||
import websocket # 使用websocket_client
|
||||
|
||||
from metagpt.configs.llm_config import LLMConfig, LLMType
|
||||
from metagpt.const import USE_CONFIG_TIMEOUT
|
||||
from metagpt.logs import logger
|
||||
from metagpt.provider.base_llm import BaseLLM
|
||||
from metagpt.provider.llm_provider_registry import register_provider
|
||||
|
|
@ -31,19 +32,19 @@ class SparkLLM(BaseLLM):
|
|||
def get_choice_text(self, rsp: dict) -> str:
|
||||
return rsp["payload"]["choices"]["text"][-1]["content"]
|
||||
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str:
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str:
|
||||
pass
|
||||
|
||||
async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 3) -> str:
|
||||
async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = USE_CONFIG_TIMEOUT) -> str:
|
||||
# 不支持
|
||||
# logger.warning("当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。")
|
||||
w = GetMessageFromWeb(messages, self.config)
|
||||
return w.run()
|
||||
|
||||
async def _achat_completion(self, messages: list[dict], timeout=3):
|
||||
async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT):
|
||||
pass
|
||||
|
||||
async def acompletion(self, messages: list[dict], timeout=3):
|
||||
async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT):
|
||||
# 不支持异步
|
||||
w = GetMessageFromWeb(messages, self.config)
|
||||
return w.run()
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from typing import Optional
|
|||
from zhipuai.types.chat.chat_completion import Completion
|
||||
|
||||
from metagpt.configs.llm_config import LLMConfig, LLMType
|
||||
from metagpt.const import USE_CONFIG_TIMEOUT
|
||||
from metagpt.logs import log_llm_stream
|
||||
from metagpt.provider.base_llm import BaseLLM
|
||||
from metagpt.provider.llm_provider_registry import register_provider
|
||||
|
|
@ -45,22 +46,22 @@ class ZhiPuAILLM(BaseLLM):
|
|||
kwargs = {"model": self.model, "messages": messages, "stream": stream, "temperature": 0.3}
|
||||
return kwargs
|
||||
|
||||
def completion(self, messages: list[dict], timeout=3) -> dict:
|
||||
def completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict:
|
||||
resp: Completion = self.llm.chat.completions.create(**self._const_kwargs(messages))
|
||||
usage = resp.usage.model_dump()
|
||||
self._update_costs(usage)
|
||||
return resp.model_dump()
|
||||
|
||||
async def _achat_completion(self, messages: list[dict], timeout=3) -> dict:
|
||||
async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict:
|
||||
resp = await self.llm.acreate(**self._const_kwargs(messages))
|
||||
usage = resp.get("usage", {})
|
||||
self._update_costs(usage)
|
||||
return resp
|
||||
|
||||
async def acompletion(self, messages: list[dict], timeout=3) -> dict:
|
||||
return await self._achat_completion(messages, timeout=timeout)
|
||||
async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict:
|
||||
return await self._achat_completion(messages, timeout=self.get_timeout(timeout))
|
||||
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout=3) -> str:
|
||||
async def _achat_completion_stream(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> str:
|
||||
response = await self.llm.acreate_stream(**self._const_kwargs(messages, stream=True))
|
||||
collected_content = []
|
||||
usage = {}
|
||||
|
|
|
|||
|
|
@ -722,14 +722,19 @@ class RepoParser(BaseModel):
|
|||
path = Path(path)
|
||||
if not path.exists():
|
||||
return
|
||||
init_file = path / "__init__.py"
|
||||
if not init_file.exists():
|
||||
raise ValueError("Failed to import module __init__ with error:No module named __init__.")
|
||||
command = f"pyreverse {str(path)} -o dot"
|
||||
result = subprocess.run(command, shell=True, check=True, cwd=str(path))
|
||||
output_dir = path / "__dot__"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
result = subprocess.run(command, shell=True, check=True, cwd=str(output_dir))
|
||||
if result.returncode != 0:
|
||||
raise ValueError(f"{result}")
|
||||
class_view_pathname = path / "classes.dot"
|
||||
class_view_pathname = output_dir / "classes.dot"
|
||||
class_views = await self._parse_classes(class_view_pathname)
|
||||
relationship_views = await self._parse_class_relationships(class_view_pathname)
|
||||
packages_pathname = path / "packages.dot"
|
||||
packages_pathname = output_dir / "packages.dot"
|
||||
class_views, relationship_views, package_root = RepoParser._repair_namespaces(
|
||||
class_views=class_views, relationship_views=relationship_views, path=path
|
||||
)
|
||||
|
|
@ -975,6 +980,8 @@ class RepoParser(BaseModel):
|
|||
file_ns = file_ns[0:ix]
|
||||
continue
|
||||
break
|
||||
if file_ns == "":
|
||||
return ""
|
||||
internal_ns = package[ix + 1 :]
|
||||
ns = mappings[file_ns] + ":" + internal_ns.replace(".", ":")
|
||||
return ns
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ from metagpt.roles.engineer import Engineer
|
|||
from metagpt.roles.qa_engineer import QaEngineer
|
||||
from metagpt.roles.searcher import Searcher
|
||||
from metagpt.roles.sales import Sales
|
||||
from metagpt.roles.customer_service import CustomerService
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
|
@ -26,5 +25,4 @@ __all__ = [
|
|||
"QaEngineer",
|
||||
"Searcher",
|
||||
"Sales",
|
||||
"CustomerService",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,48 +1,97 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pydantic import Field
|
||||
import json
|
||||
from typing import Literal, Union
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from metagpt.actions.di.ask_review import ReviewConst
|
||||
from metagpt.actions.di.execute_nb_code import ExecuteNbCode
|
||||
from metagpt.actions.di.write_analysis_code import (
|
||||
WriteCodeWithoutTools,
|
||||
WriteCodeWithTools,
|
||||
)
|
||||
from metagpt.actions.di.write_analysis_code import CheckData, WriteAnalysisCode
|
||||
from metagpt.logs import logger
|
||||
from metagpt.prompts.di.write_analysis_code import DATA_INFO
|
||||
from metagpt.roles import Role
|
||||
from metagpt.schema import Message, Task, TaskResult
|
||||
from metagpt.strategy.task_type import TaskType
|
||||
from metagpt.tools.tool_recommend import BM25ToolRecommender, ToolRecommender
|
||||
from metagpt.utils.common import CodeParser
|
||||
|
||||
REACT_THINK_PROMPT = """
|
||||
# User Requirement
|
||||
{user_requirement}
|
||||
# Context
|
||||
{context}
|
||||
|
||||
Output a json following the format:
|
||||
```json
|
||||
{{
|
||||
"thoughts": str = "Thoughts on current situation, reflect on how you should proceed to fulfill the user requirement",
|
||||
"state": bool = "Decide whether you need to take more actions to complete the user requirement. Return true if you think so. Return false if you think the requirement has been completely fulfilled."
|
||||
}}
|
||||
```
|
||||
"""
|
||||
|
||||
|
||||
class DataInterpreter(Role):
|
||||
name: str = "David"
|
||||
profile: str = "DataInterpreter"
|
||||
auto_run: bool = True
|
||||
use_tools: bool = False
|
||||
use_plan: bool = True
|
||||
use_reflection: bool = False
|
||||
execute_code: ExecuteNbCode = Field(default_factory=ExecuteNbCode, exclude=True)
|
||||
tools: list[str] = []
|
||||
tools: Union[str, list[str]] = [] # Use special symbol ["<all>"] to indicate use of all registered tools
|
||||
tool_recommender: ToolRecommender = None
|
||||
react_mode: Literal["plan_and_act", "react"] = "plan_and_act"
|
||||
max_react_loop: int = 10 # used for react mode
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
auto_run=True,
|
||||
use_tools=False,
|
||||
tools=[],
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(auto_run=auto_run, use_tools=use_tools, tools=tools, **kwargs)
|
||||
self._set_react_mode(react_mode="plan_and_act", auto_run=auto_run, use_tools=use_tools)
|
||||
if use_tools and tools:
|
||||
from metagpt.tools.tool_registry import (
|
||||
validate_tool_names, # import upon use
|
||||
)
|
||||
|
||||
self.tools = validate_tool_names(tools)
|
||||
logger.info(f"will only use {self.tools} as tools")
|
||||
@model_validator(mode="after")
|
||||
def set_plan_and_tool(self) -> "Interpreter":
|
||||
self._set_react_mode(react_mode=self.react_mode, max_react_loop=self.max_react_loop, auto_run=self.auto_run)
|
||||
self.use_plan = (
|
||||
self.react_mode == "plan_and_act"
|
||||
) # create a flag for convenience, overwrite any passed-in value
|
||||
if self.tools:
|
||||
self.tool_recommender = BM25ToolRecommender(tools=self.tools)
|
||||
self.set_actions([WriteAnalysisCode])
|
||||
self._set_state(0)
|
||||
return self
|
||||
|
||||
@property
|
||||
def working_memory(self):
|
||||
return self.rc.working_memory
|
||||
|
||||
async def _think(self) -> bool:
|
||||
"""Useful in 'react' mode. Use LLM to decide whether and what to do next."""
|
||||
user_requirement = self.get_memories()[0].content
|
||||
context = self.working_memory.get()
|
||||
|
||||
if not context:
|
||||
# just started the run, we need action certainly
|
||||
self.working_memory.add(self.get_memories()[0]) # add user requirement to working memory
|
||||
self._set_state(0)
|
||||
return True
|
||||
|
||||
prompt = REACT_THINK_PROMPT.format(user_requirement=user_requirement, context=context)
|
||||
rsp = await self.llm.aask(prompt)
|
||||
rsp_dict = json.loads(CodeParser.parse_code(block=None, text=rsp))
|
||||
self.working_memory.add(Message(content=rsp_dict["thoughts"], role="assistant"))
|
||||
need_action = rsp_dict["state"]
|
||||
self._set_state(0) if need_action else self._set_state(-1)
|
||||
|
||||
return need_action
|
||||
|
||||
async def _act(self) -> Message:
|
||||
"""Useful in 'react' mode. Return a Message conforming to Role._act interface."""
|
||||
code, _, _ = await self._write_and_exec_code()
|
||||
return Message(content=code, role="assistant", cause_by=WriteAnalysisCode)
|
||||
|
||||
async def _plan_and_act(self) -> Message:
|
||||
rsp = await super()._plan_and_act()
|
||||
await self.execute_code.terminate()
|
||||
return rsp
|
||||
|
||||
async def _act_on_task(self, current_task: Task) -> TaskResult:
|
||||
"""Useful in 'plan_and_act' mode. Wrap the output in a TaskResult for review and confirmation."""
|
||||
code, result, is_success = await self._write_and_exec_code()
|
||||
task_result = TaskResult(code=code, result=result, is_success=is_success)
|
||||
return task_result
|
||||
|
|
@ -51,14 +100,30 @@ class DataInterpreter(Role):
|
|||
counter = 0
|
||||
success = False
|
||||
|
||||
# plan info
|
||||
plan_status = self.planner.get_plan_status() if self.use_plan else ""
|
||||
|
||||
# tool info
|
||||
if self.tools:
|
||||
context = (
|
||||
self.working_memory.get()[-1].content if self.working_memory.get() else ""
|
||||
) # thoughts from _think stage in 'react' mode
|
||||
plan = self.planner.plan if self.use_plan else None
|
||||
tool_info = await self.tool_recommender.get_recommended_tool_info(context=context, plan=plan)
|
||||
else:
|
||||
tool_info = ""
|
||||
|
||||
# data info
|
||||
await self._check_data()
|
||||
|
||||
while not success and counter < max_retry:
|
||||
### write code ###
|
||||
code, cause_by = await self._write_code()
|
||||
code, cause_by = await self._write_code(counter, plan_status, tool_info)
|
||||
|
||||
self.working_memory.add(Message(content=code["code"], role="assistant", cause_by=cause_by))
|
||||
self.working_memory.add(Message(content=code, role="assistant", cause_by=cause_by))
|
||||
|
||||
### execute code ###
|
||||
result, success = await self.execute_code.run(**code)
|
||||
result, success = await self.execute_code.run(code)
|
||||
print(result)
|
||||
|
||||
self.working_memory.add(Message(content=result, role="user", cause_by=ExecuteNbCode))
|
||||
|
|
@ -72,14 +137,48 @@ class DataInterpreter(Role):
|
|||
if ReviewConst.CHANGE_WORDS[0] in review:
|
||||
counter = 0 # redo the task again with help of human suggestions
|
||||
|
||||
return code["code"], result, success
|
||||
return code, result, success
|
||||
|
||||
async def _write_code(self):
|
||||
todo = WriteCodeWithoutTools() if not self.use_tools else WriteCodeWithTools(selected_tools=self.tools)
|
||||
async def _write_code(
|
||||
self,
|
||||
counter: int,
|
||||
plan_status: str = "",
|
||||
tool_info: str = "",
|
||||
):
|
||||
todo = self.rc.todo # todo is WriteAnalysisCode
|
||||
logger.info(f"ready to {todo.name}")
|
||||
use_reflection = counter > 0 and self.use_reflection # only use reflection after the first trial
|
||||
|
||||
context = self.planner.get_useful_memories()
|
||||
# print(*context, sep="\n***\n")
|
||||
code = await todo.run(context=context, plan=self.planner.plan, temperature=0.0)
|
||||
user_requirement = self.get_memories()[0].content
|
||||
|
||||
code = await todo.run(
|
||||
user_requirement=user_requirement,
|
||||
plan_status=plan_status,
|
||||
tool_info=tool_info,
|
||||
working_memory=self.working_memory.get(),
|
||||
use_reflection=use_reflection,
|
||||
)
|
||||
|
||||
return code, todo
|
||||
|
||||
async def _check_data(self):
|
||||
if (
|
||||
not self.use_plan
|
||||
or not self.planner.plan.get_finished_tasks()
|
||||
or self.planner.plan.current_task.task_type
|
||||
not in [
|
||||
TaskType.DATA_PREPROCESS.type_name,
|
||||
TaskType.FEATURE_ENGINEERING.type_name,
|
||||
TaskType.MODEL_TRAIN.type_name,
|
||||
]
|
||||
):
|
||||
return
|
||||
logger.info("Check updated data")
|
||||
code = await CheckData().run(self.planner.plan)
|
||||
if not code.strip():
|
||||
return
|
||||
result, success = await self.execute_code.run(code)
|
||||
if success:
|
||||
print(result)
|
||||
data_info = DATA_INFO.format(info=result)
|
||||
self.working_memory.add(Message(content=data_info, role="user", cause_by=CheckData))
|
||||
|
|
|
|||
|
|
@ -1,64 +0,0 @@
|
|||
from metagpt.actions.di.debug_code import DebugCode
|
||||
from metagpt.actions.di.execute_nb_code import ExecuteNbCode
|
||||
from metagpt.actions.di.ml_action import UpdateDataColumns, WriteCodeWithToolsML
|
||||
from metagpt.logs import logger
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
from metagpt.tools.tool_type import ToolType
|
||||
from metagpt.utils.common import any_to_str
|
||||
|
||||
|
||||
class MLEngineer(DataInterpreter):
|
||||
name: str = "Mark"
|
||||
profile: str = "MLEngineer"
|
||||
debug_context: list = []
|
||||
latest_code: str = ""
|
||||
|
||||
async def _write_code(self):
|
||||
if not self.use_tools:
|
||||
return await super()._write_code()
|
||||
|
||||
# In a trial and errors settings, check whether this is our first attempt to tackle the task. If there is no code execution before, then it is.
|
||||
is_first_trial = any_to_str(ExecuteNbCode) not in [msg.cause_by for msg in self.working_memory.get()]
|
||||
|
||||
if is_first_trial:
|
||||
# For the first trial, write task code from scratch
|
||||
column_info = await self._update_data_columns()
|
||||
|
||||
logger.info("Write code with tools")
|
||||
tool_context, code = await WriteCodeWithToolsML(selected_tools=self.tools).run(
|
||||
context=[], # context assembled inside the Action
|
||||
plan=self.planner.plan,
|
||||
column_info=column_info,
|
||||
)
|
||||
self.debug_context = tool_context
|
||||
cause_by = WriteCodeWithToolsML
|
||||
|
||||
else:
|
||||
# Previous trials resulted in error, debug and rewrite the code
|
||||
logger.warning("We got a bug, now start to debug...")
|
||||
code = await DebugCode().run(
|
||||
code=self.latest_code,
|
||||
runtime_result=self.working_memory.get(),
|
||||
context=self.debug_context,
|
||||
)
|
||||
logger.info(f"new code \n{code}")
|
||||
cause_by = DebugCode
|
||||
|
||||
self.latest_code = code["code"]
|
||||
|
||||
return code, cause_by
|
||||
|
||||
async def _update_data_columns(self):
|
||||
current_task = self.planner.plan.current_task
|
||||
if current_task.task_type not in [
|
||||
ToolType.DATA_PREPROCESS.type_name,
|
||||
ToolType.FEATURE_ENGINEERING.type_name,
|
||||
ToolType.MODEL_TRAIN.type_name,
|
||||
]:
|
||||
return ""
|
||||
logger.info("Check columns in updated data")
|
||||
code = await UpdateDataColumns().run(self.planner.plan)
|
||||
success = False
|
||||
result, success = await self.execute_code.run(**code)
|
||||
print(result)
|
||||
return result if success else ""
|
||||
|
|
@ -240,8 +240,8 @@ class Engineer(Role):
|
|||
async def _think(self) -> Action | None:
|
||||
if not self.src_workspace:
|
||||
self.src_workspace = self.git_repo.workdir / self.git_repo.workdir.name
|
||||
write_plan_and_change_filters = any_to_str_set([WriteTasks])
|
||||
write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode, FixBug])
|
||||
write_plan_and_change_filters = any_to_str_set([WriteTasks, FixBug])
|
||||
write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode])
|
||||
summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview])
|
||||
if not self.rc.news:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -169,6 +169,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel):
|
|||
|
||||
self._check_actions()
|
||||
self.llm.system_prompt = self._get_prefix()
|
||||
self.llm.cost_manager = self.context.cost_manager
|
||||
self._watch(kwargs.pop("watch", [UserRequirement]))
|
||||
|
||||
if self.latest_observed_msg:
|
||||
|
|
@ -277,7 +278,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel):
|
|||
self.actions.append(i)
|
||||
self.states.append(f"{len(self.actions) - 1}. {action}")
|
||||
|
||||
def _set_react_mode(self, react_mode: str, max_react_loop: int = 1, auto_run: bool = True, use_tools: bool = False):
|
||||
def _set_react_mode(self, react_mode: str, max_react_loop: int = 1, auto_run: bool = True):
|
||||
"""Set strategy of the Role reacting to observed Message. Variation lies in how
|
||||
this Role elects action to perform during the _think stage, especially if it is capable of multiple Actions.
|
||||
|
||||
|
|
@ -298,9 +299,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel):
|
|||
if react_mode == RoleReactMode.REACT:
|
||||
self.rc.max_react_loop = max_react_loop
|
||||
elif react_mode == RoleReactMode.PLAN_AND_ACT:
|
||||
self.planner = Planner(
|
||||
goal=self.goal, working_memory=self.rc.working_memory, auto_run=auto_run, use_tools=use_tools
|
||||
)
|
||||
self.planner = Planner(goal=self.goal, working_memory=self.rc.working_memory, auto_run=auto_run)
|
||||
|
||||
def _watch(self, actions: Iterable[Type[Action]] | Iterable[Action]):
|
||||
"""Watch Actions of interest. Role will select Messages caused by these Actions from its personal message
|
||||
|
|
@ -333,6 +332,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel):
|
|||
if env:
|
||||
env.set_addresses(self, self.addresses)
|
||||
self.llm.system_prompt = self._get_prefix()
|
||||
self.llm.cost_manager = self.context.cost_manager
|
||||
self.set_actions(self.actions) # reset actions to update llm and prefix
|
||||
|
||||
def _get_prefix(self):
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue