mirror of
https://github.com/FoundationAgents/MetaGPT.git
synced 2026-06-08 15:05:17 +02:00
Merge branch 'main' into code_interpreter
This commit is contained in:
yzlin
commit
f87387d22a
377 changed files with 15703 additions and 893 deletions
5
.gitattributes
vendored
5
.gitattributes
vendored
|
|
@ -12,6 +12,11 @@
|
|||
*.jpg binary
|
||||
*.gif binary
|
||||
*.ico binary
|
||||
*.jpeg binary
|
||||
*.mp3 binary
|
||||
*.zip binary
|
||||
*.bin binary
|
||||
|
||||
|
||||
# Preserve original line endings for specific document files
|
||||
*.doc text eol=crlf
|
||||
|
|
|
|||
1
.github/ISSUE_TEMPLATE/show_me_the_bug.md
vendored
1
.github/ISSUE_TEMPLATE/show_me_the_bug.md
vendored
|
|
@ -19,6 +19,7 @@
|
|||
- LLM type and model name:
|
||||
- System version:
|
||||
- Python version:
|
||||
- MetaGPT version or branch:
|
||||
|
||||
<!-- Dependent packagess：the packages version cause the bug(like `pydantic 1.10.8`), installation method（like `pip install metagpt` or `pip install from source` or `run in docker`） -->
|
||||
|
||||
|
|
|
|||
3
.github/workflows/build-package.yaml
vendored
3
.github/workflows/build-package.yaml
vendored
|
|
@ -1,8 +1,9 @@
|
|||
name: Build and upload python package
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
release:
|
||||
types: [created]
|
||||
types: [created, published]
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
|
|
|
|||
15
.gitignore
vendored
15
.gitignore
vendored
|
|
@ -1,7 +1,7 @@
|
|||
### Python template
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
__pycache__
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
|
|
@ -27,6 +27,8 @@ share/python-wheels/
|
|||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
metagpt/tools/schemas/
|
||||
examples/data/search_kb/*.json
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python scripts from a template
|
||||
|
|
@ -151,9 +153,14 @@ allure-results
|
|||
.vscode
|
||||
|
||||
key.yaml
|
||||
data
|
||||
/data/
|
||||
data.ms
|
||||
examples/nb/
|
||||
examples/default__vector_store.json
|
||||
examples/docstore.json
|
||||
examples/graph_store.json
|
||||
examples/image__vector_store.json
|
||||
examples/index_store.json
|
||||
.chroma
|
||||
*~$*
|
||||
workspace/*
|
||||
|
|
@ -168,6 +175,7 @@ output
|
|||
tmp.png
|
||||
.dependencies.json
|
||||
tests/metagpt/utils/file_repo_git
|
||||
tests/data/rsp_cache_new.json
|
||||
*.tmp
|
||||
*.png
|
||||
htmlcov
|
||||
|
|
@ -178,4 +186,5 @@ cov.xml
|
|||
*.faiss
|
||||
*-structure.csv
|
||||
*-structure.json
|
||||
metagpt/tools/schemas
|
||||
*.dot
|
||||
.python-version
|
||||
|
|
|
|||
3
MANIFEST.in
Normal file
3
MANIFEST.in
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
recursive-include metagpt/ext/stanford_town/prompts *.txt
|
||||
recursive-include metagpt/ext/stanford_town/static_dirs *.csv
|
||||
recursive-include metagpt/ext/stanford_town/static_dirs *.json
|
||||
99
README.md
99
README.md
|
|
@ -26,7 +26,9 @@ # MetaGPT: The Multi-Agent Framework
|
|||
</p>
|
||||
|
||||
## News
|
||||
🚀 March. 01, 2024: Our Data Interpreter paper is on arxiv. Find all design and benchmark details [here](https://arxiv.org/abs/2402.18679)!
|
||||
🚀 Mar. 29, 2024: [v0.8.0](https://github.com/geekan/MetaGPT/releases/tag/v0.8.0) released. Now you can use Data Interpreter via pypi package import. Meanwhile, we integrated RAG module and supported multiple new LLMs.
|
||||
|
||||
🚀 Mar. 14, 2024: Our **Data Interpreter** paper is on [arxiv](https://arxiv.org/abs/2402.18679). Check the [example](https://docs.deepwisdom.ai/main/en/DataInterpreter/) and [code](https://github.com/geekan/MetaGPT/tree/main/examples/di)!
|
||||
|
||||
🚀 Feb. 08, 2024: [v0.7.0](https://github.com/geekan/MetaGPT/releases/tag/v0.7.0) released, supporting assigning different LLMs to different Roles. We also introduced [Data Interpreter](https://github.com/geekan/MetaGPT/blob/main/examples/di/README.md), a powerful agent capable of solving a wide range of real-world problems.
|
||||
|
||||
|
|
@ -55,40 +57,49 @@ ## Software Company as Multi-Agent System
|
|||
|
||||
<p align="center">Software Company Multi-Agent Schematic (Gradually Implementing)</p>
|
||||
|
||||
## Install
|
||||
## Get Started
|
||||
|
||||
### Pip installation
|
||||
### Installation
|
||||
|
||||
> Ensure that Python 3.9+ is installed on your system. You can check this by using: `python --version`.
|
||||
> You can use conda like this: `conda create -n metagpt python=3.9 && conda activate metagpt`
|
||||
|
||||
```bash
|
||||
pip install metagpt
|
||||
# https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html
|
||||
metagpt --init-config # it will create ~/.metagpt/config2.yaml, just modify it to your needs
|
||||
pip install --upgrade metagpt
|
||||
# or `pip install --upgrade git+https://github.com/geekan/MetaGPT.git`
|
||||
# or `git clone https://github.com/geekan/MetaGPT && cd MetaGPT && pip install --upgrade -e .`
|
||||
```
|
||||
|
||||
For detailed installation guidance, please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version)
|
||||
or [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker)
|
||||
|
||||
### Configuration
|
||||
|
||||
You can init the config of MetaGPT by running the following command, or manually create `~/.metagpt/config2.yaml` file:
|
||||
```bash
|
||||
# Check https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html for more details
|
||||
metagpt --init-config # it will create ~/.metagpt/config2.yaml, just modify it to your needs
|
||||
```
|
||||
|
||||
You can configure `~/.metagpt/config2.yaml` according to the [example](https://github.com/geekan/MetaGPT/blob/main/config/config2.example.yaml) and [doc](https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html):
|
||||
|
||||
```yaml
|
||||
llm:
|
||||
api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options
|
||||
model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
|
||||
model: "gpt-4-turbo" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
|
||||
base_url: "https://api.openai.com/v1" # or forward url / other llm url
|
||||
api_key: "YOUR_API_KEY"
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
After installation, you can use it as CLI
|
||||
After installation, you can use MetaGPT at CLI
|
||||
|
||||
```bash
|
||||
metagpt "Create a 2048 game" # this will create a repo in ./workspace
|
||||
```
|
||||
|
||||
or you can use it as library
|
||||
or use it as library
|
||||
|
||||
```python
|
||||
from metagpt.software_company import generate_repo, ProjectRepo
|
||||
|
|
@ -96,47 +107,19 @@ ### Usage
|
|||
print(repo) # it will print the repo structure with files
|
||||
```
|
||||
|
||||
detail installation please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version)
|
||||
or [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker)
|
||||
You can also use its [Data Interpreter](https://github.com/geekan/MetaGPT/tree/main/examples/di)
|
||||
|
||||
### Docker installation
|
||||
<details><summary><strong>⏬ Step 1: Download metagpt image and prepare config2.yaml </strong><i>:: click to expand ::</i></summary>
|
||||
<div>
|
||||
```python
|
||||
import asyncio
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
|
||||
```bash
|
||||
docker pull metagpt/metagpt:latest
|
||||
mkdir -p /opt/metagpt/{config,workspace}
|
||||
docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml
|
||||
vim /opt/metagpt/config/config2.yaml # Change the config
|
||||
async def main():
|
||||
di = DataInterpreter()
|
||||
await di.run("Run data analysis on sklearn Iris dataset, include a plot")
|
||||
|
||||
asyncio.run(main()) # or await main() in a jupyter notebook setting
|
||||
```
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details><summary><strong>⏬ Step 2: Run metagpt container </strong><i>:: click to expand ::</i></summary>
|
||||
<div>
|
||||
|
||||
```bash
|
||||
docker run --name metagpt -d \
|
||||
--privileged \
|
||||
-v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
|
||||
-v /opt/metagpt/workspace:/app/metagpt/workspace \
|
||||
metagpt/metagpt:latest
|
||||
```
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details><summary><strong>⏬ Step 3: Use metagpt </strong><i>:: click to expand ::</i></summary>
|
||||
<div>
|
||||
|
||||
```bash
|
||||
docker exec -it metagpt /bin/bash
|
||||
$ metagpt "Create a 2048 game" # this will create a repo in ./workspace
|
||||
```
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
### QuickStart & Demo Video
|
||||
- Try it on [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT)
|
||||
|
|
@ -156,6 +139,7 @@ ## Tutorial
|
|||
- 🧑💻 Contribution
|
||||
- [Develop Roadmap](docs/ROADMAP.md)
|
||||
- 🔖 Use Cases
|
||||
- [Data Interpreter](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/intro.html)
|
||||
- [Debate](https://docs.deepwisdom.ai/main/en/guide/use_cases/multi_agent/debate.html)
|
||||
- [Researcher](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/researcher.html)
|
||||
- [Recepit Assistant](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/receipt_assistant.html)
|
||||
|
|
@ -163,10 +147,13 @@ ## Tutorial
|
|||
|
||||
## Support
|
||||
|
||||
### Discard Join US
|
||||
📢 Join Our [Discord Channel](https://discord.gg/ZRHeExS6xv)!
|
||||
### Discord Join US
|
||||
|
||||
Looking forward to seeing you there! 🎉
|
||||
📢 Join Our [Discord Channel](https://discord.gg/ZRHeExS6xv)! Looking forward to seeing you there! 🎉
|
||||
|
||||
### Contributor form
|
||||
|
||||
📝 [Fill out the form](https://airtable.com/appInfdG0eJ9J4NNL/pagK3Fh1sGclBvVkV/form) to become a contributor. We are looking forward to your participation!
|
||||
|
||||
### Contact Information
|
||||
|
||||
|
|
@ -179,7 +166,9 @@ ### Contact Information
|
|||
|
||||
## Citation
|
||||
|
||||
For now, cite the [arXiv paper](https://arxiv.org/abs/2308.00352):
|
||||
To stay updated with the latest research and development, follow [@MetaGPT_](https://twitter.com/MetaGPT_) on Twitter.
|
||||
|
||||
To cite [MetaGPT](https://arxiv.org/abs/2308.00352) or [Data Interpreter](https://arxiv.org/abs/2402.18679) in publications, please use the following BibTeX entries.
|
||||
|
||||
```bibtex
|
||||
@misc{hong2023metagpt,
|
||||
|
|
@ -190,4 +179,14 @@ ## Citation
|
|||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
@misc{hong2024data,
|
||||
title={Data Interpreter: An LLM Agent For Data Science},
|
||||
author={Sirui Hong and Yizhang Lin and Bang Liu and Bangbang Liu and Binhao Wu and Danyang Li and Jiaqi Chen and Jiayi Zhang and Jinlin Wang and Li Zhang and Lingyao Zhang and Min Yang and Mingchen Zhuge and Taicheng Guo and Tuo Zhou and Wei Tao and Wenyi Wang and Xiangru Tang and Xiangtao Lu and Xiawu Zheng and Xinbing Liang and Yaying Fei and Yuheng Cheng and Zongze Xu and Chenglin Wu},
|
||||
year={2024},
|
||||
eprint={2402.18679},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ ## Supported Versions
|
|||
|
||||
| Version | Supported |
|
||||
|---------|--------------------|
|
||||
| 7.x | :x: |
|
||||
| 6.x | :x: |
|
||||
| < 6.x | :x: |
|
||||
| 0.7.x | :x: |
|
||||
| 0.6.x | :x: |
|
||||
| < 0.6.x | :x: |
|
||||
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ llm:
|
|||
api_key: "YOUR_API_KEY"
|
||||
model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
|
||||
proxy: "YOUR_PROXY" # for LLM API requests
|
||||
# timeout: 600 # Optional. If set to 0, default value is 300.
|
||||
pricing_plan: "" # Optional. If invalid, it will be automatically filled in with the value of the `model`.
|
||||
# Azure-exclusive pricing plan mappings：
|
||||
# - gpt-3.5-turbo 4k: "gpt-3.5-turbo-1106"
|
||||
|
|
@ -12,6 +13,16 @@ llm:
|
|||
# - gpt-4 8k: "gpt-4"
|
||||
# See for more: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/
|
||||
|
||||
# RAG Embedding.
|
||||
# For backward compatibility, if the embedding is not set and the llm's api_type is either openai or azure, the llm's config will be used.
|
||||
embedding:
|
||||
api_type: "" # openai / azure / gemini / ollama etc. Check EmbeddingType for more options.
|
||||
base_url: ""
|
||||
api_key: ""
|
||||
model: ""
|
||||
api_version: ""
|
||||
embed_batch_size: 100
|
||||
|
||||
repair_llm_output: true # when the output is not a valid json, try to repair it
|
||||
|
||||
proxy: "YOUR_PROXY" # for tools like requests, playwright, selenium, etc.
|
||||
|
|
|
|||
|
|
@ -2,6 +2,6 @@
|
|||
# Reflected Code: https://github.com/geekan/MetaGPT/blob/main/metagpt/config2.py
|
||||
llm:
|
||||
api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options
|
||||
model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
|
||||
model: "gpt-4-turbo" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
|
||||
base_url: "https://api.openai.com/v1" # or forward url / other llm url
|
||||
api_key: "YOUR_API_KEY"
|
||||
|
|
@ -116,7 +116,7 @@ ### 联系信息
|
|||
|
||||
## 引用
|
||||
|
||||
引用 [arXiv paper](https://arxiv.org/abs/2308.00352):
|
||||
如果您在研究论文中使用 MetaGPT 或 Data Interpreter，请引用我们的工作：
|
||||
|
||||
```bibtex
|
||||
@misc{hong2023metagpt,
|
||||
|
|
@ -127,4 +127,12 @@ ## 引用
|
|||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
@misc{hong2024data,
|
||||
title={Data Interpreter: An LLM Agent For Data Science},
|
||||
author={Sirui Hong and Yizhang Lin and Bang Liu and Bangbang Liu and Binhao Wu and Danyang Li and Jiaqi Chen and Jiayi Zhang and Jinlin Wang and Li Zhang and Lingyao Zhang and Min Yang and Mingchen Zhuge and Taicheng Guo and Tuo Zhou and Wei Tao and Wenyi Wang and Xiangru Tang and Xiangtao Lu and Xiawu Zheng and Xinbing Liang and Yaying Fei and Yuheng Cheng and Zongze Xu and Chenglin Wu},
|
||||
year={2024},
|
||||
eprint={2402.18679},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ ## クイックスタート
|
|||
|
||||
## 引用
|
||||
|
||||
現時点では、[arXiv 論文](https://arxiv.org/abs/2308.00352)を引用してください:
|
||||
研究論文でMetaGPTやData Interpreterを使用する場合は、以下のように当社の作業を引用してください：
|
||||
|
||||
```bibtex
|
||||
@misc{hong2023metagpt,
|
||||
|
|
@ -306,6 +306,14 @@ ## 引用
|
|||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
@misc{hong2024data,
|
||||
title={Data Interpreter: An LLM Agent For Data Science},
|
||||
author={Sirui Hong and Yizhang Lin and Bang Liu and Bangbang Liu and Binhao Wu and Danyang Li and Jiaqi Chen and Jiayi Zhang and Jinlin Wang and Li Zhang and Lingyao Zhang and Min Yang and Mingchen Zhuge and Taicheng Guo and Tuo Zhou and Wei Tao and Wenyi Wang and Xiangru Tang and Xiangtao Lu and Xiawu Zheng and Xinbing Liang and Yaying Fei and Yuheng Cheng and Zongze Xu and Chenglin Wu},
|
||||
year={2024},
|
||||
eprint={2402.18679},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.AI}
|
||||
}
|
||||
```
|
||||
|
||||
## お問い合わせ先
|
||||
|
|
|
|||
2
examples/android_assistant/requirements.txt
Normal file
2
examples/android_assistant/requirements.txt
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
pyshine==0.0.9
|
||||
opencv-python==4.6.0.66
|
||||
71
examples/android_assistant/run_assistant.py
Normal file
71
examples/android_assistant/run_assistant.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Desc : the entry of android assistant including learning and acting stage
|
||||
# See the usage README inside `metagpt/ext/android_assistant`
|
||||
# README see `metagpt/ext/android_assistant/README.md`
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
|
||||
from metagpt.config2 import config
|
||||
from metagpt.environment.android.android_env import AndroidEnv
|
||||
from metagpt.ext.android_assistant.roles.android_assistant import AndroidAssistant
|
||||
from metagpt.team import Team
|
||||
|
||||
app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False)
|
||||
|
||||
|
||||
@app.command("", help="Run a Android Assistant")
|
||||
def startup(
|
||||
task_desc: str = typer.Argument(help="the task description you want the android assistant to learn or act"),
|
||||
n_round: int = typer.Option(default=20, help="The max round to do an app operation task."),
|
||||
stage: str = typer.Option(default="learn", help="stage: learn / act"),
|
||||
mode: str = typer.Option(default="auto", help="mode: auto / manual , when state=learn"),
|
||||
app_name: str = typer.Option(default="demo", help="the name of app you want to run"),
|
||||
investment: float = typer.Option(default=5.0, help="Dollar amount to invest in the AI company."),
|
||||
refine_doc: bool = typer.Option(
|
||||
default=False, help="Refine existing operation docs based on the latest observation if True."
|
||||
),
|
||||
min_dist: int = typer.Option(
|
||||
default=30, help="The minimum distance between elements to prevent overlapping during the labeling process."
|
||||
),
|
||||
android_screenshot_dir: str = typer.Option(
|
||||
default="/sdcard/Pictures/Screenshots",
|
||||
help="The path to store screenshots on android device. Make sure it exists.",
|
||||
),
|
||||
android_xml_dir: str = typer.Option(
|
||||
default="/sdcard",
|
||||
help="The path to store xml files for determining UI elements localtion. Make sure it exists.",
|
||||
),
|
||||
device_id: str = typer.Option(default="emulator-5554", help="The Android device_id"),
|
||||
):
|
||||
config.extra = {
|
||||
"stage": stage,
|
||||
"mode": mode,
|
||||
"app_name": app_name,
|
||||
"task_desc": task_desc,
|
||||
"refine_doc": refine_doc,
|
||||
"min_dist": min_dist,
|
||||
"android_screenshot_dir": android_screenshot_dir,
|
||||
"android_xml_dir": android_xml_dir,
|
||||
"device_id": device_id,
|
||||
}
|
||||
|
||||
team = Team(
|
||||
env=AndroidEnv(
|
||||
device_id=device_id,
|
||||
xml_dir=Path(android_xml_dir),
|
||||
screenshot_dir=Path(android_screenshot_dir),
|
||||
)
|
||||
)
|
||||
|
||||
team.hire([AndroidAssistant(output_root_dir=Path(__file__).parent)])
|
||||
team.invest(investment)
|
||||
team.run_project(idea=task_desc)
|
||||
asyncio.run(team.run(n_round=n_round))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
|
|
@ -17,7 +17,7 @@ from metagpt.schema import Message
|
|||
|
||||
class SimpleWriteCode(Action):
|
||||
PROMPT_TEMPLATE: str = """
|
||||
Write a python function that can {instruction} and provide two runnnable test cases.
|
||||
Write a python function that can {instruction} and provide two runnable test cases.
|
||||
Return ```python your_code_here ``` with NO other texts,
|
||||
your code:
|
||||
"""
|
||||
|
|
|
|||
1
examples/data/rag/travel.txt
Normal file
1
examples/data/rag/travel.txt
Normal file
|
|
@ -0,0 +1 @@
|
|||
Bob likes traveling.
|
||||
109
examples/data/rag/writer.txt
Normal file
109
examples/data/rag/writer.txt
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
Productivity
|
||||
I think I am at least somewhat more productive than average, and people sometimes ask me for productivity tips. So I decided to just write them all down in one place.
|
||||
|
||||
Compound growth gets discussed as a financial concept, but it works in careers as well, and it is magic. A small productivity gain, compounded over 50 years, is worth a lot. So it’s worth figuring out how to optimize productivity. If you get 10% more done and 1% better every day compared to someone else, the compounded difference is massive.
|
||||
|
||||
What you work on
|
||||
|
||||
Famous writers have some essential qualities, creativity and discipline
|
||||
|
||||
It doesn’t matter how fast you move if it’s in a worthless direction. Picking the right thing to work on is the most important element of productivity and usually almost ignored. So think about it more! Independent thought is hard but it’s something you can get better at with practice.
|
||||
|
||||
The most impressive people I know have strong beliefs about the world, which is rare in the general population. If you find yourself always agreeing with whomever you last spoke with, that’s bad. You will of course be wrong sometimes, but develop the confidence to stick with your convictions. It will let you be courageous when you’re right about something important that most people don’t see.
|
||||
|
||||
I make sure to leave enough time in my schedule to think about what to work on. The best ways for me to do this are reading books, hanging out with interesting people, and spending time in nature.
|
||||
|
||||
I’ve learned that I can’t be very productive working on things I don’t care about or don’t like. So I just try not to put myself in a position where I have to do them (by delegating, avoiding, or something else). Stuff that you don’t like is a painful drag on morale and momentum.
|
||||
|
||||
By the way, here is an important lesson about delegation: remember that everyone else is also most productive when they’re doing what they like, and do what you’d want other people to do for you—try to figure out who likes (and is good at) doing what, and delegate that way.
|
||||
|
||||
If you find yourself not liking what you’re doing for a long period of time, seriously consider a major job change. Short-term burnout happens, but if it isn’t resolved with some time off, maybe it’s time to do something you’re more interested in.
|
||||
|
||||
I’ve been very fortunate to find work I like so much I’d do it for free, which makes it easy to be really productive.
|
||||
|
||||
It’s important to learn that you can learn anything you want, and that you can get better quickly. This feels like an unlikely miracle the first few times it happens, but eventually you learn to trust that you can do it.
|
||||
|
||||
Doing great work usually requires colleagues of some sort. Try to be around smart, productive, happy, and positive people that don’t belittle your ambitions. I love being around people who push me and inspire me to be better. To the degree you able to, avoid the opposite kind of people—the cost of letting them take up your mental cycles is horrific.
|
||||
|
||||
You have to both pick the right problem and do the work. There aren’t many shortcuts. If you’re going to do something really important, you are very likely going to work both smart and hard. The biggest prizes are heavily competed for. This isn’t true in every field (there are great mathematicians who never spend that many hours a week working) but it is in most.
|
||||
|
||||
Prioritization
|
||||
|
||||
Writers have to work hard to be successful
|
||||
|
||||
My system has three key pillars: “Make sure to get the important shit done”, “Don’t waste time on stupid shit”, and “make a lot of lists”.
|
||||
|
||||
I highly recommend using lists. I make lists of what I want to accomplish each year, each month, and each day. Lists are very focusing, and they help me with multitasking because I don’t have to keep as much in my head. If I’m not in the mood for some particular task, I can always find something else I’m excited to do.
|
||||
|
||||
I prefer lists written down on paper. It’s easy to add and remove tasks. I can access them during meetings without feeling rude. I re-transcribe lists frequently, which forces me to think about everything on the list and gives me an opportunity to add and remove items.
|
||||
|
||||
I don’t bother with categorization or trying to size tasks or anything like that (the most I do is put a star next to really important items).
|
||||
|
||||
I try to prioritize in a way that generates momentum. The more I get done, the better I feel, and then the more I get done. I like to start and end each day with something I can really make progress on.
|
||||
|
||||
I am relentless about getting my most important projects done—I’ve found that if I really want something to happen and I push hard enough, it usually happens.
|
||||
|
||||
I try to be ruthless about saying no to stuff, and doing non-critical things in the quickest way possible. I probably take this too far—for example, I am almost sure I am terse to the point of rudeness when replying to emails.
|
||||
|
||||
Passion and adaptability are key qualities to writers
|
||||
|
||||
I generally try to avoid meetings and conferences as I find the time cost to be huge—I get the most value out of time in my office. However, it is critical that you keep enough space in your schedule to allow for chance encounters and exposure to new people and ideas. Having an open network is valuable; though probably 90% of the random meetings I take are a waste of time, the other 10% really make up for it.
|
||||
|
||||
I find most meetings are best scheduled for 15-20 minutes, or 2 hours. The default of 1 hour is usually wrong, and leads to a lot of wasted time.
|
||||
|
||||
I have different times of day I try to use for different kinds of work. The first few hours of the morning are definitely my most productive time of the day, so I don’t let anyone schedule anything then. I try to do meetings in the afternoon. I take a break, or switch tasks, whenever I feel my attention starting to fade.
|
||||
|
||||
I don’t think most people value their time enough—I am surprised by the number of people I know who make $100 an hour and yet will spend a couple of hours doing something they don’t want to do to save $20.
|
||||
|
||||
Also, don’t fall into the trap of productivity porn—chasing productivity for its own sake isn’t helpful. Many people spend too much time thinking about how to perfectly optimize their system, and not nearly enough asking if they’re working on the right problems. It doesn’t matter what system you use or if you squeeze out every second if you’re working on the wrong thing.
|
||||
|
||||
The right goal is to allocate your year optimally, not your day.
|
||||
|
||||
Physical factors
|
||||
|
||||
Very likely what is optimal for me won’t be optimal for you. You’ll have to experiment to find out what works best for your body. It’s definitely worth doing—it helps in all aspects of life, and you’ll feel a lot better and happier overall.
|
||||
|
||||
It probably took a little bit of my time every week for a few years to arrive at what works best for me, but my sense is if I do a good job at all the below I’m at least 1.5x more productive than if not.
|
||||
|
||||
Sleep seems to be the most important physical factor in productivity for me. Some sort of sleep tracker to figure out how to sleep best is helpful. I’ve found the only thing I’m consistent with are in the set-it-and-forget-it category, and I really like the Emfit QS+Active.
|
||||
|
||||
I like a cold, dark, quiet room, and a great mattress (I resisted spending a bunch of money on a great mattress for years, which was stupid—it makes a huge difference to my sleep quality. I love this one). Not eating a lot in the few hours before sleep helps. Not drinking alcohol helps a lot, though I’m not willing to do that all the time.
|
||||
|
||||
I use a Chili Pad to be cold while I sleep if I can’t get the room cold enough, which is great but loud (I set it up to have the cooler unit outside my room).
|
||||
|
||||
When traveling, I use an eye mask and ear plugs.
|
||||
|
||||
Writers usually have empathy to write good books.
|
||||
|
||||
This is likely to be controversial, but I take a low dose of sleeping pills (like a third of a normal dose) or a very low dose of cannabis whenever I can’t sleep. I am a bad sleeper in general, and a particularly bad sleeper when I travel. It likely has tradeoffs, but so does not sleeping well. If you can already sleep well, I wouldn’t recommend this.
|
||||
|
||||
I use a full spectrum LED light most mornings for about 10-15 minutes while I catch up on email. It’s great—if you try nothing else in here, this is the thing I’d try. It’s a ridiculous gain for me. I like this one, and it’s easy to travel with.
|
||||
|
||||
Exercise is probably the second most important physical factor. I tried a number of different exercise programs for a few months each and the one that seemed best was lifting heavy weights 3x a week for an hour, and high intensity interval training occasionally. In addition to productivity gains, this is also the exercise program that makes me feel the best overall.
|
||||
|
||||
The third area is nutrition. I very rarely eat breakfast, so I get about 15 hours of fasting most days (except an espresso when I wake up). I know this is contrary to most advice, and I suspect it’s not optimal for most people, but it definitely works well for me.
|
||||
|
||||
Eating lots of sugar is the thing that makes me feel the worst and that I try hardest to avoid. I also try to avoid foods that aggravate my digestion or spike up inflammation (for example, very spicy foods). I don’t have much willpower when it comes to sweet things, so I mostly just try to keep junk food out of the house.
|
||||
|
||||
I have one big shot of espresso immediately when I wake up and one after lunch. I assume this is about 200mg total of caffeine per day. I tried a few other configurations; this was the one that worked by far the best. I otherwise aggressively avoid stimulants, but I will have more coffee if I’m super tired and really need to get something done.
|
||||
|
||||
If a writer want to be super, then should include innovative thinking.
|
||||
|
||||
I’m vegetarian and have been since I was a kid, and I supplement methyl B-12, Omega-3, Iron, and Vitamin D-3. I got to this list with a year or so of quarterly blood tests; it’s worked for me ever since (I re-test maybe every year and a half or so). There are many doctors who will happily work with you on a super comprehensive blood test (and services like WellnessFX). I also go out of my way to drink a lot of protein shakes, which I hate and I wouldn’t do if I weren’t vegetarian.
|
||||
|
||||
Other stuff
|
||||
|
||||
Here’s what I like in a workspace: natural light, quiet, knowing that I won’t be interrupted if I don’t want to be, long blocks of time, and being comfortable and relaxed (I’ve got a beautiful desk with a couple of 4k monitors on it in my office, but I spend almost all my time on my couch with my laptop).
|
||||
|
||||
I wrote custom software for the annoying things I have to do frequently, which is great. I also made an effort to learn to type really fast and the keyboard shortcuts that help with my workflow.
|
||||
|
||||
Like most people, I sometimes go through periods of a week or two where I just have no motivation to do anything (I suspect it may have something to do with nutrition). This sucks and always seems to happen at inconvenient times. I have not figured out what to do about it besides wait for the fog to lift, and to trust that eventually it always does. And I generally try to avoid people and situations that put me in bad moods, which is good advice whether you care about productivity or not.
|
||||
|
||||
In general, I think it’s good to overcommit a little bit. I find that I generally get done what I take on, and if I have a little bit too much to do it makes me more efficient at everything, which is a way to train to avoid distractions (a great habit to build!). However, overcommitting a lot is disastrous.
|
||||
|
||||
Don’t neglect your family and friends for the sake of productivity—that’s a very stupid tradeoff (and very likely a net productivity loss, because you’ll be less happy). Don’t neglect doing things you love or that clear your head either.
|
||||
|
||||
Finally, to repeat one more time: productivity in the wrong direction isn’t worth anything at all. Think more about what to work on.
|
||||
|
||||
Open-Mindedness and curiosity are essential to writers
|
||||
|
||||
21
examples/di/arxiv_reader.py
Normal file
21
examples/di/arxiv_reader.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
|
||||
|
||||
async def main():
|
||||
template = "https://arxiv.org/list/{tag}/pastweek?skip=0&show=300"
|
||||
tags = ["cs.ai", "cs.cl", "cs.lg", "cs.se"]
|
||||
urls = [template.format(tag=tag) for tag in tags]
|
||||
prompt = f"""This is a collection of arxiv urls: '{urls}' .
|
||||
Record each article, remove duplicates by title (they may have multiple tags), filter out papers related to
|
||||
large language model / agent / llm, print top 100 and visualize the word count of the titles"""
|
||||
di = DataInterpreter(react_mode="react", tools=["scrape_web_playwright"])
|
||||
|
||||
await di.run(prompt)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
asyncio.run(main())
|
||||
36
examples/di/custom_tool.py
Normal file
36
examples/di/custom_tool.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
@Time : 2024/3/22 10:54
|
||||
@Author : alexanderwu
|
||||
@File : custom_tool.py
|
||||
"""
|
||||
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
from metagpt.tools.tool_registry import register_tool
|
||||
|
||||
|
||||
@register_tool()
|
||||
def magic_function(arg1: str, arg2: int) -> dict:
|
||||
"""
|
||||
The magic function that does something.
|
||||
|
||||
Args:
|
||||
arg1 (str): ...
|
||||
arg2 (int): ...
|
||||
|
||||
Returns:
|
||||
dict: ...
|
||||
"""
|
||||
return {"arg1": arg1 * 3, "arg2": arg2 * 5}
|
||||
|
||||
|
||||
async def main():
|
||||
di = DataInterpreter(tools=["magic_function"])
|
||||
await di.run("Just call the magic function with arg1 'A' and arg2 2. Tell me the result.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
asyncio.run(main())
|
||||
|
|
@ -1,14 +1,17 @@
|
|||
import asyncio
|
||||
|
||||
from metagpt.logs import logger
|
||||
from metagpt.roles.di.data_interpreter import DataInterpreter
|
||||
from metagpt.utils.recovery_util import save_history
|
||||
|
||||
|
||||
async def main(requirement: str = ""):
|
||||
di = DataInterpreter()
|
||||
await di.run(requirement)
|
||||
rsp = await di.run(requirement)
|
||||
logger.info(rsp)
|
||||
save_history(role=di)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
requirement = "Run data analysis on sklearn Iris dataset, include a plot"
|
||||
|
||||
asyncio.run(main(requirement))
|
||||
|
|
|
|||
247
examples/rag_pipeline.py
Normal file
247
examples/rag_pipeline.py
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
"""RAG pipeline"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from metagpt.const import DATA_PATH, EXAMPLE_DATA_PATH
|
||||
from metagpt.logs import logger
|
||||
from metagpt.rag.engines import SimpleEngine
|
||||
from metagpt.rag.schema import (
|
||||
ChromaIndexConfig,
|
||||
ChromaRetrieverConfig,
|
||||
ElasticsearchIndexConfig,
|
||||
ElasticsearchRetrieverConfig,
|
||||
ElasticsearchStoreConfig,
|
||||
FAISSRetrieverConfig,
|
||||
LLMRankerConfig,
|
||||
)
|
||||
from metagpt.utils.exceptions import handle_exception
|
||||
|
||||
DOC_PATH = EXAMPLE_DATA_PATH / "rag/writer.txt"
|
||||
QUESTION = "What are key qualities to be a good writer?"
|
||||
|
||||
TRAVEL_DOC_PATH = EXAMPLE_DATA_PATH / "rag/travel.txt"
|
||||
TRAVEL_QUESTION = "What does Bob like?"
|
||||
|
||||
LLM_TIP = "If you not sure, just answer I don't know."
|
||||
|
||||
|
||||
class Player(BaseModel):
|
||||
"""To demonstrate rag add objs."""
|
||||
|
||||
name: str = ""
|
||||
goal: str = "Win The 100-meter Sprint."
|
||||
tool: str = "Red Bull Energy Drink."
|
||||
|
||||
def rag_key(self) -> str:
|
||||
"""For search"""
|
||||
return self.goal
|
||||
|
||||
|
||||
class RAGExample:
|
||||
"""Show how to use RAG."""
|
||||
|
||||
def __init__(self, engine: SimpleEngine = None):
|
||||
self._engine = engine
|
||||
|
||||
@property
|
||||
def engine(self):
|
||||
if not self._engine:
|
||||
self._engine = SimpleEngine.from_docs(
|
||||
input_files=[DOC_PATH],
|
||||
retriever_configs=[FAISSRetrieverConfig()],
|
||||
ranker_configs=[LLMRankerConfig()],
|
||||
)
|
||||
return self._engine
|
||||
|
||||
@engine.setter
|
||||
def engine(self, value: SimpleEngine):
|
||||
self._engine = value
|
||||
|
||||
async def run_pipeline(self, question=QUESTION, print_title=True):
|
||||
"""This example run rag pipeline, use faiss retriever and llm ranker, will print something like:
|
||||
|
||||
Retrieve Result:
|
||||
0. Productivi..., 10.0
|
||||
1. I wrote cu..., 7.0
|
||||
2. I highly r..., 5.0
|
||||
|
||||
Query Result:
|
||||
Passion, adaptability, open-mindedness, creativity, discipline, and empathy are key qualities to be a good writer.
|
||||
"""
|
||||
if print_title:
|
||||
self._print_title("Run Pipeline")
|
||||
|
||||
nodes = await self.engine.aretrieve(question)
|
||||
self._print_retrieve_result(nodes)
|
||||
|
||||
answer = await self.engine.aquery(question)
|
||||
self._print_query_result(answer)
|
||||
|
||||
async def add_docs(self):
|
||||
"""This example show how to add docs.
|
||||
|
||||
Before add docs llm anwser I don't know.
|
||||
After add docs llm give the correct answer, will print something like:
|
||||
|
||||
[Before add docs]
|
||||
Retrieve Result:
|
||||
|
||||
Query Result:
|
||||
Empty Response
|
||||
|
||||
[After add docs]
|
||||
Retrieve Result:
|
||||
0. Bob like..., 10.0
|
||||
|
||||
Query Result:
|
||||
Bob likes traveling.
|
||||
"""
|
||||
self._print_title("Add Docs")
|
||||
|
||||
travel_question = f"{TRAVEL_QUESTION}{LLM_TIP}"
|
||||
travel_filepath = TRAVEL_DOC_PATH
|
||||
|
||||
logger.info("[Before add docs]")
|
||||
await self.run_pipeline(question=travel_question, print_title=False)
|
||||
|
||||
logger.info("[After add docs]")
|
||||
self.engine.add_docs([travel_filepath])
|
||||
await self.run_pipeline(question=travel_question, print_title=False)
|
||||
|
||||
@handle_exception
|
||||
async def add_objects(self, print_title=True):
|
||||
"""This example show how to add objects.
|
||||
|
||||
Before add docs, engine retrieve nothing.
|
||||
After add objects, engine give the correct answer, will print something like:
|
||||
|
||||
[Before add objs]
|
||||
Retrieve Result:
|
||||
|
||||
[After add objs]
|
||||
Retrieve Result:
|
||||
0. 100m Sprin..., 10.0
|
||||
|
||||
[Object Detail]
|
||||
{'name': 'Mike', 'goal': 'Win The 100-meter Sprint', 'tool': 'Red Bull Energy Drink'}
|
||||
"""
|
||||
if print_title:
|
||||
self._print_title("Add Objects")
|
||||
|
||||
player = Player(name="Mike")
|
||||
question = f"{player.rag_key()}"
|
||||
|
||||
logger.info("[Before add objs]")
|
||||
await self._retrieve_and_print(question)
|
||||
|
||||
logger.info("[After add objs]")
|
||||
self.engine.add_objs([player])
|
||||
|
||||
try:
|
||||
nodes = await self._retrieve_and_print(question)
|
||||
|
||||
logger.info("[Object Detail]")
|
||||
player: Player = nodes[0].metadata["obj"]
|
||||
logger.info(player.name)
|
||||
except Exception as e:
|
||||
logger.error(f"nodes is empty, llm don't answer correctly, exception: {e}")
|
||||
|
||||
async def init_objects(self):
|
||||
"""This example show how to from objs, will print something like:
|
||||
|
||||
Same as add_objects.
|
||||
"""
|
||||
self._print_title("Init Objects")
|
||||
|
||||
pre_engine = self.engine
|
||||
self.engine = SimpleEngine.from_objs(retriever_configs=[FAISSRetrieverConfig()])
|
||||
await self.add_objects(print_title=False)
|
||||
self.engine = pre_engine
|
||||
|
||||
async def init_and_query_chromadb(self):
|
||||
"""This example show how to use chromadb. how to save and load index. will print something like:
|
||||
|
||||
Query Result:
|
||||
Bob likes traveling.
|
||||
"""
|
||||
self._print_title("Init And Query ChromaDB")
|
||||
|
||||
# 1. save index
|
||||
output_dir = DATA_PATH / "rag"
|
||||
SimpleEngine.from_docs(
|
||||
input_files=[TRAVEL_DOC_PATH],
|
||||
retriever_configs=[ChromaRetrieverConfig(persist_path=output_dir)],
|
||||
)
|
||||
|
||||
# 2. load index
|
||||
engine = SimpleEngine.from_index(index_config=ChromaIndexConfig(persist_path=output_dir))
|
||||
|
||||
# 3. query
|
||||
answer = await engine.aquery(TRAVEL_QUESTION)
|
||||
self._print_query_result(answer)
|
||||
|
||||
@handle_exception
|
||||
async def init_and_query_es(self):
|
||||
"""This example show how to use es. how to save and load index. will print something like:
|
||||
|
||||
Query Result:
|
||||
Bob likes traveling.
|
||||
"""
|
||||
self._print_title("Init And Query Elasticsearch")
|
||||
|
||||
# 1. create es index and save docs
|
||||
store_config = ElasticsearchStoreConfig(index_name="travel", es_url="http://127.0.0.1:9200")
|
||||
engine = SimpleEngine.from_docs(
|
||||
input_files=[TRAVEL_DOC_PATH],
|
||||
retriever_configs=[ElasticsearchRetrieverConfig(store_config=store_config)],
|
||||
)
|
||||
|
||||
# 2. load index
|
||||
engine = SimpleEngine.from_index(index_config=ElasticsearchIndexConfig(store_config=store_config))
|
||||
|
||||
# 3. query
|
||||
answer = await engine.aquery(TRAVEL_QUESTION)
|
||||
self._print_query_result(answer)
|
||||
|
||||
@staticmethod
|
||||
def _print_title(title):
|
||||
logger.info(f"{'#'*30} {title} {'#'*30}")
|
||||
|
||||
@staticmethod
|
||||
def _print_retrieve_result(result):
|
||||
"""Print retrieve result."""
|
||||
logger.info("Retrieve Result:")
|
||||
|
||||
for i, node in enumerate(result):
|
||||
logger.info(f"{i}. {node.text[:10]}..., {node.score}")
|
||||
|
||||
logger.info("")
|
||||
|
||||
@staticmethod
|
||||
def _print_query_result(result):
|
||||
"""Print query result."""
|
||||
logger.info("Query Result:")
|
||||
|
||||
logger.info(f"{result}\n")
|
||||
|
||||
async def _retrieve_and_print(self, question):
|
||||
nodes = await self.engine.aretrieve(question)
|
||||
self._print_retrieve_result(nodes)
|
||||
return nodes
|
||||
|
||||
|
||||
async def main():
|
||||
"""RAG pipeline"""
|
||||
e = RAGExample()
|
||||
await e.run_pipeline()
|
||||
await e.add_docs()
|
||||
await e.add_objects()
|
||||
await e.init_objects()
|
||||
await e.init_and_query_chromadb()
|
||||
await e.init_and_query_es()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
21
examples/rag_search.py
Normal file
21
examples/rag_search.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""Agent with RAG search."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from examples.rag_pipeline import DOC_PATH, QUESTION
|
||||
from metagpt.logs import logger
|
||||
from metagpt.rag.engines import SimpleEngine
|
||||
from metagpt.roles import Sales
|
||||
|
||||
|
||||
async def search():
|
||||
"""Agent with RAG search."""
|
||||
|
||||
store = SimpleEngine.from_docs(input_files=[DOC_PATH])
|
||||
role = Sales(profile="Sales", store=store)
|
||||
result = await role.run(QUESTION)
|
||||
logger.info(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(search())
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
@File : search_kb.py
|
||||
@Modified By: mashenquan, 2023-12-22. Delete useless codes.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
from langchain.embeddings import OpenAIEmbeddings
|
||||
|
||||
from metagpt.config2 import config
|
||||
from metagpt.const import DATA_PATH, EXAMPLE_PATH
|
||||
from metagpt.document_store import FaissStore
|
||||
from metagpt.logs import logger
|
||||
from metagpt.roles import Sales
|
||||
|
||||
|
||||
def get_store():
|
||||
llm = config.get_openai_llm()
|
||||
embedding = OpenAIEmbeddings(openai_api_key=llm.api_key, openai_api_base=llm.base_url)
|
||||
return FaissStore(DATA_PATH / "example.json", embedding=embedding)
|
||||
|
||||
|
||||
async def search():
|
||||
store = FaissStore(EXAMPLE_PATH / "example.json")
|
||||
role = Sales(profile="Sales", store=store)
|
||||
query = "Which facial cleanser is good for oily skin?"
|
||||
result = await role.run(query)
|
||||
logger.info(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(search())
|
||||
|
|
@ -13,7 +13,7 @@ async def main():
|
|||
question = "What are the most interesting human facts?"
|
||||
|
||||
search = Config.default().search
|
||||
kwargs = {"api_key": search.api_key, "cse_id": search.cse_id, "proxy": None}
|
||||
kwargs = search.model_dump()
|
||||
await Searcher(search_engine=SearchEngine(engine=search.api_type, **kwargs)).run(question)
|
||||
|
||||
|
||||
|
|
|
|||
0
examples/stanford_town/requirements.txt
Normal file
0
examples/stanford_town/requirements.txt
Normal file
94
examples/stanford_town/run_st_game.py
Normal file
94
examples/stanford_town/run_st_game.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Desc : entry of Stanford Town(ST/st) game
|
||||
# README see `metagpt/ext/stanford_town/README.md`
|
||||
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
|
||||
import fire
|
||||
|
||||
from metagpt.ext.stanford_town.roles.st_role import STRole
|
||||
from metagpt.ext.stanford_town.stanford_town import StanfordTown
|
||||
from metagpt.ext.stanford_town.utils.const import STORAGE_PATH
|
||||
from metagpt.ext.stanford_town.utils.mg_ga_transform import (
|
||||
get_reverie_meta,
|
||||
write_curr_sim_code,
|
||||
write_curr_step,
|
||||
)
|
||||
from metagpt.ext.stanford_town.utils.utils import copy_folder
|
||||
from metagpt.logs import logger
|
||||
|
||||
|
||||
async def startup(
|
||||
idea: str, fork_sim_code: str, sim_code: str, temp_storage_path: str, investment: float = 30.0, n_round: int = 500
|
||||
):
|
||||
town = StanfordTown()
|
||||
logger.info("StanfordTown init environment")
|
||||
|
||||
# copy `storage/{fork_sim_code}` to `storage/{sim_code}`
|
||||
copy_folder(str(STORAGE_PATH.joinpath(fork_sim_code)), str(STORAGE_PATH.joinpath(sim_code)))
|
||||
|
||||
# get role names from `storage/{simulation_name}/reverie/meta.json` and then init roles
|
||||
reverie_meta = get_reverie_meta(fork_sim_code)
|
||||
roles = []
|
||||
sim_path = STORAGE_PATH.joinpath(sim_code)
|
||||
sim_path.mkdir(exist_ok=True)
|
||||
for idx, role_name in enumerate(reverie_meta["persona_names"]):
|
||||
has_inner_voice = True if idx == 0 else False
|
||||
role = STRole(
|
||||
name=role_name,
|
||||
profile=role_name,
|
||||
sim_code=sim_code,
|
||||
step=reverie_meta.get("step", 0),
|
||||
start_time=reverie_meta.get("start_date"),
|
||||
curr_time=reverie_meta.get("curr_time"),
|
||||
sec_per_step=reverie_meta.get("sec_per_step"),
|
||||
has_inner_voice=has_inner_voice,
|
||||
)
|
||||
roles.append(role)
|
||||
|
||||
# init temp_storage
|
||||
write_curr_sim_code({"sim_code": sim_code}, temp_storage_path)
|
||||
write_curr_step({"step": reverie_meta.get("step", 0)}, temp_storage_path)
|
||||
|
||||
await town.hire(roles)
|
||||
|
||||
town.invest(investment)
|
||||
town.run_project(idea)
|
||||
|
||||
await town.run(n_round)
|
||||
|
||||
|
||||
def main(
|
||||
idea: str,
|
||||
fork_sim_code: str,
|
||||
sim_code: str,
|
||||
temp_storage_path: Optional[str] = None,
|
||||
investment: float = 30.0,
|
||||
n_round: int = 500,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
idea: idea works as an `inner voice` to the first agent.
|
||||
fork_sim_code: old simulation name to start with, choose one inside `generative_agents/environment/frontend_server/storage/`
|
||||
sim_code: new simulation name to save simulation result
|
||||
temp_storage_path: generative_agents temp_storage path inside `environment/frontend_server` to interact.
|
||||
investment: the investment of running agents
|
||||
n_round: rounds to run agents
|
||||
"""
|
||||
|
||||
asyncio.run(
|
||||
startup(
|
||||
idea=idea,
|
||||
fork_sim_code=fork_sim_code,
|
||||
sim_code=sim_code,
|
||||
temp_storage_path=temp_storage_path,
|
||||
investment=investment,
|
||||
n_round=n_round,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(main)
|
||||
4
examples/stanford_town/storage/.gitignore
vendored
Normal file
4
examples/stanford_town/storage/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
# path to store simulation data
|
||||
test_*
|
||||
unittest*
|
||||
July*
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
{
|
||||
"Isabella Rodriguez": {
|
||||
"maze": "the_ville",
|
||||
"x": 72,
|
||||
"y": 14
|
||||
},
|
||||
"Klaus Mueller": {
|
||||
"maze": "the_ville",
|
||||
"x": 126,
|
||||
"y": 46
|
||||
},
|
||||
"Maria Lopez": {
|
||||
"maze": "the_ville",
|
||||
"x": 123,
|
||||
"y": 57
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
{"kw_strength_event": {},
|
||||
"kw_strength_thought": {}}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
{
|
||||
"vision_r": 8,
|
||||
"att_bandwidth": 8,
|
||||
"retention": 8,
|
||||
"curr_time": null,
|
||||
"curr_tile": null,
|
||||
"daily_plan_req": "Isabella Rodriguez opens Hobbs Cafe at 8am everyday, and works at the counter until 8pm, at which point she closes the cafe.",
|
||||
"name": "Isabella Rodriguez",
|
||||
"first_name": "Isabella",
|
||||
"last_name": "Rodriguez",
|
||||
"age": 34,
|
||||
"innate": "friendly, outgoing, hospitable",
|
||||
"learned": "Isabella Rodriguez is a cafe owner of Hobbs Cafe who loves to make people feel welcome. She is always looking for ways to make the cafe a place where people can come to relax and enjoy themselves.",
|
||||
"currently": "Isabella Rodriguez is planning on having a Valentine's Day party at Hobbs Cafe with her customers on February 14th, 2023 at 5pm. She is gathering party material, and is telling everyone to join the party at Hobbs Cafe on February 14th, 2023, from 5pm to 7pm.",
|
||||
"lifestyle": "Isabella Rodriguez goes to bed around 11pm, awakes up around 6am.",
|
||||
"living_area": "the Ville:Isabella Rodriguez's apartment:main room",
|
||||
"concept_forget": 100,
|
||||
"daily_reflection_time": 180,
|
||||
"daily_reflection_size": 5,
|
||||
"overlap_reflect_th": 4,
|
||||
"kw_strg_event_reflect_th": 10,
|
||||
"kw_strg_thought_reflect_th": 9,
|
||||
|
||||
"recency_w": 1,
|
||||
"relevance_w": 1,
|
||||
"importance_w": 1,
|
||||
"recency_decay": 0.995,
|
||||
"importance_trigger_max": 150,
|
||||
"importance_trigger_curr": 150,
|
||||
"importance_ele_n": 0,
|
||||
"thought_count": 5,
|
||||
|
||||
"daily_req": [],
|
||||
"f_daily_schedule": [],
|
||||
"f_daily_schedule_hourly_org": [],
|
||||
"act_address": null,
|
||||
"act_start_time": null,
|
||||
"act_duration": null,
|
||||
"act_description": null,
|
||||
"act_pronunciatio": null,
|
||||
"act_event": ["Isabella Rodriguez", null, null],
|
||||
"act_obj_description": null,
|
||||
"act_obj_pronunciatio": null,
|
||||
"act_obj_event": [null, null, null],
|
||||
"chatting_with": null,
|
||||
"chat": null,
|
||||
"chatting_with_buffer": {},
|
||||
"chatting_end_time": null,
|
||||
"act_path_set": false,
|
||||
"planned_path": []
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
{
|
||||
"the Ville": {
|
||||
"Hobbs Cafe": {
|
||||
"cafe": [
|
||||
"refrigerator",
|
||||
"cafe customer seating",
|
||||
"cooking area",
|
||||
"kitchen sink",
|
||||
"behind the cafe counter",
|
||||
"piano"
|
||||
]
|
||||
},
|
||||
"Isabella Rodriguez's apartment": {
|
||||
"main room": [
|
||||
"bed",
|
||||
"desk",
|
||||
"refrigerator",
|
||||
"closet",
|
||||
"shelf"
|
||||
]
|
||||
},
|
||||
"The Rose and Crown Pub": {
|
||||
"pub": [
|
||||
"shelf",
|
||||
"refrigerator",
|
||||
"bar customer seating",
|
||||
"behind the bar counter",
|
||||
"kitchen sink",
|
||||
"cooking area",
|
||||
"microphone"
|
||||
]
|
||||
},
|
||||
"Harvey Oak Supply Store": {
|
||||
"supply store": [
|
||||
"supply store product shelf",
|
||||
"behind the supply store counter",
|
||||
"supply store counter"
|
||||
]
|
||||
},
|
||||
"The Willows Market and Pharmacy": {
|
||||
"store": [
|
||||
"behind the pharmacy counter",
|
||||
"pharmacy store shelf",
|
||||
"pharmacy store counter",
|
||||
"grocery store shelf",
|
||||
"behind the grocery counter",
|
||||
"grocery store counter"
|
||||
]
|
||||
},
|
||||
"Dorm for Oak Hill College": {
|
||||
"garden": [
|
||||
"dorm garden"
|
||||
],
|
||||
"common room": [
|
||||
"common room sofa",
|
||||
"pool table",
|
||||
"common room table"
|
||||
]
|
||||
},
|
||||
"Johnson Park": {
|
||||
"park": [
|
||||
"park garden"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
{"kw_strength_event": {},
|
||||
"kw_strength_thought": {}}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
{
|
||||
"vision_r": 8,
|
||||
"att_bandwidth": 8,
|
||||
"retention": 8,
|
||||
"curr_time": null,
|
||||
"curr_tile": null,
|
||||
"daily_plan_req": "Klaus Mueller goes to the library at Oak Hill College early in the morning, spends his days writing, and eats at Hobbs Cafe.",
|
||||
"name": "Klaus Mueller",
|
||||
"first_name": "Klaus",
|
||||
"last_name": "Mueller",
|
||||
"age": 20,
|
||||
"innate": "kind, inquisitive, passionate",
|
||||
"learned": "Klaus Mueller is a student at Oak Hill College studying sociology. He is passionate about social justice and loves to explore different perspectives.",
|
||||
"currently": "Klaus Mueller is writing a research paper on the effects of gentrification in low-income communities.",
|
||||
"lifestyle": "Klaus Mueller goes to bed around 11pm, awakes up around 7am, eats dinner around 5pm.",
|
||||
"living_area": "the Ville:Dorm for Oak Hill College:Klaus Mueller's room",
|
||||
"concept_forget": 100,
|
||||
"daily_reflection_time": 180,
|
||||
"daily_reflection_size": 5,
|
||||
"overlap_reflect_th": 4,
|
||||
"kw_strg_event_reflect_th": 10,
|
||||
"kw_strg_thought_reflect_th": 9,
|
||||
|
||||
"recency_w": 1,
|
||||
"relevance_w": 1,
|
||||
"importance_w": 1,
|
||||
"recency_decay": 0.99,
|
||||
"importance_trigger_max": 150,
|
||||
"importance_trigger_curr": 150,
|
||||
"importance_ele_n": 0,
|
||||
"thought_count": 5,
|
||||
|
||||
"daily_req": [],
|
||||
"f_daily_schedule": [],
|
||||
"f_daily_schedule_hourly_org": [],
|
||||
"act_address": null,
|
||||
"act_start_time": null,
|
||||
"act_duration": null,
|
||||
"act_description": null,
|
||||
"act_pronunciatio": null,
|
||||
"act_event": ["Klaus Mueller", null, null],
|
||||
"act_obj_description": null,
|
||||
"act_obj_pronunciatio": null,
|
||||
"act_obj_event": [null, null, null],
|
||||
"chatting_with": null,
|
||||
"chat": null,
|
||||
"chatting_with_buffer": {},
|
||||
"chatting_end_time": null,
|
||||
"act_path_set": false,
|
||||
"planned_path": []
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
{
|
||||
"the Ville": {
|
||||
"Oak Hill College": {
|
||||
"hallway": [],
|
||||
"library": [
|
||||
"library sofa",
|
||||
"library table",
|
||||
"bookshelf"
|
||||
],
|
||||
"classroom": [
|
||||
"blackboard",
|
||||
"classroom podium",
|
||||
"classroom student seating"
|
||||
]
|
||||
},
|
||||
"Dorm for Oak Hill College": {
|
||||
"garden": [
|
||||
"dorm garden"
|
||||
],
|
||||
"Klaus Mueller's room": [
|
||||
"bed",
|
||||
"game console",
|
||||
"closet",
|
||||
"desk"
|
||||
],
|
||||
"woman's bathroom": [
|
||||
"toilet",
|
||||
"shower",
|
||||
"bathroom sink"
|
||||
],
|
||||
"common room": [
|
||||
"common room sofa",
|
||||
"pool table",
|
||||
"common room table"
|
||||
],
|
||||
"man's bathroom": [
|
||||
"shower",
|
||||
"bathroom sink",
|
||||
"toilet"
|
||||
]
|
||||
},
|
||||
"The Willows Market and Pharmacy": {
|
||||
"store": [
|
||||
"grocery store shelf",
|
||||
"behind the grocery counter",
|
||||
"grocery store counter",
|
||||
"pharmacy store shelf",
|
||||
"pharmacy store counter",
|
||||
"behind the pharmacy counter"
|
||||
]
|
||||
},
|
||||
"Harvey Oak Supply Store": {
|
||||
"supply store": [
|
||||
"supply store product shelf",
|
||||
"behind the supply store counter",
|
||||
"supply store counter"
|
||||
]
|
||||
},
|
||||
"Johnson Park": {
|
||||
"park": [
|
||||
"park garden"
|
||||
]
|
||||
},
|
||||
"The Rose and Crown Pub": {
|
||||
"pub": [
|
||||
"shelf",
|
||||
"refrigerator",
|
||||
"bar customer seating",
|
||||
"behind the bar counter",
|
||||
"kitchen sink",
|
||||
"cooking area",
|
||||
"microphone"
|
||||
]
|
||||
},
|
||||
"Hobbs Cafe": {
|
||||
"cafe": [
|
||||
"refrigerator",
|
||||
"cafe customer seating",
|
||||
"cooking area",
|
||||
"kitchen sink",
|
||||
"behind the cafe counter",
|
||||
"piano"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
{"kw_strength_event": {},
|
||||
"kw_strength_thought": {}}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
{
|
||||
"vision_r": 8,
|
||||
"att_bandwidth": 8,
|
||||
"retention": 8,
|
||||
"curr_time": null,
|
||||
"curr_tile": null,
|
||||
"daily_plan_req": "Maria Lopez spends at least 3 hours a day Twitch streaming or gaming.",
|
||||
"name": "Maria Lopez",
|
||||
"first_name": "Maria",
|
||||
"last_name": "Lopez",
|
||||
"age": 21,
|
||||
"innate": "energetic, enthusiastic, inquisitive",
|
||||
"learned": "Maria Lopez is a student at Oak Hill College studying physics and a part time Twitch game streamer who loves to connect with people and explore new ideas.",
|
||||
"currently": "Maria Lopez is working on her physics degree and streaming games on Twitch to make some extra money. She visits Hobbs Cafe for studying and eating just about everyday.",
|
||||
"lifestyle": "Maria Lopez goes to bed around 2am, awakes up around 9am, eats dinner around 6pm. She likes to hang out at Hobbs Cafe if it's before 6pm.",
|
||||
"living_area": "the Ville:Dorm for Oak Hill College:Maria Lopez's room",
|
||||
"concept_forget": 100,
|
||||
"daily_reflection_time": 180,
|
||||
"daily_reflection_size": 5,
|
||||
"overlap_reflect_th": 4,
|
||||
"kw_strg_event_reflect_th": 10,
|
||||
"kw_strg_thought_reflect_th": 9,
|
||||
|
||||
"recency_w": 1,
|
||||
"relevance_w": 1,
|
||||
"importance_w": 1,
|
||||
"recency_decay": 0.99,
|
||||
"importance_trigger_max": 150,
|
||||
"importance_trigger_curr": 150,
|
||||
"importance_ele_n": 0,
|
||||
"thought_count": 5,
|
||||
|
||||
"daily_req": [],
|
||||
"f_daily_schedule": [],
|
||||
"f_daily_schedule_hourly_org": [],
|
||||
"act_address": null,
|
||||
"act_start_time": null,
|
||||
"act_duration": null,
|
||||
"act_description": null,
|
||||
"act_pronunciatio": null,
|
||||
"act_event": ["Maria Lopez", null, null],
|
||||
"act_obj_description": null,
|
||||
"act_obj_pronunciatio": null,
|
||||
"act_obj_event": [null, null, null],
|
||||
"chatting_with": null,
|
||||
"chat": null,
|
||||
"chatting_with_buffer": {},
|
||||
"chatting_end_time": null,
|
||||
"act_path_set": false,
|
||||
"planned_path": []
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
{
|
||||
"the Ville": {
|
||||
"Oak Hill College": {
|
||||
"hallway": [],
|
||||
"library": [
|
||||
"library sofa",
|
||||
"library table",
|
||||
"bookshelf"
|
||||
],
|
||||
"classroom": [
|
||||
"blackboard",
|
||||
"classroom podium",
|
||||
"classroom student seating"
|
||||
]
|
||||
},
|
||||
"Dorm for Oak Hill College": {
|
||||
"garden": [
|
||||
"dorm garden"
|
||||
],
|
||||
"Maria Lopez's room": [
|
||||
"closet",
|
||||
"desk",
|
||||
"bed",
|
||||
"computer",
|
||||
"blackboard"
|
||||
],
|
||||
"woman's bathroom": [
|
||||
"toilet",
|
||||
"shower",
|
||||
"bathroom sink"
|
||||
],
|
||||
"common room": [
|
||||
"common room sofa",
|
||||
"pool table",
|
||||
"common room table"
|
||||
],
|
||||
"man's bathroom": [
|
||||
"shower",
|
||||
"bathroom sink",
|
||||
"toilet"
|
||||
]
|
||||
},
|
||||
"The Willows Market and Pharmacy": {
|
||||
"store": [
|
||||
"grocery store shelf",
|
||||
"behind the grocery counter",
|
||||
"grocery store counter",
|
||||
"pharmacy store shelf",
|
||||
"pharmacy store counter",
|
||||
"behind the pharmacy counter"
|
||||
]
|
||||
},
|
||||
"Harvey Oak Supply Store": {
|
||||
"supply store": [
|
||||
"supply store product shelf",
|
||||
"behind the supply store counter",
|
||||
"supply store counter"
|
||||
]
|
||||
},
|
||||
"Johnson Park": {
|
||||
"park": [
|
||||
"park garden"
|
||||
]
|
||||
},
|
||||
"The Rose and Crown Pub": {
|
||||
"pub": [
|
||||
"shelf",
|
||||
"refrigerator",
|
||||
"bar customer seating",
|
||||
"behind the bar counter",
|
||||
"kitchen sink",
|
||||
"cooking area",
|
||||
"microphone"
|
||||
]
|
||||
},
|
||||
"Hobbs Cafe": {
|
||||
"cafe": [
|
||||
"refrigerator",
|
||||
"cafe customer seating",
|
||||
"cooking area",
|
||||
"kitchen sink",
|
||||
"behind the cafe counter",
|
||||
"piano"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"fork_sim_code": "base_the_ville_isabella_maria_klaus",
|
||||
"start_date": "February 13, 2023",
|
||||
"curr_time": "February 13, 2023, 00:00:00",
|
||||
"sec_per_step": 10,
|
||||
"maze_name": "the_ville",
|
||||
"persona_names": [
|
||||
"Isabella Rodriguez",
|
||||
"Maria Lopez",
|
||||
"Klaus Mueller"
|
||||
],
|
||||
"step": 0
|
||||
}
|
||||
117
examples/stream_output_via_api.py
Normal file
117
examples/stream_output_via_api.py
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
@Time : 2024/3/27 9:44
|
||||
@Author : leiwu30
|
||||
@File : stream_output_via_api.py
|
||||
@Description : Stream log information and communicate over the network via web api.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import socket
|
||||
import threading
|
||||
from contextvars import ContextVar
|
||||
|
||||
from flask import Flask, Response, jsonify, request, send_from_directory
|
||||
|
||||
from metagpt.const import TUTORIAL_PATH
|
||||
from metagpt.logs import logger, set_llm_stream_logfunc
|
||||
from metagpt.roles.tutorial_assistant import TutorialAssistant
|
||||
from metagpt.utils.stream_pipe import StreamPipe
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
def stream_pipe_log(content):
|
||||
print(content, end="")
|
||||
stream_pipe = stream_pipe_var.get(None)
|
||||
if stream_pipe:
|
||||
stream_pipe.set_message(content)
|
||||
|
||||
|
||||
def write_tutorial(message):
|
||||
async def main(idea, stream_pipe):
|
||||
stream_pipe_var.set(stream_pipe)
|
||||
role = TutorialAssistant()
|
||||
await role.run(idea)
|
||||
|
||||
def thread_run(idea: str, stream_pipe: StreamPipe = None):
|
||||
"""
|
||||
Convert asynchronous function to thread function
|
||||
"""
|
||||
asyncio.run(main(idea, stream_pipe))
|
||||
|
||||
stream_pipe = StreamPipe()
|
||||
thread = threading.Thread(
|
||||
target=thread_run,
|
||||
args=(
|
||||
message["content"],
|
||||
stream_pipe,
|
||||
),
|
||||
)
|
||||
thread.start()
|
||||
|
||||
while thread.is_alive():
|
||||
msg = stream_pipe.get_message()
|
||||
yield stream_pipe.msg2stream(msg)
|
||||
|
||||
|
||||
@app.route("/v1/chat/completions", methods=["POST"])
|
||||
def completions():
|
||||
"""
|
||||
data: {
|
||||
"model": "write_tutorial",
|
||||
"stream": true,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Write a tutorial about MySQL"
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
data = json.loads(request.data)
|
||||
logger.info(json.dumps(data, indent=4, ensure_ascii=False))
|
||||
|
||||
# Non-streaming interfaces are not supported yet
|
||||
stream_type = True if data.get("stream") else False
|
||||
if not stream_type:
|
||||
return jsonify({"status": 400, "msg": "Non-streaming requests are not supported, please use `stream=True`."})
|
||||
|
||||
# Only accept the last user information
|
||||
# openai['model'] ~ MetaGPT['agent']
|
||||
last_message = data["messages"][-1]
|
||||
model = data["model"]
|
||||
|
||||
# write_tutorial
|
||||
if model == "write_tutorial":
|
||||
return Response(write_tutorial(last_message), mimetype="text/plain")
|
||||
else:
|
||||
return jsonify({"status": 400, "msg": "No suitable agent found."})
|
||||
|
||||
|
||||
@app.route("/download/<path:filename>")
|
||||
def download_file(filename):
|
||||
return send_from_directory(TUTORIAL_PATH, filename, as_attachment=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""
|
||||
curl https://$server_address:$server_port/v1/chat/completions -X POST -d '{
|
||||
"model": "write_tutorial",
|
||||
"stream": true,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Write a tutorial about MySQL"
|
||||
}
|
||||
]
|
||||
}'
|
||||
"""
|
||||
server_port = 7860
|
||||
server_address = socket.gethostbyname(socket.gethostname())
|
||||
|
||||
set_llm_stream_logfunc(stream_pipe_log)
|
||||
stream_pipe_var: ContextVar[StreamPipe] = ContextVar("stream_pipe")
|
||||
app.run(port=server_port, host=server_address)
|
||||
218
examples/werewolf_game/evals/eval.py
Normal file
218
examples/werewolf_game/evals/eval.py
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
"""
|
||||
Filename: MetaGPT/examples/werewolf_game/evals/eval.py
|
||||
Created Date: Oct 18, 2023
|
||||
Updated Date: Oct 24, 2023
|
||||
Author: [Aria](https://github.com/ariafyy)
|
||||
Info: eval the Voting Accuracy Rate of non_werewolves and Vote Difficulity
|
||||
"""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from tqdm import tqdm
|
||||
from utils import Utils
|
||||
|
||||
from metagpt.const import DEFAULT_WORKSPACE_ROOT, METAGPT_ROOT
|
||||
from metagpt.environment.werewolf.const import RoleType
|
||||
|
||||
|
||||
class Vote:
|
||||
"""Vote Evaluation"""
|
||||
|
||||
def __init__(self):
|
||||
self.OUT_PATH = DEFAULT_WORKSPACE_ROOT / "outputs"
|
||||
os.makedirs(self.OUT_PATH, exist_ok=True)
|
||||
self.SUB_FOLDER_LIST = ["01-10", "11-20", "21-30"]
|
||||
|
||||
def _get_log_fileslist(self, IN_PATH) -> list[str]:
|
||||
files_list = []
|
||||
for SUB_FOLDER in self.SUB_FOLDER_LIST:
|
||||
files_list.extend(glob.glob(str(IN_PATH / SUB_FOLDER / "*.txt")))
|
||||
return files_list
|
||||
|
||||
def extract_votes_from_logs(self, files_list: list):
|
||||
for in_logfile in tqdm(files_list):
|
||||
SUB_FOLDER = (Path(in_logfile).parent).stem
|
||||
out_txtfile = self.OUT_PATH / "# {0}_{1}.txt".format(SUB_FOLDER, Path(in_logfile).stem)
|
||||
Utils().pick_vote_log(in_logfile, out_txtfile)
|
||||
votefiles_list = Utils().get_file_list(self.OUT_PATH)
|
||||
return votefiles_list
|
||||
|
||||
@staticmethod
|
||||
def parse_vote_text2chunks(text: str):
|
||||
"""
|
||||
parse each game vote log into text chunks
|
||||
|
||||
one chunk example:
|
||||
['Player1', 'Player2', 'Player3', 'Player5', 'Player6']. Say ONLY: I vote to eliminate ...
|
||||
Player1(Witch): 49 | I vote to eliminate Player5
|
||||
Player2(Villager): 49 | I vote to eliminate Player5
|
||||
Player3(Villager): 49 | I vote to eliminate Player5
|
||||
Player5(Werewolf): 49 | I vote to eliminate Player6
|
||||
Player6(Seer): 49 | I vote to eliminate Player5
|
||||
"""
|
||||
pattern = re.compile(r"""\[([^\]]+)\]. Say ONLY: I vote to eliminate ...""")
|
||||
chunks = {}
|
||||
chunk_id = 0
|
||||
last_end = 0
|
||||
for match in pattern.finditer(text):
|
||||
start = match.start()
|
||||
chunk = text[last_end:start]
|
||||
chunks[f"vote_{chunk_id}"] = chunk.strip()
|
||||
last_end = match.end()
|
||||
chunk_id += 1
|
||||
final_chunk = text[last_end:].strip()
|
||||
if final_chunk:
|
||||
chunks[f"vote_{chunk_id}"] = final_chunk
|
||||
return chunks
|
||||
|
||||
def _vote_rate_players(self, text: str):
|
||||
"""
|
||||
# calculate the rate of goodteam vote werewolves
|
||||
:example:
|
||||
|
||||
input:
|
||||
['Player1', 'Player2', 'Player3', 'Player5', 'Player6']. Say ONLY: I vote to eliminate ...
|
||||
Player1(Witch): 49 | I vote to eliminate Player5
|
||||
Player2(Villager): 49 | I vote to eliminate Player5
|
||||
Player3(Villager): 49 | I vote to eliminate Player5
|
||||
Player5(Werewolf): 49 | I vote to eliminate Player6
|
||||
Player6(Seer): 49 | I vote to eliminate Player5
|
||||
|
||||
output:
|
||||
werewolves: ['Player5']
|
||||
non_werewolves: ['Player1', 'Player2', 'Player3', 'Player6']
|
||||
as you can see :Player2(Villager) and Player3(Villager) vote to eliminate Player5(Werewolf)
|
||||
:return goodteam vote rateability: 100.00%
|
||||
"""
|
||||
pattern = re.compile(r"(\w+)\(([^\)]+)\): \d+ \| I vote to eliminate (\w+)")
|
||||
# find all werewolves
|
||||
werewolves = []
|
||||
for match in pattern.finditer(text):
|
||||
if match.group(2) == RoleType.WEREWOLF.value:
|
||||
werewolves.append(match.group(1))
|
||||
|
||||
# find all non_werewolves
|
||||
non_werewolves = []
|
||||
for match in pattern.finditer(text):
|
||||
if match.group(2) != RoleType.WEREWOLF.value:
|
||||
non_werewolves.append(match.group(1))
|
||||
num_non_werewolves = len(non_werewolves)
|
||||
|
||||
# count players other than werewolves made the correct votes
|
||||
correct_votes = 0
|
||||
for match in pattern.finditer(text):
|
||||
if match.group(2) != RoleType.WEREWOLF.value and match.group(3) in werewolves:
|
||||
correct_votes += 1
|
||||
|
||||
# cal the rateability of non_werewolves
|
||||
rate = correct_votes / num_non_werewolves
|
||||
good_vote_rate = round(rate, 2)
|
||||
return {"good_vote_rate": good_vote_rate, "werewolves": werewolves, "non_werewolves": non_werewolves}
|
||||
|
||||
def get_goodteam_vote_rate(self, text: str) -> float:
|
||||
goodteam_vote_rate = self._vote_rate_players(text)["good_vote_rate"]
|
||||
return goodteam_vote_rate
|
||||
|
||||
def get_werewolves(self, text: str) -> list:
|
||||
werewolves_list = self._vote_rate_players(text)["werewolves"]
|
||||
return werewolves_list
|
||||
|
||||
def get_non_werewolves(self, text: str) -> list:
|
||||
non_werewolves_list = self._vote_rate_players(text)["non_werewolves"]
|
||||
return non_werewolves_list
|
||||
|
||||
def get_votewolf_difficulty(self, werewolves: list, non_werewolves: list) -> str:
|
||||
num_living_wolfs = len(werewolves)
|
||||
num_living_players = len(werewolves) + len(non_werewolves)
|
||||
votewolf_difficulty = "_{0} / {1}".format(num_living_wolfs, num_living_players)
|
||||
return votewolf_difficulty
|
||||
|
||||
def get_result_df(self, out_txtfile: str) -> pd.DataFrame:
|
||||
"""
|
||||
folder: sub folders for evals
|
||||
file: evaluation file, each file represents one game
|
||||
votes: the number of votes, eg. vote_1 represents the first vote of this game,
|
||||
good_vote_rate:the rateability of a good person voting against a werewolf,
|
||||
correct_votes / the total number of players other than werewolves
|
||||
total_votes:the total number of votes cast
|
||||
"""
|
||||
with open(out_txtfile, "r") as out_file:
|
||||
text = out_file.read()
|
||||
chunks = self.parse_vote_text2chunks(text)
|
||||
res = []
|
||||
for k, v in chunks.items():
|
||||
if v != "":
|
||||
chunks_list = list(chunks.keys())
|
||||
total_votes = len(chunks_list) - 1
|
||||
werewolves = self.get_werewolves(v)
|
||||
non_werewolves = self.get_non_werewolves(v)
|
||||
good_vote_rate = self.get_goodteam_vote_rate(v)
|
||||
votewolf_difficulty = self.get_votewolf_difficulty(werewolves, non_werewolves)
|
||||
folder = Utils().filename_to_foldername(out_txtfile)
|
||||
result = {
|
||||
"folder": folder,
|
||||
"file": Path(out_txtfile).stem + ".txt",
|
||||
"vote_round": k,
|
||||
"good_vote_rate": good_vote_rate,
|
||||
"total_votes": total_votes,
|
||||
"votewolf_difficulty": votewolf_difficulty,
|
||||
}
|
||||
res.append(result)
|
||||
df = pd.DataFrame(res)
|
||||
return df
|
||||
|
||||
def calc_avg_rate(self, IN_PATH) -> pd.DataFrame:
|
||||
"""
|
||||
get avg_rate for each game
|
||||
avg_rate : the good_rate/total number of votes in the game
|
||||
vote1_rate: First Round Voting Accuracy Rate
|
||||
"""
|
||||
infiles_list = self._get_log_fileslist(IN_PATH)
|
||||
votefiles_list = self.extract_votes_from_logs(infiles_list)
|
||||
df_list = [self._load_df_from_file(file) for file in votefiles_list]
|
||||
combined_df = pd.concat(df_list, ignore_index=True)
|
||||
# calculate the average good_vote_rate for each file
|
||||
mean_rates = self._calculate_mean_rates(combined_df)
|
||||
combined_df["avg_rate"] = combined_df["file"].map(mean_rates)
|
||||
# calculate vote1 rate
|
||||
vote1_rates = self._calc_vote1_rates(combined_df)
|
||||
combined_df["vote1_rate"] = combined_df["folder"].map(vote1_rates.set_index("folder")["good_vote_rate"])
|
||||
combined_df.loc[combined_df["vote_round"] != "vote_1", "vote1_rate"] = np.nan
|
||||
combined_df["vote1_rate"] = combined_df["vote1_rate"].apply(self._format_rates)
|
||||
combined_df["good_vote_rate"] = combined_df["good_vote_rate"].apply(self._format_rates)
|
||||
combined_df["avg_rate"] = combined_df["avg_rate"].apply(self._format_rates)
|
||||
combined_df.sort_values(["file"], ascending=True, inplace=True)
|
||||
return combined_df
|
||||
|
||||
def _calc_vote1_rates(self, df):
|
||||
df_vote1 = df[df["vote_round"] == "vote_1"]
|
||||
vote1_rates = df_vote1.groupby("folder")["good_vote_rate"].mean().reset_index()
|
||||
return vote1_rates
|
||||
|
||||
def _load_df_from_file(self, file):
|
||||
return self.get_result_df(file)
|
||||
|
||||
def _calculate_mean_rates(self, df):
|
||||
return df.groupby("file")["good_vote_rate"].mean()
|
||||
|
||||
def _format_rates(self, s):
|
||||
return Utils().float_to_percent(s)
|
||||
|
||||
def get_eval_csv(self, IN_PATH, EVAL_RESULT):
|
||||
"""
|
||||
IN_PATH : parent folder of ["01-10", "11-20", "21-30"]
|
||||
EVAL_RESULT : output csv file path
|
||||
"""
|
||||
combined_df = self.calc_avg_rate(IN_PATH)
|
||||
combined_df.to_csv(EVAL_RESULT, index=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
IN_PATH = METAGPT_ROOT / "examples/werewolf_game/evals"
|
||||
EVAL_RESULT = DEFAULT_WORKSPACE_ROOT / "outputs" / "goodteam_vote_rate.csv"
|
||||
Vote().get_eval_csv(IN_PATH, EVAL_RESULT)
|
||||
134
examples/werewolf_game/evals/utils.py
Normal file
134
examples/werewolf_game/evals/utils.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
"""
|
||||
Filename: MetaGPT/examples/werewolf_game/evals/utils.py
|
||||
Created Date: Oct 11, 2023
|
||||
Revised Date: Oct 20, 2023
|
||||
Author: [Aria](https://github.com/ariafyy)
|
||||
"""
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from metagpt.const import METAGPT_ROOT
|
||||
|
||||
|
||||
class Utils:
|
||||
"""Utils: utils of logs"""
|
||||
|
||||
@staticmethod
|
||||
def polish_log(in_logfile, out_txtfile):
|
||||
"""polish logs for evaluation"""
|
||||
pattern_text = r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \| (\w+) +\| ([\w\.]+:\w+:\d+) - (.*\S)"
|
||||
pattern_player = r"(Player(\d{1}): \w+)"
|
||||
pattern_start = False
|
||||
json_start = False
|
||||
|
||||
with open(in_logfile, "r") as f, open(out_txtfile, "w") as out:
|
||||
for line in f.readlines():
|
||||
matches = re.match(pattern_text, line)
|
||||
if matches:
|
||||
message = matches.group(4).strip()
|
||||
pattern_start = True
|
||||
json_start = False
|
||||
|
||||
if (
|
||||
"Moderator(Moderator) ready to InstructSpeak" not in message
|
||||
and "Moderator(Moderator) ready to ParseSpeak" not in message
|
||||
and "Total running cost:" not in message
|
||||
):
|
||||
out.write("- " + message + "\n")
|
||||
else:
|
||||
out.write("\n")
|
||||
|
||||
elif pattern_start and not matches:
|
||||
if "gpt-4 may update over time" in line:
|
||||
line = ""
|
||||
out.write(line)
|
||||
|
||||
elif line.strip().startswith("{"):
|
||||
out.write(line.strip())
|
||||
json_start = True
|
||||
|
||||
elif json_start and not line.strip().endswith("}"):
|
||||
out.write(line.strip())
|
||||
|
||||
elif json_start and line.strip().endswith("}"):
|
||||
out.write(line.strip())
|
||||
json_start = False
|
||||
|
||||
elif (
|
||||
line.startswith("(User):") or line.startswith("********** STEP:") or re.search(pattern_player, line)
|
||||
):
|
||||
out.write(line)
|
||||
|
||||
else:
|
||||
out.write("\n")
|
||||
|
||||
@staticmethod
|
||||
def pick_vote_log(in_logfile, out_txtfile):
|
||||
"""
|
||||
pick the vote log from the log file.
|
||||
ready to AnnounceGameResult serves as the 'HINT_TEXT ' which indicates the end of the game.
|
||||
based on bservation and reflection, then discuss is not in vote session.
|
||||
"""
|
||||
pattern_vote = r"(Player\d+)\(([A-Za-z]+)\): (\d+) \| (I vote to eliminate Player\d+)"
|
||||
ignore_text = """reflection"""
|
||||
HINT_TEXT = r"ready to AnnounceGameResult"
|
||||
pattern_moderator = r"\[([^\]]+)\]\. Say ONLY: I vote to eliminate ..."
|
||||
in_valid_block = False
|
||||
|
||||
with open(in_logfile, "r") as f:
|
||||
lines = f.read()
|
||||
split_lines = lines.split(HINT_TEXT)
|
||||
|
||||
if len(split_lines) < 2:
|
||||
print(f"Key text :{HINT_TEXT} not found in {in_logfile}")
|
||||
return
|
||||
|
||||
relevant_lines = split_lines[1].split("\n")
|
||||
with open(out_txtfile, "w") as out:
|
||||
for line in relevant_lines:
|
||||
if re.search(pattern_moderator, line):
|
||||
in_valid_block = True
|
||||
out.write(line.lstrip() + "\n")
|
||||
|
||||
elif in_valid_block and re.search(pattern_vote, line):
|
||||
out.write(line + "\n")
|
||||
elif ignore_text in line:
|
||||
in_valid_block = False
|
||||
|
||||
@staticmethod
|
||||
def get_file_list(path: str) -> list:
|
||||
file_pattern = os.path.join(path, "*.txt")
|
||||
files_list = glob.glob(file_pattern)
|
||||
return files_list
|
||||
|
||||
@staticmethod
|
||||
def filename_to_foldername(out_txtfile: str):
|
||||
"""
|
||||
convert filename into its parent folder name
|
||||
input:"....../# 01-10_10132100.txt"
|
||||
output:# 01-10
|
||||
"""
|
||||
s = Path(out_txtfile).stem
|
||||
pattern_folder = r"([^_]*)_"
|
||||
match = re.match(pattern_folder, s)
|
||||
if match:
|
||||
folder = match.group(1)
|
||||
return folder
|
||||
|
||||
@staticmethod
|
||||
def float_to_percent(decimal: float) -> str:
|
||||
"""
|
||||
input: 1.00
|
||||
output: 100.00%
|
||||
"""
|
||||
percent = decimal * 100
|
||||
return f"{percent:.2f}%"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
in_logfile = METAGPT_ROOT / "logs/log.txt"
|
||||
out_txtfile = "input your wish path"
|
||||
# Utils().polish_log(in_logfile, out_txtfile)
|
||||
Utils().pick_vote_log(in_logfile, out_txtfile)
|
||||
68
examples/werewolf_game/start_game.py
Normal file
68
examples/werewolf_game/start_game.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import asyncio
|
||||
|
||||
import fire
|
||||
|
||||
from metagpt.ext.werewolf.roles import Guard, Moderator, Seer, Villager, Werewolf, Witch
|
||||
from metagpt.ext.werewolf.roles.human_player import prepare_human_player
|
||||
from metagpt.ext.werewolf.werewolf_game import WerewolfGame
|
||||
from metagpt.logs import logger
|
||||
|
||||
|
||||
async def start_game(
|
||||
investment: float = 3.0,
|
||||
n_round: int = 5,
|
||||
shuffle: bool = True,
|
||||
add_human: bool = False,
|
||||
use_reflection: bool = True,
|
||||
use_experience: bool = False,
|
||||
use_memory_selection: bool = False,
|
||||
new_experience_version: str = "",
|
||||
):
|
||||
game = WerewolfGame()
|
||||
game_setup, players = game.env.init_game_setup(
|
||||
role_uniq_objs=[Villager, Werewolf, Guard, Seer, Witch],
|
||||
num_werewolf=2,
|
||||
num_villager=2,
|
||||
shuffle=shuffle,
|
||||
add_human=add_human,
|
||||
use_reflection=use_reflection,
|
||||
use_experience=use_experience,
|
||||
use_memory_selection=use_memory_selection,
|
||||
new_experience_version=new_experience_version,
|
||||
prepare_human_player=prepare_human_player,
|
||||
)
|
||||
logger.info(f"{game_setup}")
|
||||
|
||||
players = [Moderator()] + players
|
||||
game.hire(players)
|
||||
game.invest(investment)
|
||||
game.run_project(game_setup)
|
||||
await game.run(n_round=n_round)
|
||||
|
||||
|
||||
def main(
|
||||
investment: float = 20.0,
|
||||
n_round: int = 100,
|
||||
shuffle: bool = True,
|
||||
add_human: bool = False,
|
||||
use_reflection: bool = True,
|
||||
use_experience: bool = False,
|
||||
use_memory_selection: bool = False,
|
||||
new_experience_version: str = "",
|
||||
):
|
||||
asyncio.run(
|
||||
start_game(
|
||||
investment,
|
||||
n_round,
|
||||
shuffle,
|
||||
add_human,
|
||||
use_reflection,
|
||||
use_experience,
|
||||
use_memory_selection,
|
||||
new_experience_version,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(main)
|
||||
|
|
@ -17,6 +17,7 @@ from pydantic import BaseModel, Field, create_model, model_validator
|
|||
from tenacity import retry, stop_after_attempt, wait_random_exponential
|
||||
|
||||
from metagpt.actions.action_outcls_registry import register_action_outcls
|
||||
from metagpt.const import USE_CONFIG_TIMEOUT
|
||||
from metagpt.llm import BaseLLM
|
||||
from metagpt.logs import logger
|
||||
from metagpt.provider.postprocess.llm_output_postprocess import llm_output_postprocess
|
||||
|
|
@ -330,7 +331,7 @@ class ActionNode:
|
|||
|
||||
def compile_to(self, i: Dict, schema, kv_sep) -> str:
|
||||
if schema == "json":
|
||||
return json.dumps(i, indent=4)
|
||||
return json.dumps(i, indent=4, ensure_ascii=False)
|
||||
elif schema == "markdown":
|
||||
return dict_to_markdown(i, kv_sep=kv_sep)
|
||||
else:
|
||||
|
|
@ -339,10 +340,7 @@ class ActionNode:
|
|||
def tagging(self, text, schema, tag="") -> str:
|
||||
if not tag:
|
||||
return text
|
||||
if schema == "json":
|
||||
return f"[{tag}]\n" + text + f"\n[/{tag}]"
|
||||
else: # markdown
|
||||
return f"[{tag}]\n" + text + f"\n[/{tag}]"
|
||||
return f"[{tag}]\n{text}\n[/{tag}]"
|
||||
|
||||
def _compile_f(self, schema, mode, tag, format_func, kv_sep, exclude=None) -> str:
|
||||
nodes = self.to_dict(format_func=format_func, mode=mode, exclude=exclude)
|
||||
|
|
@ -374,7 +372,7 @@ class ActionNode:
|
|||
schema="markdown": 编译context, example(markdown), instruction(markdown), constraint, action
|
||||
"""
|
||||
if schema == "raw":
|
||||
return context + "\n\n## Actions\n" + LANGUAGE_CONSTRAINT + "\n" + self.instruction
|
||||
return f"{context}\n\n## Actions\n{LANGUAGE_CONSTRAINT}\n{self.instruction}"
|
||||
|
||||
### 直接使用 pydantic BaseModel 生成 instruction 与 example，仅限 JSON
|
||||
# child_class = self._create_children_class()
|
||||
|
|
@ -416,7 +414,7 @@ class ActionNode:
|
|||
images: Optional[Union[str, list[str]]] = None,
|
||||
system_msgs: Optional[list[str]] = None,
|
||||
schema="markdown", # compatible to original format
|
||||
timeout=3,
|
||||
timeout=USE_CONFIG_TIMEOUT,
|
||||
) -> (str, BaseModel):
|
||||
"""Use ActionOutput to wrap the output of aask"""
|
||||
content = await self.llm.aask(prompt, system_msgs, images=images, timeout=timeout)
|
||||
|
|
@ -448,9 +446,10 @@ class ActionNode:
|
|||
def set_context(self, context):
|
||||
self.set_recursive("context", context)
|
||||
|
||||
async def simple_fill(self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=3, exclude=None):
|
||||
async def simple_fill(
|
||||
self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=USE_CONFIG_TIMEOUT, exclude=None
|
||||
):
|
||||
prompt = self.compile(context=self.context, schema=schema, mode=mode, exclude=exclude)
|
||||
|
||||
if schema != "raw":
|
||||
mapping = self.get_mapping(mode, exclude=exclude)
|
||||
class_name = f"{self.key}_AN"
|
||||
|
|
@ -473,7 +472,7 @@ class ActionNode:
|
|||
mode="auto",
|
||||
strgy="simple",
|
||||
images: Optional[Union[str, list[str]]] = None,
|
||||
timeout=3,
|
||||
timeout=USE_CONFIG_TIMEOUT,
|
||||
exclude=[],
|
||||
):
|
||||
"""Fill the node(s) with mode.
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from metagpt.prompts.di.write_analysis_code import (
|
|||
STRUCTUAL_PROMPT,
|
||||
)
|
||||
from metagpt.schema import Message, Plan
|
||||
from metagpt.utils.common import CodeParser, process_message, remove_comments
|
||||
from metagpt.utils.common import CodeParser, remove_comments
|
||||
|
||||
|
||||
class WriteAnalysisCode(Action):
|
||||
|
|
@ -50,7 +50,7 @@ class WriteAnalysisCode(Action):
|
|||
)
|
||||
|
||||
working_memory = working_memory or []
|
||||
context = process_message([Message(content=structual_prompt, role="user")] + working_memory)
|
||||
context = self.llm.format_msg([Message(content=structual_prompt, role="user")] + working_memory)
|
||||
|
||||
# LLM call
|
||||
if use_reflection:
|
||||
|
|
|
|||
|
|
@ -486,7 +486,7 @@ class RebuildSequenceView(Action):
|
|||
Returns:
|
||||
List[str]: A list of participants extracted from the sequence diagram.
|
||||
"""
|
||||
pattern = r"participant ([a-zA-Z\.0-9_]+)"
|
||||
pattern = r"participant ([\w\.]+)"
|
||||
matches = re.findall(pattern, mermaid_sequence_diagram)
|
||||
matches = [re.sub(r"[\\/'\"]+", "", i) for i in matches]
|
||||
return matches
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ class CollectLinks(Action):
|
|||
break
|
||||
|
||||
model_name = config.llm.model
|
||||
prompt = reduce_message_length(gen_msg(), model_name, system_text, 4096)
|
||||
prompt = reduce_message_length(gen_msg(), model_name, system_text, config.llm.max_token)
|
||||
logger.debug(prompt)
|
||||
queries = await self._aask(prompt, [system_text])
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -128,6 +128,9 @@ CODE_PLAN_AND_CHANGE_CONTEXT = """
|
|||
## User New Requirements
|
||||
{requirement}
|
||||
|
||||
## Issue
|
||||
{issue}
|
||||
|
||||
## PRD
|
||||
{prd}
|
||||
|
||||
|
|
@ -211,7 +214,8 @@ class WriteCodePlanAndChange(Action):
|
|||
design_doc = await self.repo.docs.system_design.get(filename=self.i_context.design_filename)
|
||||
task_doc = await self.repo.docs.task.get(filename=self.i_context.task_filename)
|
||||
context = CODE_PLAN_AND_CHANGE_CONTEXT.format(
|
||||
requirement=self.i_context.requirement,
|
||||
requirement=f"```text\n{self.i_context.requirement}\n```",
|
||||
issue=f"```text\n{self.i_context.issue}\n```",
|
||||
prd=prd_doc.content,
|
||||
design=design_doc.content,
|
||||
task=task_doc.content,
|
||||
|
|
|
|||
|
|
@ -133,10 +133,10 @@ REQUIREMENT_ANALYSIS = ActionNode(
|
|||
REFINED_REQUIREMENT_ANALYSIS = ActionNode(
|
||||
key="Refined Requirement Analysis",
|
||||
expected_type=List[str],
|
||||
instruction="Review and refine the existing requirement analysis to align with the evolving needs of the project "
|
||||
instruction="Review and refine the existing requirement analysis into a string list to align with the evolving needs of the project "
|
||||
"due to incremental development. Ensure the analysis comprehensively covers the new features and enhancements "
|
||||
"required for the refined project scope.",
|
||||
example=["Require add/update/modify ..."],
|
||||
example=["Require add ...", "Require modify ..."],
|
||||
)
|
||||
|
||||
REQUIREMENT_POOL = ActionNode(
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import Dict, Iterable, List, Literal, Optional
|
|||
from pydantic import BaseModel, model_validator
|
||||
|
||||
from metagpt.configs.browser_config import BrowserConfig
|
||||
from metagpt.configs.embedding_config import EmbeddingConfig
|
||||
from metagpt.configs.llm_config import LLMConfig, LLMType
|
||||
from metagpt.configs.mermaid_config import MermaidConfig
|
||||
from metagpt.configs.redis_config import RedisConfig
|
||||
|
|
@ -47,6 +48,9 @@ class Config(CLIParams, YamlModel):
|
|||
# Key Parameters
|
||||
llm: LLMConfig
|
||||
|
||||
# RAG Embedding
|
||||
embedding: EmbeddingConfig = EmbeddingConfig()
|
||||
|
||||
# Global Proxy. Will be used if llm.proxy is not set
|
||||
proxy: str = ""
|
||||
|
||||
|
|
@ -75,6 +79,7 @@ class Config(CLIParams, YamlModel):
|
|||
iflytek_api_key: str = ""
|
||||
azure_tts_subscription_key: str = ""
|
||||
azure_tts_region: str = ""
|
||||
_extra: dict = dict() # extra config dict
|
||||
|
||||
@classmethod
|
||||
def from_home(cls, path):
|
||||
|
|
@ -92,7 +97,7 @@ class Config(CLIParams, YamlModel):
|
|||
"""
|
||||
default_config_paths: List[Path] = [
|
||||
METAGPT_ROOT / "config/config2.yaml",
|
||||
Path.home() / ".metagpt/config2.yaml",
|
||||
CONFIG_ROOT / "config2.yaml",
|
||||
]
|
||||
|
||||
dicts = [dict(os.environ)]
|
||||
|
|
@ -100,6 +105,20 @@ class Config(CLIParams, YamlModel):
|
|||
final = merge_dict(dicts)
|
||||
return Config(**final)
|
||||
|
||||
@classmethod
|
||||
def from_llm_config(cls, llm_config: dict):
|
||||
"""user config llm
|
||||
example:
|
||||
llm_config = {"api_type": "xxx", "api_key": "xxx", "model": "xxx"}
|
||||
gpt4 = Config.from_llm_config(llm_config)
|
||||
A = Role(name="A", profile="Democratic candidate", goal="Win the election", actions=[a1], watch=[a2], config=gpt4)
|
||||
"""
|
||||
llm_config = LLMConfig.model_validate(llm_config)
|
||||
dicts = [dict(os.environ)]
|
||||
dicts += [{"llm": llm_config}]
|
||||
final = merge_dict(dicts)
|
||||
return Config(**final)
|
||||
|
||||
def update_via_cli(self, project_path, project_name, inc, reqa_file, max_auto_summarize_code):
|
||||
"""update config via cli"""
|
||||
|
||||
|
|
@ -113,6 +132,14 @@ class Config(CLIParams, YamlModel):
|
|||
self.reqa_file = reqa_file
|
||||
self.max_auto_summarize_code = max_auto_summarize_code
|
||||
|
||||
@property
|
||||
def extra(self):
|
||||
return self._extra
|
||||
|
||||
@extra.setter
|
||||
def extra(self, value: dict):
|
||||
self._extra = value
|
||||
|
||||
def get_openai_llm(self) -> Optional[LLMConfig]:
|
||||
"""Get OpenAI LLMConfig by name. If no OpenAI, raise Exception"""
|
||||
if self.llm.api_type == LLMType.OPENAI:
|
||||
|
|
|
|||
50
metagpt/configs/embedding_config.py
Normal file
50
metagpt/configs/embedding_config.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import field_validator
|
||||
|
||||
from metagpt.utils.yaml_model import YamlModel
|
||||
|
||||
|
||||
class EmbeddingType(Enum):
|
||||
OPENAI = "openai"
|
||||
AZURE = "azure"
|
||||
GEMINI = "gemini"
|
||||
OLLAMA = "ollama"
|
||||
|
||||
|
||||
class EmbeddingConfig(YamlModel):
|
||||
"""Config for Embedding.
|
||||
|
||||
Examples:
|
||||
---------
|
||||
api_type: "openai"
|
||||
api_key: "YOU_API_KEY"
|
||||
|
||||
api_type: "azure"
|
||||
api_key: "YOU_API_KEY"
|
||||
base_url: "YOU_BASE_URL"
|
||||
api_version: "YOU_API_VERSION"
|
||||
|
||||
api_type: "gemini"
|
||||
api_key: "YOU_API_KEY"
|
||||
|
||||
api_type: "ollama"
|
||||
base_url: "YOU_BASE_URL"
|
||||
model: "YOU_MODEL"
|
||||
"""
|
||||
|
||||
api_type: Optional[EmbeddingType] = None
|
||||
api_key: Optional[str] = None
|
||||
base_url: Optional[str] = None
|
||||
api_version: Optional[str] = None
|
||||
|
||||
model: Optional[str] = None
|
||||
embed_batch_size: Optional[int] = None
|
||||
|
||||
@field_validator("api_type", mode="before")
|
||||
@classmethod
|
||||
def check_api_type(cls, v):
|
||||
if v == "":
|
||||
return None
|
||||
return v
|
||||
|
|
@ -10,6 +10,7 @@ from typing import Optional
|
|||
|
||||
from pydantic import field_validator
|
||||
|
||||
from metagpt.const import LLM_API_TIMEOUT
|
||||
from metagpt.utils.yaml_model import YamlModel
|
||||
|
||||
|
||||
|
|
@ -29,6 +30,7 @@ class LLMType(Enum):
|
|||
DASHSCOPE = "dashscope" # Aliyun LingJi DashScope
|
||||
MOONSHOT = "moonshot"
|
||||
MISTRAL = "mistral"
|
||||
YI = "yi" # lingyiwanwu
|
||||
|
||||
def __missing__(self, key):
|
||||
return self.OPENAI
|
||||
|
|
@ -73,7 +75,7 @@ class LLMConfig(YamlModel):
|
|||
stream: bool = False
|
||||
logprobs: Optional[bool] = None # https://cookbook.openai.com/examples/using_logprobs
|
||||
top_logprobs: Optional[int] = None
|
||||
timeout: int = 60
|
||||
timeout: int = 600
|
||||
|
||||
# For Network
|
||||
proxy: Optional[str] = None
|
||||
|
|
@ -87,3 +89,8 @@ class LLMConfig(YamlModel):
|
|||
if v in ["", None, "YOUR_API_KEY"]:
|
||||
raise ValueError("Please set your API key in config2.yaml")
|
||||
return v
|
||||
|
||||
@field_validator("timeout")
|
||||
@classmethod
|
||||
def check_timeout(cls, v):
|
||||
return v or LLM_API_TIMEOUT
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from metagpt.utils.yaml_model import YamlModel
|
|||
class MermaidConfig(YamlModel):
|
||||
"""Config for Mermaid"""
|
||||
|
||||
engine: Literal["nodejs", "ink", "playwright", "pyppeteer"] = "nodejs"
|
||||
engine: Literal["nodejs", "ink", "playwright", "pyppeteer", "none"] = "nodejs"
|
||||
path: str = "mmdc" # mmdc
|
||||
puppeteer_config: str = ""
|
||||
pyppeteer_path: str = "/usr/bin/google-chrome-stable"
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
"""
|
||||
from typing import Callable, Optional
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from metagpt.tools import SearchEngineType
|
||||
from metagpt.utils.yaml_model import YamlModel
|
||||
|
||||
|
|
@ -18,3 +20,11 @@ class SearchConfig(YamlModel):
|
|||
api_key: str = ""
|
||||
cse_id: str = "" # for google
|
||||
search_func: Optional[Callable] = None
|
||||
params: dict = Field(
|
||||
default_factory=lambda: {
|
||||
"engine": "google",
|
||||
"google_domain": "google.com",
|
||||
"gl": "us",
|
||||
"hl": "en",
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ METAGPT_ROOT = get_metagpt_root() # Dependent on METAGPT_PROJECT_ROOT
|
|||
DEFAULT_WORKSPACE_ROOT = METAGPT_ROOT / "workspace"
|
||||
|
||||
EXAMPLE_PATH = METAGPT_ROOT / "examples"
|
||||
EXAMPLE_DATA_PATH = EXAMPLE_PATH / "data"
|
||||
DATA_PATH = METAGPT_ROOT / "data"
|
||||
TEST_DATA_PATH = METAGPT_ROOT / "tests/data"
|
||||
RESEARCH_PATH = DATA_PATH / "research"
|
||||
|
|
@ -122,7 +123,6 @@ BASE64_FORMAT = "base64"
|
|||
|
||||
# REDIS
|
||||
REDIS_KEY = "REDIS_KEY"
|
||||
LLM_API_TIMEOUT = 300
|
||||
|
||||
# Message id
|
||||
IGNORED_MESSAGE_ID = "0"
|
||||
|
|
@ -131,3 +131,7 @@ IGNORED_MESSAGE_ID = "0"
|
|||
GENERALIZATION = "Generalize"
|
||||
COMPOSITION = "Composite"
|
||||
AGGREGATION = "Aggregate"
|
||||
|
||||
# Timeout
|
||||
USE_CONFIG_TIMEOUT = 0 # Using llm.timeout configuration.
|
||||
LLM_API_TIMEOUT = 300
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
|
@ -78,12 +78,6 @@ class Context(BaseModel):
|
|||
# env.update({k: v for k, v in i.items() if isinstance(v, str)})
|
||||
return env
|
||||
|
||||
# def use_llm(self, name: Optional[str] = None, provider: LLMType = LLMType.OPENAI) -> BaseLLM:
|
||||
# """Use a LLM instance"""
|
||||
# self._llm_config = self.config.get_llm_config(name, provider)
|
||||
# self._llm = None
|
||||
# return self._llm
|
||||
|
||||
def _select_costmanager(self, llm_config: LLMConfig) -> CostManager:
|
||||
"""Return a CostManager instance"""
|
||||
if llm_config.api_type == LLMType.FIREWORKS:
|
||||
|
|
@ -108,3 +102,38 @@ class Context(BaseModel):
|
|||
if llm.cost_manager is None:
|
||||
llm.cost_manager = self._select_costmanager(llm_config)
|
||||
return llm
|
||||
|
||||
def serialize(self) -> Dict[str, Any]:
|
||||
"""Serialize the object's attributes into a dictionary.
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: A dictionary containing serialized data.
|
||||
"""
|
||||
return {
|
||||
"workdir": str(self.repo.workdir) if self.repo else "",
|
||||
"kwargs": {k: v for k, v in self.kwargs.__dict__.items()},
|
||||
"cost_manager": self.cost_manager.model_dump_json(),
|
||||
}
|
||||
|
||||
def deserialize(self, serialized_data: Dict[str, Any]):
|
||||
"""Deserialize the given serialized data and update the object's attributes accordingly.
|
||||
|
||||
Args:
|
||||
serialized_data (Dict[str, Any]): A dictionary containing serialized data.
|
||||
"""
|
||||
if not serialized_data:
|
||||
return
|
||||
workdir = serialized_data.get("workdir")
|
||||
if workdir:
|
||||
self.git_repo = GitRepository(local_path=workdir, auto_init=True)
|
||||
self.repo = ProjectRepo(self.git_repo)
|
||||
src_workspace = self.git_repo.workdir / self.git_repo.workdir.name
|
||||
if src_workspace.exists():
|
||||
self.src_workspace = src_workspace
|
||||
kwargs = serialized_data.get("kwargs")
|
||||
if kwargs:
|
||||
for k, v in kwargs.items():
|
||||
self.kwargs.set(k, v)
|
||||
cost_manager = serialized_data.get("cost_manager")
|
||||
if cost_manager:
|
||||
self.cost_manager.model_validate_json(cost_manager)
|
||||
|
|
|
|||
|
|
@ -11,12 +11,9 @@ from pathlib import Path
|
|||
from typing import Optional, Union
|
||||
|
||||
import pandas as pd
|
||||
from langchain.text_splitter import CharacterTextSplitter
|
||||
from langchain_community.document_loaders import (
|
||||
TextLoader,
|
||||
UnstructuredPDFLoader,
|
||||
UnstructuredWordDocumentLoader,
|
||||
)
|
||||
from llama_index.core import Document, SimpleDirectoryReader
|
||||
from llama_index.core.node_parser import SimpleNodeParser
|
||||
from llama_index.readers.file import PDFReader
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from tqdm import tqdm
|
||||
|
||||
|
|
@ -29,7 +26,7 @@ def validate_cols(content_col: str, df: pd.DataFrame):
|
|||
raise ValueError("Content column not found in DataFrame.")
|
||||
|
||||
|
||||
def read_data(data_path: Path):
|
||||
def read_data(data_path: Path) -> Union[pd.DataFrame, list[Document]]:
|
||||
suffix = data_path.suffix
|
||||
if ".xlsx" == suffix:
|
||||
data = pd.read_excel(data_path)
|
||||
|
|
@ -38,14 +35,13 @@ def read_data(data_path: Path):
|
|||
elif ".json" == suffix:
|
||||
data = pd.read_json(data_path)
|
||||
elif suffix in (".docx", ".doc"):
|
||||
data = UnstructuredWordDocumentLoader(str(data_path), mode="elements").load()
|
||||
data = SimpleDirectoryReader(input_files=[str(data_path)]).load_data()
|
||||
elif ".txt" == suffix:
|
||||
data = TextLoader(str(data_path)).load()
|
||||
text_splitter = CharacterTextSplitter(separator="\n", chunk_size=256, chunk_overlap=0)
|
||||
texts = text_splitter.split_documents(data)
|
||||
data = texts
|
||||
data = SimpleDirectoryReader(input_files=[str(data_path)]).load_data()
|
||||
node_parser = SimpleNodeParser.from_defaults(separator="\n", chunk_size=256, chunk_overlap=0)
|
||||
data = node_parser.get_nodes_from_documents(data)
|
||||
elif ".pdf" == suffix:
|
||||
data = UnstructuredPDFLoader(str(data_path), mode="elements").load()
|
||||
data = PDFReader.load_data(str(data_path))
|
||||
else:
|
||||
raise NotImplementedError("File format not supported.")
|
||||
return data
|
||||
|
|
@ -150,9 +146,9 @@ class IndexableDocument(Document):
|
|||
metadatas.append({})
|
||||
return docs, metadatas
|
||||
|
||||
def _get_docs_and_metadatas_by_langchain(self) -> (list, list):
|
||||
def _get_docs_and_metadatas_by_llamaindex(self) -> (list, list):
|
||||
data = self.data
|
||||
docs = [i.page_content for i in data]
|
||||
docs = [i.text for i in data]
|
||||
metadatas = [i.metadata for i in data]
|
||||
return docs, metadatas
|
||||
|
||||
|
|
@ -160,7 +156,7 @@ class IndexableDocument(Document):
|
|||
if isinstance(self.data, pd.DataFrame):
|
||||
return self._get_docs_and_metadatas_by_df()
|
||||
elif isinstance(self.data, list):
|
||||
return self._get_docs_and_metadatas_by_langchain()
|
||||
return self._get_docs_and_metadatas_by_llamaindex()
|
||||
else:
|
||||
raise NotImplementedError("Data type not supported for metadata extraction.")
|
||||
|
||||
|
|
|
|||
|
|
@ -38,9 +38,9 @@ class LocalStore(BaseStore, ABC):
|
|||
if not self.store:
|
||||
self.store = self.write()
|
||||
|
||||
def _get_index_and_store_fname(self, index_ext=".index", pkl_ext=".pkl"):
|
||||
index_file = self.cache_dir / f"{self.fname}{index_ext}"
|
||||
store_file = self.cache_dir / f"{self.fname}{pkl_ext}"
|
||||
def _get_index_and_store_fname(self, index_ext=".json", docstore_ext=".json"):
|
||||
index_file = self.cache_dir / "default__vector_store" / index_ext
|
||||
store_file = self.cache_dir / "docstore" / docstore_ext
|
||||
return index_file, store_file
|
||||
|
||||
@abstractmethod
|
||||
|
|
|
|||
|
|
@ -11,9 +11,9 @@ import chromadb
|
|||
class ChromaStore:
|
||||
"""If inherited from BaseStore, or importing other modules from metagpt, a Python exception occurs, which is strange."""
|
||||
|
||||
def __init__(self, name):
|
||||
def __init__(self, name: str, get_or_create: bool = False):
|
||||
client = chromadb.Client()
|
||||
collection = client.create_collection(name)
|
||||
collection = client.create_collection(name, get_or_create=get_or_create)
|
||||
self.client = client
|
||||
self.collection = collection
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,14 @@
|
|||
"""
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from langchain.vectorstores import FAISS
|
||||
from langchain_core.embeddings import Embeddings
|
||||
import faiss
|
||||
from llama_index.core import VectorStoreIndex, load_index_from_storage
|
||||
from llama_index.core.embeddings import BaseEmbedding
|
||||
from llama_index.core.schema import Document, QueryBundle, TextNode
|
||||
from llama_index.core.storage import StorageContext
|
||||
from llama_index.vector_stores.faiss import FaissVectorStore
|
||||
|
||||
from metagpt.document import IndexableDocument
|
||||
from metagpt.document_store.base_store import LocalStore
|
||||
|
|
@ -20,36 +24,50 @@ from metagpt.utils.embedding import get_embedding
|
|||
|
||||
class FaissStore(LocalStore):
|
||||
def __init__(
|
||||
self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output", embedding: Embeddings = None
|
||||
self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output", embedding: BaseEmbedding = None
|
||||
):
|
||||
self.meta_col = meta_col
|
||||
self.content_col = content_col
|
||||
self.embedding = embedding or get_embedding()
|
||||
self.store: VectorStoreIndex
|
||||
super().__init__(raw_data, cache_dir)
|
||||
|
||||
def _load(self) -> Optional["FaissStore"]:
|
||||
index_file, store_file = self._get_index_and_store_fname(index_ext=".faiss") # langchain FAISS using .faiss
|
||||
def _load(self) -> Optional["VectorStoreIndex"]:
|
||||
index_file, store_file = self._get_index_and_store_fname()
|
||||
|
||||
if not (index_file.exists() and store_file.exists()):
|
||||
logger.info("Missing at least one of index_file/store_file, load failed and return None")
|
||||
return None
|
||||
vector_store = FaissVectorStore.from_persist_dir(persist_dir=self.cache_dir)
|
||||
storage_context = StorageContext.from_defaults(persist_dir=self.cache_dir, vector_store=vector_store)
|
||||
index = load_index_from_storage(storage_context, embed_model=self.embedding)
|
||||
|
||||
return FAISS.load_local(self.raw_data_path.parent, self.embedding, self.fname)
|
||||
return index
|
||||
|
||||
def _write(self, docs, metadatas):
|
||||
store = FAISS.from_texts(docs, self.embedding, metadatas=metadatas)
|
||||
return store
|
||||
def _write(self, docs: list[str], metadatas: list[dict[str, Any]]) -> VectorStoreIndex:
|
||||
assert len(docs) == len(metadatas)
|
||||
documents = [Document(text=doc, metadata=metadatas[idx]) for idx, doc in enumerate(docs)]
|
||||
|
||||
vector_store = FaissVectorStore(faiss_index=faiss.IndexFlatL2(1536))
|
||||
storage_context = StorageContext.from_defaults(vector_store=vector_store)
|
||||
index = VectorStoreIndex.from_documents(
|
||||
documents=documents, storage_context=storage_context, embed_model=self.embedding
|
||||
)
|
||||
|
||||
return index
|
||||
|
||||
def persist(self):
|
||||
self.store.save_local(self.raw_data_path.parent, self.fname)
|
||||
self.store.storage_context.persist(self.cache_dir)
|
||||
|
||||
def search(self, query: str, expand_cols=False, sep="\n", *args, k=5, **kwargs):
|
||||
retriever = self.store.as_retriever(similarity_top_k=k)
|
||||
rsp = retriever.retrieve(QueryBundle(query_str=query, embedding=self.embedding.get_text_embedding(query)))
|
||||
|
||||
def search(self, query, expand_cols=False, sep="\n", *args, k=5, **kwargs):
|
||||
rsp = self.store.similarity_search(query, k=k, **kwargs)
|
||||
logger.debug(rsp)
|
||||
if expand_cols:
|
||||
return str(sep.join([f"{x.page_content}: {x.metadata}" for x in rsp]))
|
||||
return str(sep.join([f"{x.node.text}: {x.node.metadata}" for x in rsp]))
|
||||
else:
|
||||
return str(sep.join([f"{x.page_content}" for x in rsp]))
|
||||
return str(sep.join([f"{x.node.text}" for x in rsp]))
|
||||
|
||||
async def asearch(self, *args, **kwargs):
|
||||
return await asyncio.to_thread(self.search, *args, **kwargs)
|
||||
|
|
@ -67,8 +85,12 @@ class FaissStore(LocalStore):
|
|||
|
||||
def add(self, texts: list[str], *args, **kwargs) -> list[str]:
|
||||
"""FIXME: Currently, the store is not updated after adding."""
|
||||
return self.store.add_texts(texts)
|
||||
texts_embeds = self.embedding.get_text_embedding_batch(texts)
|
||||
nodes = [TextNode(text=texts[idx], embedding=embed) for idx, embed in enumerate(texts_embeds)]
|
||||
self.store.insert_nodes(nodes)
|
||||
|
||||
return []
|
||||
|
||||
def delete(self, *args, **kwargs):
|
||||
"""Currently, langchain does not provide a delete interface."""
|
||||
"""Currently, faiss does not provide a delete interface."""
|
||||
raise NotImplementedError
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ ## Usage
|
|||
from metagpt.environment.api.env_api import EnvAPIAbstract
|
||||
|
||||
# get screenshot from ExtEnv
|
||||
screenshot_path: Path = env.observe(
|
||||
screenshot_path: Path = await env.observe(
|
||||
EnvAPIAbstract(
|
||||
api_name="get_screenshot", kwargs={"ss_name": f"{round_count}_before", "local_save_dir": task_dir}
|
||||
)
|
||||
|
|
@ -34,5 +34,5 @@ # do a `tap` action on the screen
|
|||
## TODO
|
||||
- add android app operation assistant under `examples/android_assistant`
|
||||
- migrate roles/actions of werewolf game from old version into current version
|
||||
- migrate roles/actions of mincraft game from old version into current version
|
||||
- migrate roles/actions of minecraft game from old version into current version
|
||||
- migrate roles/actions of stanford_town game from old version into current version
|
||||
|
|
|
|||
|
|
@ -3,11 +3,10 @@
|
|||
# @Desc :
|
||||
|
||||
from metagpt.environment.base_env import Environment
|
||||
from metagpt.environment.android_env.android_env import AndroidEnv
|
||||
from metagpt.environment.mincraft_env.mincraft_env import MincraftExtEnv
|
||||
from metagpt.environment.werewolf_env.werewolf_env import WerewolfEnv
|
||||
from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv
|
||||
from metagpt.environment.software_env.software_env import SoftwareEnv
|
||||
from metagpt.environment.android.android_env import AndroidEnv
|
||||
from metagpt.environment.werewolf.werewolf_env import WerewolfEnv
|
||||
from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv
|
||||
from metagpt.environment.software.software_env import SoftwareEnv
|
||||
|
||||
|
||||
__all__ = ["AndroidEnv", "MincraftExtEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"]
|
||||
__all__ = ["AndroidEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"]
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@
|
|||
|
||||
from pydantic import Field
|
||||
|
||||
from metagpt.environment.android_env.android_ext_env import AndroidExtEnv
|
||||
from metagpt.environment.android.android_ext_env import AndroidExtEnv
|
||||
from metagpt.environment.base_env import Environment
|
||||
|
||||
|
||||
class AndroidEnv(Environment, AndroidExtEnv):
|
||||
class AndroidEnv(AndroidExtEnv, Environment):
|
||||
"""in order to use actual `reset`&`observe`, inherited order: AndroidExtEnv, Environment"""
|
||||
|
||||
rows: int = Field(default=0, description="rows of a grid on the screenshot")
|
||||
cols: int = Field(default=0, description="cols of a grid on the screenshot")
|
||||
|
|
@ -8,7 +8,14 @@ from typing import Any, Optional
|
|||
|
||||
from pydantic import Field
|
||||
|
||||
from metagpt.environment.android_env.const import ADB_EXEC_FAIL
|
||||
from metagpt.environment.android.const import ADB_EXEC_FAIL
|
||||
from metagpt.environment.android.env_space import (
|
||||
EnvAction,
|
||||
EnvActionType,
|
||||
EnvObsParams,
|
||||
EnvObsType,
|
||||
EnvObsValType,
|
||||
)
|
||||
from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable
|
||||
|
||||
|
||||
|
|
@ -21,11 +28,70 @@ class AndroidExtEnv(ExtEnv):
|
|||
|
||||
def __init__(self, **data: Any):
|
||||
super().__init__(**data)
|
||||
if data.get("device_id"):
|
||||
device_id = data.get("device_id")
|
||||
if device_id:
|
||||
devices = self.list_devices()
|
||||
if device_id not in devices:
|
||||
raise RuntimeError(f"device-id: {device_id} not found")
|
||||
(width, height) = self.device_shape
|
||||
self.width = data.get("width", width)
|
||||
self.height = data.get("height", height)
|
||||
|
||||
self.create_device_path(self.screenshot_dir)
|
||||
self.create_device_path(self.xml_dir)
|
||||
|
||||
def reset(
|
||||
self,
|
||||
*,
|
||||
seed: Optional[int] = None,
|
||||
options: Optional[dict[str, Any]] = None,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
super().reset(seed=seed, options=options)
|
||||
|
||||
obs = self._get_obs()
|
||||
|
||||
return obs, {}
|
||||
|
||||
def _get_obs(self) -> dict[str, EnvObsValType]:
|
||||
pass
|
||||
|
||||
def observe(self, obs_params: Optional[EnvObsParams] = None) -> Any:
|
||||
obs_type = obs_params.obs_type if obs_params else EnvObsType.NONE
|
||||
if obs_type == EnvObsType.NONE:
|
||||
pass
|
||||
elif obs_type == EnvObsType.GET_SCREENSHOT:
|
||||
obs = self.get_screenshot(ss_name=obs_params.ss_name, local_save_dir=obs_params.local_save_dir)
|
||||
elif obs_type == EnvObsType.GET_XML:
|
||||
obs = self.get_xml(xml_name=obs_params.xml_name, local_save_dir=obs_params.local_save_dir)
|
||||
return obs
|
||||
|
||||
def step(self, action: EnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]:
|
||||
res = self._execute_env_action(action)
|
||||
|
||||
obs = {}
|
||||
|
||||
ret = (obs, 1.0, False, False, {"res": res})
|
||||
return ret
|
||||
|
||||
def _execute_env_action(self, action: EnvAction):
|
||||
action_type = action.action_type
|
||||
res = None
|
||||
if action_type == EnvActionType.NONE:
|
||||
pass
|
||||
elif action_type == EnvActionType.SYSTEM_BACK:
|
||||
res = self.system_back()
|
||||
elif action_type == EnvActionType.SYSTEM_TAP:
|
||||
res = self.system_tap(x=action.coord[0], y=action.coord[1])
|
||||
elif action_type == EnvActionType.USER_INPUT:
|
||||
res = self.user_input(input_txt=action.input_txt)
|
||||
elif action_type == EnvActionType.USER_LONGPRESS:
|
||||
res = self.user_longpress(x=action.coord[0], y=action.coord[1])
|
||||
elif action_type == EnvActionType.USER_SWIPE:
|
||||
res = self.user_swipe(x=action.coord[0], y=action.coord[1], orient=action.orient, dist=action.dist)
|
||||
elif action_type == EnvActionType.USER_SWIPE_TO:
|
||||
res = self.user_swipe_to(start=action.coord, end=action.tgt_coord)
|
||||
return res
|
||||
|
||||
@property
|
||||
def adb_prefix_si(self):
|
||||
"""adb cmd prefix with `device_id` and `shell input`"""
|
||||
|
|
@ -42,12 +108,19 @@ class AndroidExtEnv(ExtEnv):
|
|||
return f"adb -s {self.device_id} "
|
||||
|
||||
def execute_adb_with_cmd(self, adb_cmd: str) -> str:
|
||||
adb_cmd = adb_cmd.replace("\\", "/")
|
||||
res = subprocess.run(adb_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
exec_res = ADB_EXEC_FAIL
|
||||
if not res.returncode:
|
||||
exec_res = res.stdout.strip()
|
||||
return exec_res
|
||||
|
||||
def create_device_path(self, folder_path: Path):
|
||||
adb_cmd = f"{self.adb_prefix_shell} mkdir {folder_path} -p"
|
||||
res = self.execute_adb_with_cmd(adb_cmd)
|
||||
if res == ADB_EXEC_FAIL:
|
||||
raise RuntimeError(f"create device path: {folder_path} failed")
|
||||
|
||||
@property
|
||||
def device_shape(self) -> tuple[int, int]:
|
||||
adb_cmd = f"{self.adb_prefix_shell} wm size"
|
||||
92
metagpt/environment/android/env_space.py
Normal file
92
metagpt/environment/android/env_space.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Desc :
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
from gymnasium import spaces
|
||||
from pydantic import ConfigDict, Field, field_validator
|
||||
|
||||
from metagpt.environment.base_env_space import (
|
||||
BaseEnvAction,
|
||||
BaseEnvActionType,
|
||||
BaseEnvObsParams,
|
||||
BaseEnvObsType,
|
||||
)
|
||||
|
||||
|
||||
class EnvActionType(BaseEnvActionType):
|
||||
NONE = 0 # no action to run, just get observation
|
||||
|
||||
SYSTEM_BACK = 1
|
||||
SYSTEM_TAP = 2
|
||||
USER_INPUT = 3
|
||||
USER_LONGPRESS = 4
|
||||
USER_SWIPE = 5
|
||||
USER_SWIPE_TO = 6
|
||||
|
||||
|
||||
class EnvAction(BaseEnvAction):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
action_type: int = Field(default=EnvActionType.NONE, description="action type")
|
||||
coord: npt.NDArray[np.int64] = Field(
|
||||
default_factory=lambda: np.zeros(2, dtype=np.int64), description="operation coordinate"
|
||||
)
|
||||
tgt_coord: npt.NDArray[np.int64] = Field(
|
||||
default_factory=lambda: np.zeros(2, dtype=np.int64), description="target operation coordinate"
|
||||
)
|
||||
input_txt: str = Field(default="", description="user input text")
|
||||
orient: str = Field(default="up", description="swipe orient")
|
||||
dist: str = Field(default="medium", description="swipe dist")
|
||||
|
||||
@field_validator("coord", "tgt_coord", mode="before")
|
||||
@classmethod
|
||||
def check_coord(cls, coord) -> npt.NDArray[np.int64]:
|
||||
if not isinstance(coord, np.ndarray):
|
||||
return np.array(coord)
|
||||
|
||||
|
||||
class EnvObsType(BaseEnvObsType):
|
||||
NONE = 0 # get whole observation from env
|
||||
|
||||
GET_SCREENSHOT = 1
|
||||
GET_XML = 2
|
||||
|
||||
|
||||
class EnvObsParams(BaseEnvObsParams):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
obs_type: int = Field(default=EnvObsType.NONE, description="observation type")
|
||||
ss_name: str = Field(default="", description="screenshot file name")
|
||||
xml_name: str = Field(default="", description="xml file name")
|
||||
local_save_dir: Union[str, Path] = Field(default="", description="local dir to save file")
|
||||
|
||||
|
||||
EnvObsValType = str
|
||||
|
||||
|
||||
def get_observation_space() -> spaces.Dict:
|
||||
space = spaces.Dict({"screenshot": spaces.Text(256), "xml": spaces.Text(256)})
|
||||
return space
|
||||
|
||||
|
||||
def get_action_space(device_shape: tuple[int, int]) -> spaces.Dict:
|
||||
space = spaces.Dict(
|
||||
{
|
||||
"action_type": spaces.Discrete(len(EnvActionType)),
|
||||
"coord": spaces.Box(
|
||||
np.array([0, 0], dtype=np.int64), np.array([device_shape[0], device_shape[1]], dtype=np.int64)
|
||||
),
|
||||
"tgt_coord": spaces.Box(
|
||||
np.array([0, 0], dtype=np.int64), np.array([device_shape[0], device_shape[1]], dtype=np.int64)
|
||||
),
|
||||
"input_txt": spaces.Text(256),
|
||||
"orient": spaces.Text(16),
|
||||
"dist": spaces.Text(16),
|
||||
}
|
||||
)
|
||||
return space
|
||||
|
|
@ -18,11 +18,11 @@ class EnvAPIAbstract(BaseModel):
|
|||
class EnvAPIRegistry(BaseModel):
|
||||
"""the registry to store environment w&r api/interface"""
|
||||
|
||||
registry: dict[str, dict[str, Union[dict, Any, str]]] = Field(default=dict(), exclude=True)
|
||||
registry: dict[str, Callable] = Field(default=dict(), exclude=True)
|
||||
|
||||
def get(self, api_name: str):
|
||||
if api_name not in self.registry:
|
||||
raise ValueError
|
||||
raise KeyError(f"api_name: {api_name} not found")
|
||||
return self.registry.get(api_name)
|
||||
|
||||
def __getitem__(self, api_name: str) -> Callable:
|
||||
|
|
|
|||
|
|
@ -3,9 +3,12 @@
|
|||
# @Desc : base env of executing environment
|
||||
|
||||
import asyncio
|
||||
from abc import abstractmethod
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Set, Union
|
||||
|
||||
from gymnasium import spaces
|
||||
from gymnasium.core import ActType, ObsType
|
||||
from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, model_validator
|
||||
|
||||
from metagpt.context import Context
|
||||
|
|
@ -14,6 +17,7 @@ from metagpt.environment.api.env_api import (
|
|||
ReadAPIRegistry,
|
||||
WriteAPIRegistry,
|
||||
)
|
||||
from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams
|
||||
from metagpt.logs import logger
|
||||
from metagpt.schema import Message
|
||||
from metagpt.utils.common import get_function_schema, is_coroutine_func, is_send_to
|
||||
|
|
@ -26,7 +30,7 @@ class EnvType(Enum):
|
|||
ANDROID = "Android"
|
||||
GYM = "Gym"
|
||||
WEREWOLF = "Werewolf"
|
||||
MINCRAFT = "Mincraft"
|
||||
MINECRAFT = "Minecraft"
|
||||
STANFORDTOWN = "StanfordTown"
|
||||
|
||||
|
||||
|
|
@ -47,7 +51,12 @@ def mark_as_writeable(func):
|
|||
|
||||
|
||||
class ExtEnv(BaseModel):
|
||||
"""External Env to intergate actual game environment"""
|
||||
"""External Env to integrate actual game environment"""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
action_space: spaces.Space[ActType] = Field(default_factory=spaces.Space, exclude=True)
|
||||
observation_space: spaces.Space[ObsType] = Field(default_factory=spaces.Space, exclude=True)
|
||||
|
||||
def _check_api_exist(self, rw_api: Optional[str] = None):
|
||||
if not rw_api:
|
||||
|
|
@ -61,39 +70,56 @@ class ExtEnv(BaseModel):
|
|||
else:
|
||||
return env_write_api_registry.get_apis()
|
||||
|
||||
async def observe(self, env_action: Union[str, EnvAPIAbstract]):
|
||||
async def read_from_api(self, env_action: Union[str, EnvAPIAbstract]):
|
||||
"""get observation from particular api of ExtEnv"""
|
||||
if isinstance(env_action, str):
|
||||
read_api = env_read_api_registry.get(api_name=env_action)["func"]
|
||||
self._check_api_exist(read_api)
|
||||
if is_coroutine_func(read_api):
|
||||
res = await read_api(self)
|
||||
env_read_api = env_read_api_registry.get(api_name=env_action)["func"]
|
||||
self._check_api_exist(env_read_api)
|
||||
if is_coroutine_func(env_read_api):
|
||||
res = await env_read_api(self)
|
||||
else:
|
||||
res = read_api(self)
|
||||
res = env_read_api(self)
|
||||
elif isinstance(env_action, EnvAPIAbstract):
|
||||
read_api = env_read_api_registry.get(api_name=env_action.api_name)["func"]
|
||||
self._check_api_exist(read_api)
|
||||
if is_coroutine_func(read_api):
|
||||
res = await read_api(self, *env_action.args, **env_action.kwargs)
|
||||
env_read_api = env_read_api_registry.get(api_name=env_action.api_name)["func"]
|
||||
self._check_api_exist(env_read_api)
|
||||
if is_coroutine_func(env_read_api):
|
||||
res = await env_read_api(self, *env_action.args, **env_action.kwargs)
|
||||
else:
|
||||
res = read_api(self, *env_action.args, **env_action.kwargs)
|
||||
res = env_read_api(self, *env_action.args, **env_action.kwargs)
|
||||
return res
|
||||
|
||||
async def step(self, env_action: Union[str, Message, EnvAPIAbstract, list[EnvAPIAbstract]]):
|
||||
async def write_thru_api(self, env_action: Union[str, Message, EnvAPIAbstract, list[EnvAPIAbstract]]):
|
||||
"""execute through particular api of ExtEnv"""
|
||||
res = None
|
||||
if isinstance(env_action, Message):
|
||||
self.publish_message(env_action)
|
||||
elif isinstance(env_action, EnvAPIAbstract):
|
||||
write_api = env_write_api_registry.get(env_action.api_name)["func"]
|
||||
self._check_api_exist(write_api)
|
||||
if is_coroutine_func(write_api):
|
||||
res = await write_api(self, *env_action.args, **env_action.kwargs)
|
||||
env_write_api = env_write_api_registry.get(env_action.api_name)["func"]
|
||||
self._check_api_exist(env_write_api)
|
||||
if is_coroutine_func(env_write_api):
|
||||
res = await env_write_api(self, *env_action.args, **env_action.kwargs)
|
||||
else:
|
||||
res = write_api(self, *env_action.args, **env_action.kwargs)
|
||||
res = env_write_api(self, *env_action.args, **env_action.kwargs)
|
||||
|
||||
return res
|
||||
|
||||
@abstractmethod
|
||||
def reset(
|
||||
self,
|
||||
*,
|
||||
seed: Optional[int] = None,
|
||||
options: Optional[dict[str, Any]] = None,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
"""Implement this to get init observation"""
|
||||
|
||||
@abstractmethod
|
||||
def observe(self, obs_params: Optional[BaseEnvObsParams] = None) -> Any:
|
||||
"""Implement this if you want to get partial observation from the env"""
|
||||
|
||||
@abstractmethod
|
||||
def step(self, action: BaseEnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]:
|
||||
"""Implement this to feed a action and then get new observation from the env"""
|
||||
|
||||
|
||||
class Environment(ExtEnv):
|
||||
"""环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到
|
||||
|
|
@ -108,6 +134,20 @@ class Environment(ExtEnv):
|
|||
history: str = "" # For debug
|
||||
context: Context = Field(default_factory=Context, exclude=True)
|
||||
|
||||
def reset(
|
||||
self,
|
||||
*,
|
||||
seed: Optional[int] = None,
|
||||
options: Optional[dict[str, Any]] = None,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
pass
|
||||
|
||||
def observe(self, obs_params: Optional[BaseEnvObsParams] = None) -> Any:
|
||||
pass
|
||||
|
||||
def step(self, action: BaseEnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]:
|
||||
pass
|
||||
|
||||
@model_validator(mode="after")
|
||||
def init_roles(self):
|
||||
self.add_roles(self.roles.values())
|
||||
|
|
@ -129,8 +169,8 @@ class Environment(ExtEnv):
|
|||
self.roles[role.profile] = role
|
||||
|
||||
for role in roles: # setup system message with roles
|
||||
role.set_env(self)
|
||||
role.context = self.context
|
||||
role.set_env(self)
|
||||
|
||||
def publish_message(self, message: Message, peekable: bool = True) -> bool:
|
||||
"""
|
||||
|
|
|
|||
33
metagpt/environment/base_env_space.py
Normal file
33
metagpt/environment/base_env_space.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Desc :
|
||||
|
||||
from enum import IntEnum
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class BaseEnvActionType(IntEnum):
|
||||
# # NONE = 0 # no action to run, just get observation
|
||||
pass
|
||||
|
||||
|
||||
class BaseEnvAction(BaseModel):
|
||||
"""env action type and its related params of action functions/apis"""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
action_type: int = Field(default=0, description="action type")
|
||||
|
||||
|
||||
class BaseEnvObsType(IntEnum):
|
||||
# # NONE = 0 # get whole observation from env
|
||||
pass
|
||||
|
||||
|
||||
class BaseEnvObsParams(BaseModel):
|
||||
"""observation params for different EnvObsType to get its observe result"""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
obs_type: int = Field(default=0, description="observation type")
|
||||
|
|
@ -4,8 +4,8 @@
|
|||
|
||||
from metagpt.const import METAGPT_ROOT
|
||||
|
||||
# For Mincraft Game Agent
|
||||
MC_CKPT_DIR = METAGPT_ROOT / "data/mincraft/ckpt"
|
||||
# For Minecraft Game Agent
|
||||
MC_CKPT_DIR = METAGPT_ROOT / "data/minecraft/ckpt"
|
||||
MC_LOG_DIR = METAGPT_ROOT / "logs"
|
||||
MC_DEFAULT_WARMUP = {
|
||||
"context": 15,
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Desc : MG Mincraft Env
|
||||
# @Desc : MG Minecraft Env
|
||||
# refs to `voyager voyager.py`
|
||||
|
||||
import json
|
||||
|
|
@ -8,20 +8,19 @@ import re
|
|||
import time
|
||||
from typing import Any, Iterable
|
||||
|
||||
from langchain.embeddings.openai import OpenAIEmbeddings
|
||||
from langchain.vectorstores import Chroma
|
||||
from llama_index.vector_stores.chroma import ChromaVectorStore
|
||||
from pydantic import ConfigDict, Field
|
||||
|
||||
from metagpt.config2 import config as CONFIG
|
||||
from metagpt.environment.base_env import Environment
|
||||
from metagpt.environment.mincraft_env.const import MC_CKPT_DIR
|
||||
from metagpt.environment.mincraft_env.mincraft_ext_env import MincraftExtEnv
|
||||
from metagpt.environment.minecraft.const import MC_CKPT_DIR
|
||||
from metagpt.environment.minecraft.minecraft_ext_env import MinecraftExtEnv
|
||||
from metagpt.logs import logger
|
||||
from metagpt.utils.common import load_mc_skills_code, read_json_file, write_json_file
|
||||
|
||||
|
||||
class MincraftEnv(Environment, MincraftExtEnv):
|
||||
"""MincraftEnv, including shared memory of cache and infomation between roles"""
|
||||
class MinecraftEnv(MinecraftExtEnv, Environment):
|
||||
"""MinecraftEnv, including shared memory of cache and information between roles"""
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
|
|
@ -48,9 +47,9 @@ class MincraftEnv(Environment, MincraftExtEnv):
|
|||
|
||||
runtime_status: bool = False # equal to action execution status: success or failed
|
||||
|
||||
vectordb: Chroma = Field(default_factory=Chroma)
|
||||
vectordb: ChromaVectorStore = Field(default_factory=ChromaVectorStore)
|
||||
|
||||
qa_cache_questions_vectordb: Chroma = Field(default_factory=Chroma)
|
||||
qa_cache_questions_vectordb: ChromaVectorStore = Field(default_factory=ChromaVectorStore)
|
||||
|
||||
@property
|
||||
def progress(self):
|
||||
|
|
@ -73,16 +72,14 @@ class MincraftEnv(Environment, MincraftExtEnv):
|
|||
self.set_mc_resume()
|
||||
|
||||
def set_mc_resume(self):
|
||||
self.qa_cache_questions_vectordb = Chroma(
|
||||
self.qa_cache_questions_vectordb = ChromaVectorStore(
|
||||
collection_name="qa_cache_questions_vectordb",
|
||||
embedding_function=OpenAIEmbeddings(),
|
||||
persist_directory=f"{MC_CKPT_DIR}/curriculum/vectordb",
|
||||
persist_dir=f"{MC_CKPT_DIR}/curriculum/vectordb",
|
||||
)
|
||||
|
||||
self.vectordb = Chroma(
|
||||
self.vectordb = ChromaVectorStore(
|
||||
collection_name="skill_vectordb",
|
||||
embedding_function=OpenAIEmbeddings(),
|
||||
persist_directory=f"{MC_CKPT_DIR}/skill/vectordb",
|
||||
persist_dir=f"{MC_CKPT_DIR}/skill/vectordb",
|
||||
)
|
||||
|
||||
if CONFIG.resume:
|
||||
|
|
@ -285,7 +282,7 @@ class MincraftEnv(Environment, MincraftExtEnv):
|
|||
position = event["status"]["position"]
|
||||
blocks.append(block)
|
||||
positions.append(position)
|
||||
new_events = self.step(
|
||||
new_events = self._step(
|
||||
f"await givePlacedItemBack(bot, {json.dumps(blocks)}, {json.dumps(positions)})",
|
||||
programs=self.programs,
|
||||
)
|
||||
|
|
@ -326,7 +323,7 @@ class MincraftEnv(Environment, MincraftExtEnv):
|
|||
Exception: If there is an issue retrieving events.
|
||||
"""
|
||||
try:
|
||||
self.reset(
|
||||
self._reset(
|
||||
options={
|
||||
"mode": "soft",
|
||||
"wait_ticks": 20,
|
||||
|
|
@ -335,13 +332,13 @@ class MincraftEnv(Environment, MincraftExtEnv):
|
|||
# difficulty = "easy" if len(self.completed_tasks) > 15 else "peaceful"
|
||||
difficulty = "peaceful"
|
||||
|
||||
events = self.step("bot.chat(`/time set ${getNextTime()}`);\n" + f"bot.chat('/difficulty {difficulty}');")
|
||||
events = self._step("bot.chat(`/time set ${getNextTime()}`);\n" + f"bot.chat('/difficulty {difficulty}');")
|
||||
self.update_event(events)
|
||||
return events
|
||||
except Exception as e:
|
||||
time.sleep(3) # wait for mineflayer to exit
|
||||
# reset bot status here
|
||||
events = self.reset(
|
||||
events = self._reset(
|
||||
options={
|
||||
"mode": "hard",
|
||||
"wait_ticks": 20,
|
||||
|
|
@ -368,7 +365,7 @@ class MincraftEnv(Environment, MincraftExtEnv):
|
|||
Exception: If there is an issue retrieving events.
|
||||
"""
|
||||
try:
|
||||
events = self.step(
|
||||
events = self._step(
|
||||
code=self.code,
|
||||
programs=self.programs,
|
||||
)
|
||||
|
|
@ -377,7 +374,7 @@ class MincraftEnv(Environment, MincraftExtEnv):
|
|||
except Exception as e:
|
||||
time.sleep(3) # wait for mineflayer to exit
|
||||
# reset bot status here
|
||||
events = self.reset(
|
||||
events = self._reset(
|
||||
options={
|
||||
"mode": "hard",
|
||||
"wait_ticks": 20,
|
||||
|
|
@ -1,28 +1,29 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Desc : The Mincraft external environment to integrate with Mincraft game
|
||||
# @Desc : The Minecraft external environment to integrate with Minecraft game
|
||||
# refs to `voyager bridge.py`
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
import requests
|
||||
from pydantic import ConfigDict, Field, model_validator
|
||||
|
||||
from metagpt.environment.base_env import ExtEnv, mark_as_writeable
|
||||
from metagpt.environment.mincraft_env.const import (
|
||||
from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams
|
||||
from metagpt.environment.minecraft.const import (
|
||||
MC_CKPT_DIR,
|
||||
MC_CORE_INVENTORY_ITEMS,
|
||||
MC_CURRICULUM_OB,
|
||||
MC_DEFAULT_WARMUP,
|
||||
METAGPT_ROOT,
|
||||
)
|
||||
from metagpt.environment.mincraft_env.process_monitor import SubprocessMonitor
|
||||
from metagpt.environment.minecraft.process_monitor import SubprocessMonitor
|
||||
from metagpt.logs import logger
|
||||
|
||||
|
||||
class MincraftExtEnv(ExtEnv):
|
||||
class MinecraftExtEnv(ExtEnv):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
mc_port: Optional[int] = Field(default=None)
|
||||
|
|
@ -38,6 +39,20 @@ class MincraftExtEnv(ExtEnv):
|
|||
server_paused: bool = Field(default=False)
|
||||
warm_up: dict = Field(default=dict())
|
||||
|
||||
def reset(
|
||||
self,
|
||||
*,
|
||||
seed: Optional[int] = None,
|
||||
options: Optional[dict[str, Any]] = None,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
pass
|
||||
|
||||
def observe(self, obs_params: Optional[BaseEnvObsParams] = None) -> Any:
|
||||
pass
|
||||
|
||||
def step(self, action: BaseEnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]:
|
||||
pass
|
||||
|
||||
@property
|
||||
def server(self) -> str:
|
||||
return f"{self.server_host}:{self.server_port}"
|
||||
|
|
@ -48,7 +63,7 @@ class MincraftExtEnv(ExtEnv):
|
|||
self.mineflayer = SubprocessMonitor(
|
||||
commands=[
|
||||
"node",
|
||||
METAGPT_ROOT.joinpath("metagpt", "environment", "mincraft_env", "mineflayer", "index.js"),
|
||||
METAGPT_ROOT.joinpath("metagpt", "environment", "minecraft", "mineflayer", "index.js"),
|
||||
str(self.server_port),
|
||||
],
|
||||
name="mineflayer",
|
||||
|
|
@ -115,7 +130,7 @@ class MincraftExtEnv(ExtEnv):
|
|||
return res.json()
|
||||
|
||||
@mark_as_writeable
|
||||
def reset(self, *, seed=None, options=None) -> dict:
|
||||
def _reset(self, *, seed=None, options=None) -> dict:
|
||||
if options is None:
|
||||
options = {}
|
||||
if options.get("inventory", {}) and options.get("mode", "hard") != "hard":
|
||||
|
|
@ -145,7 +160,7 @@ class MincraftExtEnv(ExtEnv):
|
|||
return json.loads(returned_data)
|
||||
|
||||
@mark_as_writeable
|
||||
def step(self, code: str, programs: str = "") -> dict:
|
||||
def _step(self, code: str, programs: str = "") -> dict:
|
||||
if not self.has_reset:
|
||||
raise RuntimeError("Environment has not been reset yet")
|
||||
self.check_process()
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue