From f20064a5786540f7364a5cf60b2fa237af3ef4e5 Mon Sep 17 00:00:00 2001 From: eukub Date: Thu, 28 Dec 2023 20:10:33 +0300 Subject: [PATCH 001/127] =?UTF-8?q?=D1=81hanged=20concatenation=20of=20str?= =?UTF-8?q?ings=20to=20f-strings=20to=20improve=20readability=20and=20unif?= =?UTF-8?q?y=20with=20the=20rest=20of=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/action_node.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 3529942c3..e44e5e96e 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -195,10 +195,7 @@ class ActionNode: def tagging(self, text, schema, tag="") -> str: if not tag: return text - if schema == "json": - return f"[{tag}]\n" + text + f"\n[/{tag}]" - else: # markdown - return f"[{tag}]\n" + text + f"\n[/{tag}]" + return f"[{tag}]\n{text}\n[/{tag}]" def _compile_f(self, schema, mode, tag, format_func, kv_sep) -> str: nodes = self.to_dict(format_func=format_func, mode=mode) @@ -230,7 +227,7 @@ class ActionNode: schema="markdown": 编译context, example(markdown), instruction(markdown), constraint, action """ if schema == "raw": - return context + "\n\n## Actions\n" + LANGUAGE_CONSTRAINT + "\n" + self.instruction + return f"{context}\n\n## Actions\n{LANGUAGE_CONSTRAINT}\n{self.instruction}" # FIXME: json instruction会带来格式问题，如："Project name": "web_2048 # 项目名称使用下划线", # compile example暂时不支持markdown From 791ab749adab3d527c6a700bfd89d855a2ba347a Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 21 Feb 2024 16:49:59 +0800 Subject: [PATCH 002/127] refine code to avoid config error --- metagpt/software_company.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 26bb29cd1..f290d497a 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -2,14 +2,11 @@ # -*- coding: utf-8 -*- import asyncio -import shutil from pathlib import Path import typer -from metagpt.config2 import config -from metagpt.const import CONFIG_ROOT, METAGPT_ROOT -from metagpt.context import Context +from metagpt.const import CONFIG_ROOT from metagpt.utils.project_repo import ProjectRepo app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False) @@ -30,6 +27,8 @@ def generate_repo( recover_path=None, ) -> ProjectRepo: """Run the startup logic. Can be called from CLI or other Python scripts.""" + from metagpt.config2 import config + from metagpt.context import Context from metagpt.roles import ( Architect, Engineer, @@ -122,7 +121,17 @@ def startup( ) -def copy_config_to(config_path=METAGPT_ROOT / "config" / "config2.yaml"): +DEFAULT_CONFIG = """# Full Example: https://github.com/geekan/MetaGPT/blob/main/config/config2.example.yaml +# Reflected Code: https://github.com/geekan/MetaGPT/blob/main/metagpt/config2.py +llm: + api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options + model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview + base_url: "https://api.openai.com/v1" # or forward url / other llm url + api_key: "YOUR_API_KEY" +""" + + +def copy_config_to(): """Initialize the configuration file for MetaGPT.""" target_path = CONFIG_ROOT / "config2.yaml" @@ -136,7 +145,7 @@ def copy_config_to(config_path=METAGPT_ROOT / "config" / "config2.yaml"): print(f"Existing configuration file backed up at {backup_path}") # 复制文件 - shutil.copy(str(config_path), target_path) + target_path.write_text(DEFAULT_CONFIG, encoding="utf-8") print(f"Configuration file initialized at {target_path}") From 0a6dc8f7e7d888f7556a0c009c422d61933c0406 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Fri, 23 Feb 2024 10:15:49 +0800 Subject: [PATCH 003/127] update version to 0.7.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ed3d5f78f..f5d880ac9 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.2", + version="0.7.3", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", From ad9492a639502ce990c1a7f3269da927256ea8b2 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 26 Feb 2024 20:02:16 +0800 Subject: [PATCH 004/127] update st_game using MG v0.7 --- examples/st_game/.gitignore | 2 + examples/st_game/README.md | 26 + examples/st_game/README_CN.md | 25 + examples/st_game/__init__.py | 3 + examples/st_game/actions/__init__.py | 3 + .../st_game/actions/agent_chat_sum_rel.py | 39 + examples/st_game/actions/decide_to_talk.py | 97 +++ examples/st_game/actions/dummy_action.py | 23 + .../st_game/actions/gen_action_details.py | 403 ++++++++++ .../st_game/actions/gen_daily_schedule.py | 61 ++ .../st_game/actions/gen_hourly_schedule.py | 181 +++++ examples/st_game/actions/gen_iter_chat_utt.py | 125 +++ .../st_game/actions/inner_voice_action.py | 31 + .../st_game/actions/new_decomp_schedule.py | 154 ++++ .../st_game/actions/run_reflect_action.py | 277 +++++++ examples/st_game/actions/st_action.py | 119 +++ examples/st_game/actions/summarize_conv.py | 47 ++ examples/st_game/actions/task_decomp.py | 175 +++++ examples/st_game/actions/wake_up.py | 43 ++ examples/st_game/memory/__init__.py | 0 examples/st_game/memory/agent_memory.py | 374 +++++++++ examples/st_game/memory/retrieve.py | 180 +++++ examples/st_game/memory/scratch.py | 397 ++++++++++ examples/st_game/memory/spatial_memory.py | 115 +++ examples/st_game/plan/__init__.py | 3 + examples/st_game/plan/converse.py | 93 +++ examples/st_game/plan/st_plan.py | 719 ++++++++++++++++++ examples/st_game/prompts/__init__.py | 3 + .../prompts/action_location_object_vMar11.txt | 30 + .../prompts/action_location_sector_v1.txt | 34 + examples/st_game/prompts/action_object_v2.txt | 32 + .../st_game/prompts/daily_planning_v6.txt | 14 + .../st_game/prompts/decide_to_talk_v2.txt | 18 + .../prompts/generate_event_triple_v1.txt | 30 + .../st_game/prompts/generate_focal_pt_v1.txt | 11 + .../prompts/generate_hourly_schedule_v2.txt | 18 + .../st_game/prompts/generate_obj_event_v1.txt | 16 + .../prompts/generate_pronunciatio_v1.txt | 10 + .../prompts/insight_and_evidence_v1.txt | 12 + .../st_game/prompts/iterative_convo_v1.txt | 46 ++ examples/st_game/prompts/memo_on_convo_v1.txt | 15 + .../prompts/new_decomp_schedule_v1.txt | 24 + .../prompts/planning_thought_on_convo_v1.txt | 15 + .../st_game/prompts/poignancy_action_v1.txt | 15 + .../st_game/prompts/poignancy_chat_v1.txt | 17 + .../st_game/prompts/poignancy_event_v1.txt | 15 + .../st_game/prompts/poignancy_thought_v1.txt | 15 + .../summarize_chat_relationship_v2.txt | 15 + .../prompts/summarize_conversation_v1.txt | 11 + examples/st_game/prompts/task_decomp_v3.txt | 39 + examples/st_game/prompts/wake_up_hour_v1.txt | 12 + .../prompts/whisper_inner_thought_v1.txt | 11 + examples/st_game/reflect/__init__.py | 3 + examples/st_game/reflect/reflect.py | 245 ++++++ examples/st_game/requirements.txt | 0 examples/st_game/roles/__init__.py | 3 + examples/st_game/roles/st_role.py | 627 +++++++++++++++ examples/st_game/run_st_game.py | 75 ++ examples/st_game/stanford_town.py | 48 ++ .../the_ville/agent_history_init_n25.csv | 26 + .../the_ville/agent_history_init_n3.csv | 4 + .../the_ville/matrix/maze/arena_maze.csv | 0 .../the_ville/matrix/maze/collision_maze.csv | 0 .../matrix/maze/game_object_maze.csv | 0 .../the_ville/matrix/maze/sector_maze.csv | 0 .../matrix/maze/spawning_location_maze.csv | 0 .../the_ville/matrix/maze_meta_info.json | 0 .../matrix/special_blocks/arena_blocks.csv | 0 .../special_blocks/game_object_blocks.csv | 0 .../matrix/special_blocks/sector_blocks.csv | 0 .../spawning_location_blocks.csv | 0 .../matrix/special_blocks/world_blocks.csv | 0 examples/st_game/storage/.gitignore | 2 + .../environment/0.json | 26 + .../associative_memory/embeddings.json | 1 + .../associative_memory/kw_strength.json | 2 + .../associative_memory/nodes.json | 1 + .../bootstrap_memory/scratch.json | 51 ++ .../bootstrap_memory/spatial_memory.json | 66 ++ .../test_memory/embeddings.json | 1 + .../test_memory/kw_strength.json | 4 + .../bootstrap_memory/test_memory/nodes.json | 1 + .../associative_memory/embeddings.json | 1 + .../associative_memory/kw_strength.json | 2 + .../associative_memory/nodes.json | 1 + .../bootstrap_memory/scratch.json | 51 ++ .../bootstrap_memory/spatial_memory.json | 86 +++ .../associative_memory/embeddings.json | 1 + .../associative_memory/kw_strength.json | 2 + .../associative_memory/nodes.json | 1 + .../Maria Lopez/bootstrap_memory/scratch.json | 51 ++ .../bootstrap_memory/spatial_memory.json | 87 +++ .../reverie/meta.json | 13 + examples/st_game/tests/__init__.py | 0 examples/st_game/tests/actions/__init__.py | 3 + .../tests/actions/test_gen_action_details.py | 78 ++ .../tests/actions/test_summarize_conv.py | 15 + examples/st_game/tests/plan/__init__.py | 3 + .../st_game/tests/plan/test_conversation.py | 66 ++ examples/st_game/tests/plan/test_st_plan.py | 36 + examples/st_game/tests/roles/__init__.py | 3 + examples/st_game/tests/roles/test_st_role.py | 25 + examples/st_game/tests/test_agent_memory.py | 89 +++ examples/st_game/tests/test_basic_memory.py | 76 ++ examples/st_game/tests/test_reflect.py | 47 ++ examples/st_game/tests/test_spatial_memory.py | 17 + examples/st_game/utils/__init__.py | 3 + examples/st_game/utils/const.py | 13 + examples/st_game/utils/mg_ga_transform.py | 61 ++ examples/st_game/utils/utils.py | 225 ++++++ metagpt/environment/README.md | 2 +- .../test_stanford_town_ext_env.py | 2 +- 112 files changed, 6437 insertions(+), 2 deletions(-) create mode 100644 examples/st_game/.gitignore create mode 100644 examples/st_game/README.md create mode 100644 examples/st_game/README_CN.md create mode 100644 examples/st_game/__init__.py create mode 100644 examples/st_game/actions/__init__.py create mode 100644 examples/st_game/actions/agent_chat_sum_rel.py create mode 100644 examples/st_game/actions/decide_to_talk.py create mode 100644 examples/st_game/actions/dummy_action.py create mode 100644 examples/st_game/actions/gen_action_details.py create mode 100644 examples/st_game/actions/gen_daily_schedule.py create mode 100644 examples/st_game/actions/gen_hourly_schedule.py create mode 100644 examples/st_game/actions/gen_iter_chat_utt.py create mode 100644 examples/st_game/actions/inner_voice_action.py create mode 100644 examples/st_game/actions/new_decomp_schedule.py create mode 100644 examples/st_game/actions/run_reflect_action.py create mode 100644 examples/st_game/actions/st_action.py create mode 100644 examples/st_game/actions/summarize_conv.py create mode 100644 examples/st_game/actions/task_decomp.py create mode 100644 examples/st_game/actions/wake_up.py create mode 100644 examples/st_game/memory/__init__.py create mode 100644 examples/st_game/memory/agent_memory.py create mode 100644 examples/st_game/memory/retrieve.py create mode 100644 examples/st_game/memory/scratch.py create mode 100644 examples/st_game/memory/spatial_memory.py create mode 100644 examples/st_game/plan/__init__.py create mode 100644 examples/st_game/plan/converse.py create mode 100644 examples/st_game/plan/st_plan.py create mode 100644 examples/st_game/prompts/__init__.py create mode 100644 examples/st_game/prompts/action_location_object_vMar11.txt create mode 100644 examples/st_game/prompts/action_location_sector_v1.txt create mode 100644 examples/st_game/prompts/action_object_v2.txt create mode 100644 examples/st_game/prompts/daily_planning_v6.txt create mode 100644 examples/st_game/prompts/decide_to_talk_v2.txt create mode 100644 examples/st_game/prompts/generate_event_triple_v1.txt create mode 100644 examples/st_game/prompts/generate_focal_pt_v1.txt create mode 100644 examples/st_game/prompts/generate_hourly_schedule_v2.txt create mode 100644 examples/st_game/prompts/generate_obj_event_v1.txt create mode 100644 examples/st_game/prompts/generate_pronunciatio_v1.txt create mode 100644 examples/st_game/prompts/insight_and_evidence_v1.txt create mode 100644 examples/st_game/prompts/iterative_convo_v1.txt create mode 100644 examples/st_game/prompts/memo_on_convo_v1.txt create mode 100644 examples/st_game/prompts/new_decomp_schedule_v1.txt create mode 100644 examples/st_game/prompts/planning_thought_on_convo_v1.txt create mode 100644 examples/st_game/prompts/poignancy_action_v1.txt create mode 100644 examples/st_game/prompts/poignancy_chat_v1.txt create mode 100644 examples/st_game/prompts/poignancy_event_v1.txt create mode 100644 examples/st_game/prompts/poignancy_thought_v1.txt create mode 100644 examples/st_game/prompts/summarize_chat_relationship_v2.txt create mode 100644 examples/st_game/prompts/summarize_conversation_v1.txt create mode 100644 examples/st_game/prompts/task_decomp_v3.txt create mode 100644 examples/st_game/prompts/wake_up_hour_v1.txt create mode 100644 examples/st_game/prompts/whisper_inner_thought_v1.txt create mode 100644 examples/st_game/reflect/__init__.py create mode 100644 examples/st_game/reflect/reflect.py create mode 100644 examples/st_game/requirements.txt create mode 100644 examples/st_game/roles/__init__.py create mode 100644 examples/st_game/roles/st_role.py create mode 100644 examples/st_game/run_st_game.py create mode 100644 examples/st_game/stanford_town.py create mode 100644 examples/st_game/static_dirs/assets/the_ville/agent_history_init_n25.csv create mode 100644 examples/st_game/static_dirs/assets/the_ville/agent_history_init_n3.csv rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze/arena_maze.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze/collision_maze.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze/game_object_maze.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze/sector_maze.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze/spawning_location_maze.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze_meta_info.json (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/special_blocks/arena_blocks.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/special_blocks/game_object_blocks.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/special_blocks/sector_blocks.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/special_blocks/spawning_location_blocks.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/special_blocks/world_blocks.csv (100%) create mode 100644 examples/st_game/storage/.gitignore create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/environment/0.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/embeddings.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/kw_strength.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/nodes.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/scratch.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/spatial_memory.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/embeddings.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/kw_strength.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/nodes.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/embeddings.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/kw_strength.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/nodes.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/scratch.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/spatial_memory.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/embeddings.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/kw_strength.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/nodes.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/scratch.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/spatial_memory.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/reverie/meta.json create mode 100644 examples/st_game/tests/__init__.py create mode 100644 examples/st_game/tests/actions/__init__.py create mode 100644 examples/st_game/tests/actions/test_gen_action_details.py create mode 100644 examples/st_game/tests/actions/test_summarize_conv.py create mode 100644 examples/st_game/tests/plan/__init__.py create mode 100644 examples/st_game/tests/plan/test_conversation.py create mode 100644 examples/st_game/tests/plan/test_st_plan.py create mode 100644 examples/st_game/tests/roles/__init__.py create mode 100644 examples/st_game/tests/roles/test_st_role.py create mode 100644 examples/st_game/tests/test_agent_memory.py create mode 100644 examples/st_game/tests/test_basic_memory.py create mode 100644 examples/st_game/tests/test_reflect.py create mode 100644 examples/st_game/tests/test_spatial_memory.py create mode 100644 examples/st_game/utils/__init__.py create mode 100644 examples/st_game/utils/const.py create mode 100644 examples/st_game/utils/mg_ga_transform.py create mode 100644 examples/st_game/utils/utils.py diff --git a/examples/st_game/.gitignore b/examples/st_game/.gitignore new file mode 100644 index 000000000..fe77153f9 --- /dev/null +++ b/examples/st_game/.gitignore @@ -0,0 +1,2 @@ +storage/test* +storage/unittest* \ No newline at end of file diff --git a/examples/st_game/README.md b/examples/st_game/README.md new file mode 100644 index 000000000..cc4ada230 --- /dev/null +++ b/examples/st_game/README.md @@ -0,0 +1,26 @@ +## Stanford Town Game + +### Pre-Description +The path configured in `examples/st_game/utils/const.py` is the storage path of the current project. In order to facilitate GA(generative_agents)'s frontend docking data (to avoid changing its code), you can change the path under `const.py` like beflow + +``` +STORAGE_PATH = ROOT_PATH.joinpath("storage") +TEMP_STORAGE_PATH = ROOT_PATH.joinpath("temp_storage") +# updated +STORAGE_PATH = Path("{path/to/ga/storage}") +TEMP_STORAGE_PATH = Path("{path/to/ga/temp_storage}") +``` + +This can be used to achieve docking of simulation data without changing the GA code. Otherwise, the GA code must be modified to adapt to the MG output path. + +### Backend service startup +The execution entry is `python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` + +`idea` is the user's voice to the first Agent, and it is disseminated through this voice to see whether the final multi-agents achieve the goal of hosting or participating in the event. + +### Frontend service startup +Enter `generative_agents/environment/frontend_server` and use `python manage.py runserver` to start the front-end service. +Visit `http://localhost:8000/simulator_home` to enter the current simulation interface. + +## Appreciation +The reproduction work has referred the `https://github.com/joonspk-research/generative_agents`, let's make a general statement here. diff --git a/examples/st_game/README_CN.md b/examples/st_game/README_CN.md new file mode 100644 index 000000000..9780b3ed8 --- /dev/null +++ b/examples/st_game/README_CN.md @@ -0,0 +1,25 @@ +## Stanford Town Game + +### 前置 +`examples/st_game/utils/const.py`配置的路径为当前项目的存储路径，为了方便GA（generative_agents）的前端对接数据（避免改动它那块的代码），可将`const.py`下的 + +``` +STORAGE_PATH = ROOT_PATH.joinpath("storage") +TEMP_STORAGE_PATH = ROOT_PATH.joinpath("temp_storage") +# 更新为 +STORAGE_PATH = Path("{path/to/ga/storage}") +TEMP_STORAGE_PATH = Path("{path/to/ga/temp_storage}") +``` +这样可用实现不改变GA代码情况下，实现仿真数据的对接。不然得修改GA的代码来适配MG的输出路径。 + +### 后端服务启动 +执行入口为：`python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` + +`idea`为用户给第一个Agent的用户心声，并通过这个心声进行传播，看最后多智能体是否达到举办、参加活动的目标。 + +### 前端服务启动 +进入`generative_agents/environment/frontend_server`，使用`python manage.py runserver`启动前端服务。 +访问`http://localhost:8000/simulator_home` 进入当前的仿真界面。 + +## Appreciation +The reproduction work has referred the `https://github.com/joonspk-research/generative_agents`, let's make a general statement here. diff --git a/examples/st_game/__init__.py b/examples/st_game/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/examples/st_game/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/examples/st_game/actions/__init__.py b/examples/st_game/actions/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/examples/st_game/actions/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/examples/st_game/actions/agent_chat_sum_rel.py b/examples/st_game/actions/agent_chat_sum_rel.py new file mode 100644 index 000000000..00f15e82f --- /dev/null +++ b/examples/st_game/actions/agent_chat_sum_rel.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : summarize relationship in a agent chat + +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +class AgentChatSumRel(STAction): + name: str = "AgentChatSumRel" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + resp = False + try: + _ = llm_resp.split('"')[0].strip() + resp = True + except Exception: + pass + return resp + + def _func_cleanup(self, llm_resp: str, prompt: str) -> str: + return llm_resp.split('"')[0].strip() + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, init_role: "STRole", target_role: "STRole", statements: str) -> str: + def create_prompt_input(init_role: "STRole", target_role: "STRole", statements: str) -> str: + prompt_input = [statements, init_role.name, target_role.name] + return prompt_input + + prompt_input = create_prompt_input(init_role, target_role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "summarize_chat_relationship_v2.txt") + + example_output = "Jane Doe is working on a project" + special_instruction = "The output should be a string that responds to the question." + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {init_role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/decide_to_talk.py b/examples/st_game/actions/decide_to_talk.py new file mode 100644 index 000000000..cabb5ceff --- /dev/null +++ b/examples/st_game/actions/decide_to_talk.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : device to talk to another role, return yes or no + +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +class DecideToTalk(STAction): + name: str = "DecideToTalk" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + resp = False + try: + if llm_resp.split("Answer in yes or no:")[-1].strip().lower() in ["yes", "no"]: + resp = True + except ValueError: + pass + return resp + + def _func_cleanup(self, llm_resp: str, prompt: str) -> str: + return llm_resp.split("Answer in yes or no:")[-1].strip().lower() + + def _func_fail_default_resp(self) -> str: + return "yes" + + async def run(self, init_role: "STRole", target_role: "STRole", retrieved: dict, *args, **kwargs) -> bool: + """Run action""" + + def create_prompt_input(init_role: "STRole", target_role: "STRole", retrieved: dict) -> str: + scratch = init_role.rc.scratch + target_scratch = target_role.rc.scratch + last_chat = init_role.rc.memory.get_last_chat(target_role.name) + last_chatted_time = "" + last_chat_about = "" + if last_chat: + last_chatted_time = last_chat.created.strftime("%B %d, %Y, %H:%M:%S") + last_chat_about = last_chat.description + + context = "" + for c_node in retrieved["events"]: + curr_desc = c_node.description.split(" ") + curr_desc[2:3] = ["was"] + curr_desc = " ".join(curr_desc) + context += f"{curr_desc}. " + context += "\n" + for c_node in retrieved["thoughts"]: + context += f"{c_node.description}. " + + curr_time = scratch.curr_time.strftime("%B %d, %Y, %H:%M:%S %p") + init_act_desc = scratch.act_description + if "(" in init_act_desc: + init_act_desc = init_act_desc.split("(")[-1][:-1] + + if len(scratch.planned_path) == 0 and "waiting" not in init_act_desc: + init_p_desc = f"{init_role.name} is already {init_act_desc}" + elif "waiting" in init_act_desc: + init_p_desc = f"{init_role.name} is {init_act_desc}" + else: + init_p_desc = f"{init_role.name} is on the way to {init_act_desc}" + + target_act_desc = scratch.act_description + if "(" in target_act_desc: + target_act_desc = target_act_desc.split("(")[-1][:-1] + + if len(target_scratch.planned_path) == 0 and "waiting" not in init_act_desc: + target_p_desc = f"{target_role.name} is already {target_act_desc}" + elif "waiting" in init_act_desc: + target_p_desc = f"{init_role.name} is {init_act_desc}" + else: + target_p_desc = f"{target_role.name} is on the way to {target_act_desc}" + + prompt_input = [] + prompt_input += [context] + + prompt_input += [curr_time] + + prompt_input += [init_role.name] + prompt_input += [target_role.name] + prompt_input += [last_chatted_time] + prompt_input += [last_chat_about] + + prompt_input += [init_p_desc] + prompt_input += [target_p_desc] + prompt_input += [init_role.name] + prompt_input += [target_role.name] + return prompt_input + + prompt_input = create_prompt_input(init_role, target_role, retrieved) + prompt = self.generate_prompt_with_tmpl_filename( + prompt_input=prompt_input, tmpl_filename="decide_to_talk_v2.txt" + ) + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=20) # yes or no + result = True if output == "yes" else False + logger.info(f"Role: {init_role.name} Action: {self.cls_name} output: {result}") + return result diff --git a/examples/st_game/actions/dummy_action.py b/examples/st_game/actions/dummy_action.py new file mode 100644 index 000000000..ba87d8516 --- /dev/null +++ b/examples/st_game/actions/dummy_action.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : dummy action to make every STRole can deal DummyMessage which is caused by DummyAction + +from dataclasses import dataclass + +from metagpt.actions import Action +from metagpt.schema import Message + + +class DummyAction(Action): + async def run(self, *args, **kwargs): + raise NotImplementedError + + +@dataclass +class DummyMessage(Message): + """ + dummy message to pass to role and make them to have a execution every round + """ + + content: str = "dummy" + cause_by: str = "DummyAction" diff --git a/examples/st_game/actions/gen_action_details.py b/examples/st_game/actions/gen_action_details.py new file mode 100644 index 000000000..aacc6852a --- /dev/null +++ b/examples/st_game/actions/gen_action_details.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : gen_action_details + +import random + +from metagpt.environment.api.env_api import EnvAPIAbstract +from metagpt.logs import logger + +from .st_action import STAction + + +class GenActionSector(STAction): + name: str = "GenActionSector" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cleaned_response = llm_resp.split("}")[0] + return cleaned_response + + def _func_validate(self, llm_resp: str, prompt: str): + if len(llm_resp.strip()) < 1: + return False + if "}" not in llm_resp: + return False + if "," in llm_resp: + return False + return True + + def _func_fail_default_resp(self): + fs = "kitchen" + return fs + + async def run(self, role: "STRole", access_tile: dict[str, str], act_desp: str): + def create_prompt_input(role, access_tile: dict[str, str], act_desp): + act_world = f"{access_tile['world']}" + + prompt_input = [] + + prompt_input += [role.scratch.get_str_name()] + prompt_input += [role.scratch.living_area.split(":")[1]] + x = f"{act_world}:{role.scratch.living_area.split(':')[1]}" + prompt_input += [role.s_mem.get_str_accessible_sector_arenas(x)] + + prompt_input += [role.scratch.get_str_name()] + prompt_input += [f"{access_tile['sector']}"] + x = f"{act_world}:{access_tile['sector']}" + prompt_input += [role.s_mem.get_str_accessible_sector_arenas(x)] + + if role.scratch.get_str_daily_plan_req() != "": + prompt_input += [f"\n{role.scratch.get_str_daily_plan_req()}"] + else: + prompt_input += [""] + + # MAR 11 TEMP + prompt_input = [] + act_world = access_tile["world"] + accessible_sector_str = role.s_mem.get_str_accessible_sectors(act_world) + curr = accessible_sector_str.split(", ") + fin_accessible_sectors = [] + for i in curr: + if "'s house" in i: + if role.scratch.last_name in i: + fin_accessible_sectors += [i] + else: + fin_accessible_sectors += [i] + accessible_sector_str = ", ".join(fin_accessible_sectors) + # END MAR 11 TEMP + + prompt_input += [accessible_sector_str] + + act_desp_1 = act_desp + act_desp_2 = act_desp + if "(" in act_desp: + act_desp_1 = act_desp.split("(")[0].strip() + act_desp_2 = act_desp.split("(")[-1][:-1] + prompt_input += [role.scratch.get_str_name()] + prompt_input += [act_desp_1] + + prompt_input += [act_desp_2] + prompt_input += [role.scratch.get_str_name()] + return prompt_input + + prompt_template = "action_location_sector_v1.txt" + prompt_input = create_prompt_input(role, access_tile, act_desp) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=15) + y = f"{access_tile['world']}" + x = [i.strip() for i in role.s_mem.get_str_accessible_sectors(y).split(",")] + if output not in x: + # output = random.choice(x) + output = role.scratch.living_area.split(":")[1] + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenActionArena(STAction): + name: str = "GenActionArena" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cleaned_response = llm_resp.split("}")[0] + return cleaned_response + + def _func_validate(self, llm_resp: str, prompt: str): + if len(llm_resp.strip()) < 1: + return False + if "}" not in llm_resp: + return False + if "," in llm_resp: + return False + return True + + def _func_fail_default_resp(self): + fs = "kitchen" + return fs + + async def run(self, role: "STRole", act_desp: str, act_world: str, act_sector: str): + def create_prompt_input(role, act_desp, act_world, act_sector): + prompt_input = [] + prompt_input += [role.scratch.get_str_name()] + x = f"{act_world}:{act_sector}" + prompt_input += [act_sector] + + # MAR 11 TEMP + accessible_arena_str = role.s_mem.get_str_accessible_sector_arenas(x) + curr = accessible_arena_str.split(", ") + fin_accessible_arenas = [] + for i in curr: + if "'s room" in i: + if role.scratch.last_name in i: + fin_accessible_arenas += [i] + else: + fin_accessible_arenas += [i] + accessible_arena_str = ", ".join(fin_accessible_arenas) + # END MAR 11 TEMP + prompt_input += [accessible_arena_str] + act_desp_1 = act_desp + act_desp_2 = act_desp + if "(" in act_desp: + act_desp_1 = act_desp.split("(")[0].strip() + act_desp_2 = act_desp.split("(")[-1][:-1] + prompt_input += [role.scratch.get_str_name()] + prompt_input += [act_desp_1] + + prompt_input += [act_desp_2] + prompt_input += [role.scratch.get_str_name()] + + prompt_input += [act_sector] + prompt_input += [accessible_arena_str] + return prompt_input + + prompt_template = "action_location_object_vMar11.txt" + prompt_input = create_prompt_input(role, act_desp, act_world, act_sector) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp() + print('prompt ', prompt) + output = await self._run_gpt35_max_tokens(prompt, max_tokens=15) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenActionObject(STAction): + name: str = "GenActionObject" + + def _func_validate(self, llm_resp: str, prompt: str): + if len(llm_resp.strip()) < 1: + return False + return True + + def _func_cleanup(self, llm_resp: str, prompt: str): + cleaned_response = llm_resp.strip() + return cleaned_response + + def _func_fail_default_resp(self): + fs = "bed" + return fs + + async def run(self, role: "STRole", act_desp: str, temp_address: str): + def create_prompt_input(role, act_desp, temp_address): + prompt_input = [] + if "(" in act_desp: + act_desp = act_desp.split("(")[-1][:-1] + + prompt_input += [act_desp] + prompt_input += [role.s_mem.get_str_accessible_arena_game_objects(temp_address)] + return prompt_input + + prompt_template = "action_object_v2.txt" + prompt_input = create_prompt_input(role, act_desp, temp_address) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=15) + x = [i.strip() for i in role.s_mem.get_str_accessible_arena_game_objects(temp_address).split(",")] + if output not in x: + output = random.choice(x) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenPronunciatio(STAction): + name: str = "GenPronunciatio" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cr = llm_resp.strip() + if len(cr) > 3: + cr = cr[:3] + return cr + + def _func_validate(self, llm_resp: str, prompt: str): + try: + self._func_cleanup(llm_resp, prompt="") + if len(llm_resp) == 0: + return False + except Exception: + return False + return True + + def _func_fail_default_resp(self): + fs = "😋" + return fs + + async def run(self, role: "STRole", act_desp: str): + def create_prompt_input(act_desp): + if "(" in act_desp: + act_desp = act_desp.split("(")[-1].split(")")[0] + prompt_input = [act_desp] + return prompt_input + + prompt_template = "generate_pronunciatio_v1.txt" + prompt_input = create_prompt_input(act_desp) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + example_output = "🛁🧖‍♀️" + special_instruction = "The value for the output must ONLY contain the emojis." + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenEventTriple(STAction): + name: str = "GenEventTriple" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cr = llm_resp.strip() + cr = [i.strip() for i in cr.split(")")[0].split(",")] + return cr + + def _func_validate(self, llm_resp: str, prompt: str): + try: + llm_resp = self._func_cleanup(llm_resp, prompt="") + if len(llm_resp) != 2: + return False + except Exception: + return False + return True + + def _func_fail_default_resp(self, role): + fs = (role.name, "is", "idle") + return fs + + async def run(self, role: "STRole", act_desp: str): + def create_prompt_input(role, act_desp): + if "(" in act_desp: + act_desp = act_desp.split("(")[-1].split(")")[0] + prompt_input = [role.name, act_desp, role.name] + return prompt_input + + prompt_template = "generate_event_triple_v1.txt" + prompt_input = create_prompt_input(role, act_desp) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp(role) + output = await self._run_gpt35_max_tokens(prompt, max_tokens=30) + output = (role.name, output[0], output[1]) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenActObjDescription(STAction): + name: str = "GenActObjDescription" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cr = llm_resp.strip() + if cr[-1] == ".": + cr = cr[:-1] + return cr + + def _func_validate(self, llm_resp: str, prompt: str): + try: + llm_resp = self._func_cleanup(llm_resp, prompt="") + except Exception: + return False + return True + + def _func_fail_default_resp(self, act_game_object): + fs = f"{act_game_object} is idle" + return fs + + async def run(self, role: "STRole", act_game_object: str, act_desp: str): + def create_prompt_input(act_game_object, act_desp, role): + prompt_input = [act_game_object, role.name, act_desp, act_game_object, act_game_object] + return prompt_input + + prompt_template = "generate_obj_event_v1.txt" + prompt_input = create_prompt_input(act_game_object, act_desp, role) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + example_output = "being fixed" + special_instruction = "The output should ONLY contain the phrase that should go in ." + self.fail_default_resp = self._func_fail_default_resp(act_game_object) + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenObjEventTriple(STAction): + name: str = "GenObjEventTriple" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cr = llm_resp.strip() + cr = [i.strip() for i in cr.split(")")[0].split(",")] + return cr + + def _func_validate(self, llm_resp: str, prompt: str): + try: + llm_resp = self._func_cleanup(llm_resp, prompt="") + if len(llm_resp) != 2: + return False + except Exception: + return False + return True + + def _func_fail_default_resp(self, act_game_object: str): + fs = (act_game_object, "is", "idle") + return fs + + async def run(self, role: "STRole", act_game_object, act_obj_desp): + def create_prompt_input(act_game_object, act_obj_desp): + prompt_input = [act_game_object, act_obj_desp, act_game_object] + return prompt_input + + prompt_template = "generate_event_triple_v1.txt" + prompt_input = create_prompt_input(act_game_object, act_obj_desp) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp(act_game_object) + output = await self._run_gpt35_max_tokens(prompt, max_tokens=30) + output = (act_game_object, output[0], output[1]) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenActionDetails(STAction): + name: str = "GenActionDetails" + + def _func_cleanup(self, llm_resp: str, prompt: str) -> list: + pass + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + # TODO -- this sometimes generates error + try: + self._func_cleanup(llm_resp) + except Exception: + return False + return True + + def _func_fail_default_resp(self): + fs = {} + return fs + + async def run(self, role: "STRole", act_desp: str, act_dura): + access_tile = await role.rc.env.observe( + EnvAPIAbstract(api_name="access_tile", kwargs={"tile": role.scratch.curr_tile}) + ) + act_world = access_tile["world"] + act_sector = await GenActionSector().run(role, access_tile, act_desp) + act_arena = await GenActionArena().run(role, act_desp, act_world, act_sector) + act_address = f"{act_world}:{act_sector}:{act_arena}" + if not role.s_mem.get_str_accessible_arena_game_objects(act_address): + act_game_object = "" + else: + act_game_object = await GenActionObject().run(role, act_desp, act_address) + new_address = f"{act_world}:{act_sector}:{act_arena}:{act_game_object}" + act_pron = await GenPronunciatio().run(role, act_desp) + act_event = await GenEventTriple().run(role, act_desp) + # Persona's actions also influence the object states. We set those up here. + act_obj_desp = await GenActObjDescription().run(role, act_game_object, act_desp) + act_obj_pron = await GenPronunciatio().run(role, act_obj_desp) + act_obj_event = await GenObjEventTriple().run(role, act_game_object, act_obj_desp) + result_dict = { + "action_address": new_address, + "action_duration": int(act_dura), + "action_description": act_desp, + "action_pronunciatio": act_pron, + "action_event": act_event, + "chatting_with": None, + "chat": None, + "chatting_with_buffer": None, + "chatting_end_time": None, + "act_obj_description": act_obj_desp, + "act_obj_pronunciatio": act_obj_pron, + "act_obj_event": act_obj_event, + } + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {result_dict}") + return result_dict diff --git a/examples/st_game/actions/gen_daily_schedule.py b/examples/st_game/actions/gen_daily_schedule.py new file mode 100644 index 000000000..6deb6e6eb --- /dev/null +++ b/examples/st_game/actions/gen_daily_schedule.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : gen_daily_schedule + + +from metagpt.logs import logger + +from .st_action import STAction + + +class GenDailySchedule(STAction): + name: str = "GenDailySchedule" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt="") + except Exception: + return False + return True + + def _func_cleanup(self, llm_resp: str, prompt: str) -> list: + cr = [] + _cr = llm_resp.split(")") + for i in _cr: + if i[-1].isdigit(): + i = i[:-1].strip() + if i[-1] == "." or i[-1] == ",": + cr += [i[:-1].strip()] + return cr + + def _func_fail_default_resp(self) -> int: + fs = [ + "wake up and complete the morning routine at 6:00 am", + "eat breakfast at 7:00 am", + "read a book from 8:00 am to 12:00 pm", + "have lunch at 12:00 pm", + "take a nap from 1:00 pm to 4:00 pm", + "relax and watch TV from 7:00 pm to 8:00 pm", + "go to bed at 11:00 pm", + ] + return fs + + async def run(self, role: "STRole", wake_up_hour: str): + def create_prompt_input(role, wake_up_hour): + prompt_input = [] + prompt_input += [role.scratch.get_str_iss()] + prompt_input += [role.scratch.get_str_lifestyle()] + prompt_input += [role.scratch.get_str_curr_date_str()] + prompt_input += [role.scratch.get_str_firstname()] + prompt_input += [f"{str(wake_up_hour)}:00 am"] + return prompt_input + + wake_up_hour = int(wake_up_hour) + prompt_template = "daily_planning_v6.txt" + prompt_input = create_prompt_input(role, wake_up_hour) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=500) + output = [f"wake up and complete the morning routine at {wake_up_hour}:00 am"] + output + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/gen_hourly_schedule.py b/examples/st_game/actions/gen_hourly_schedule.py new file mode 100644 index 000000000..5d59f96dd --- /dev/null +++ b/examples/st_game/actions/gen_hourly_schedule.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : gen_hourly_schedule + +import random +import string + +from metagpt.logs import logger + +from .st_action import STAction + + +def get_random_alphanumeric(i=6, j=6): + """ + Returns a random alpha numeric strength that has the length of somewhere + between i and j. + + INPUT: + i: min_range for the length + j: max_range for the length + OUTPUT: + an alpha numeric str with the length of somewhere between i and j. + """ + k = random.randint(i, j) + x = "".join(random.choices(string.ascii_letters + string.digits, k=k)) + return x + + +class GenHourlySchedule(STAction): + name: str = "GenHourlySchedule" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt="") + except Exception: + return False + return True + + def _func_cleanup(self, llm_resp: str, prompt: str) -> list: + cr = llm_resp.strip() + if cr[-1] == ".": + cr = cr[:-1] + # to only use the first line of output + cr = cr.split("\n")[0] + return cr + + def _func_fail_default_resp(self) -> int: + fs = "asleep" + return fs + + async def _generate_schedule_for_given_hour( + self, role: "STRole", curr_hour_str, p_f_ds_hourly_org, hour_str, intermission2=None + ): + def create_prompt_input(persona, curr_hour_str, p_f_ds_hourly_org, hour_str, intermission2=None): + schedule_format = "" + for i in hour_str: + schedule_format += f"[{persona.scratch.get_str_curr_date_str()} -- {i}]" + schedule_format += " Activity: [Fill in]\n" + schedule_format = schedule_format[:-1] + + intermission_str = "Here the originally intended hourly breakdown of" + intermission_str += f" {persona.scratch.get_str_firstname()}'s schedule today: " + for count, i in enumerate(persona.scratch.daily_req): + intermission_str += f"{str(count + 1)}) {i}, " + intermission_str = intermission_str[:-2] + + prior_schedule = "" + if p_f_ds_hourly_org: + prior_schedule = "\n" + for count, i in enumerate(p_f_ds_hourly_org): + prior_schedule += f"[(ID:{get_random_alphanumeric()})" + prior_schedule += f" {persona.scratch.get_str_curr_date_str()} --" + prior_schedule += f" {hour_str[count]}] Activity:" + prior_schedule += f" {persona.scratch.get_str_firstname()}" + prior_schedule += f" is {i}\n" + + prompt_ending = f"[(ID:{get_random_alphanumeric()})" + prompt_ending += f" {persona.scratch.get_str_curr_date_str()}" + prompt_ending += f" -- {curr_hour_str}] Activity:" + prompt_ending += f" {persona.scratch.get_str_firstname()} is" + + if intermission2: + intermission2 = f"\n{intermission2}" + + prompt_input = [] + prompt_input += [schedule_format] + prompt_input += [persona.scratch.get_str_iss()] + + prompt_input += [prior_schedule + "\n"] + prompt_input += [intermission_str] + if intermission2: + prompt_input += [intermission2] + else: + prompt_input += [""] + prompt_input += [prompt_ending] + + return prompt_input + + prompt_template = "generate_hourly_schedule_v2.txt" + prompt_input = create_prompt_input(role, curr_hour_str, p_f_ds_hourly_org, hour_str, intermission2) + prompt_input_str = "\n".join(prompt_input) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=50) + logger.info( + f"Role: {role.name} _generate_schedule_for_given_hour prompt_input: {prompt_input_str}, " + f"output: {output}" + ) + return output + + async def run(self, role: "STRole", wake_up_hour: int): + hour_str = [ + "00:00 AM", + "01:00 AM", + "02:00 AM", + "03:00 AM", + "04:00 AM", + "05:00 AM", + "06:00 AM", + "07:00 AM", + "08:00 AM", + "09:00 AM", + "10:00 AM", + "11:00 AM", + "12:00 PM", + "01:00 PM", + "02:00 PM", + "03:00 PM", + "04:00 PM", + "05:00 PM", + "06:00 PM", + "07:00 PM", + "08:00 PM", + "09:00 PM", + "10:00 PM", + "11:00 PM", + ] + n_m1_activity = [] + diversity_repeat_count = 1 # TODO mg 1->3 + for i in range(diversity_repeat_count): + logger.info(f"diversity_repeat_count idx: {i}") + n_m1_activity_set = set(n_m1_activity) + if len(n_m1_activity_set) < 5: + n_m1_activity = [] + for count, curr_hour_str in enumerate(hour_str): + if wake_up_hour > 0: + n_m1_activity += ["sleeping"] + wake_up_hour -= 1 + else: + logger.info(f"_generate_schedule_for_given_hour idx: {count}, n_m1_activity: {n_m1_activity}") + n_m1_activity += [ + await self._generate_schedule_for_given_hour(role, curr_hour_str, n_m1_activity, hour_str) + ] + + # Step 1. Compressing the hourly schedule to the following format: + # The integer indicates the number of hours. They should add up to 24. + # [['sleeping', 6], ['waking up and starting her morning routine', 1], + # ['eating breakfast', 1], ['getting ready for the day', 1], + # ['working on her painting', 2], ['taking a break', 1], + # ['having lunch', 1], ['working on her painting', 3], + # ['taking a break', 2], ['working on her painting', 2], + # ['relaxing and watching TV', 1], ['going to bed', 1], ['sleeping', 2]] + _n_m1_hourly_compressed = [] + prev = None + prev_count = 0 + for i in n_m1_activity: + if i != prev: + prev_count = 1 + _n_m1_hourly_compressed += [[i, prev_count]] + prev = i + elif _n_m1_hourly_compressed: + _n_m1_hourly_compressed[-1][1] += 1 + + # Step 2. Expand to min scale (from hour scale) + # [['sleeping', 360], ['waking up and starting her morning routine', 60], + # ['eating breakfast', 60],.. + n_m1_hourly_compressed = [] + for task, duration in _n_m1_hourly_compressed: + n_m1_hourly_compressed += [[task, duration * 60]] + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {n_m1_hourly_compressed}") + return n_m1_hourly_compressed diff --git a/examples/st_game/actions/gen_iter_chat_utt.py b/examples/st_game/actions/gen_iter_chat_utt.py new file mode 100644 index 000000000..2b0d46f4e --- /dev/null +++ b/examples/st_game/actions/gen_iter_chat_utt.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : generate_iterative_chat_utt + +from examples.st_game.actions.st_action import STAction +from examples.st_game.utils.utils import extract_first_json_dict +from metagpt.environment.api.env_api import EnvAPIAbstract +from metagpt.logs import logger + + +class GenIterChatUTT(STAction): + name: str = "GenIterChatUTT" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + resp = False + try: + _ = extract_first_json_dict(llm_resp) + resp = True + except Exception: + pass + return resp + + def _func_cleanup(self, llm_resp: str, prompt: str) -> dict: + gpt_response = extract_first_json_dict(llm_resp) + + cleaned_dict = dict() + cleaned = [] + for key, val in gpt_response.items(): + cleaned += [val] + cleaned_dict["utterance"] = cleaned[0] + cleaned_dict["end"] = True + if "f" in str(cleaned[1]) or "F" in str(cleaned[1]): + cleaned_dict["end"] = False + + return cleaned_dict + + def _func_fail_default_resp(self) -> dict: + cleaned_dict = dict() + cleaned_dict["utterance"] = "..." + cleaned_dict["end"] = False + return cleaned_dict + + async def run( + self, + init_role: "STRole", + target_role: "STRole", + retrieved: dict, + curr_context: str, + curr_chat: list[str], + *args, + **kwargs, + ) -> dict: + def create_prompt_input( + access_tile: dict[str, str], + init_role: "STRole", + target_role: "STRole", + retrieved: dict, + curr_context: str, + curr_chat: list[str], + ): + role = init_role + scratch = role.rc.scratch + target_scratch = target_role.rc.scratch + prev_convo_insert = "\n" + if role.rc.memory.chat_list: + for i in role.rc.memory.chat_list: + if i.object == target_role.name: + v1 = int((scratch.curr_time - i.created).total_seconds() / 60) + prev_convo_insert += ( + f"{str(v1)} minutes ago, {scratch.name} and " + f"{target_scratch.name} were already {i.description} " + f"This context takes place after that conversation." + ) + break + if prev_convo_insert == "\n": + prev_convo_insert = "" + if role.rc.memory.chat_list: + if int((scratch.curr_time - role.rc.memory.chat_list[-1].created).total_seconds() / 60) > 480: + prev_convo_insert = "" + print(prev_convo_insert) + + curr_sector = f"{access_tile['sector']}" + curr_arena = f"{access_tile['arena']}" + curr_location = f"{curr_arena} in {curr_sector}" + + retrieved_str = "" + for key, vals in retrieved.items(): + for v in vals: + retrieved_str += f"- {v.description}\n" + + convo_str = "" + for i in curr_chat: + convo_str += ": ".join(i) + "\n" + if convo_str == "": + convo_str = "[The conversation has not started yet -- start it!]" + + init_iss = f"Here is Here is a brief description of {scratch.name}.\n{scratch.get_str_iss()}" + prompt_input = [ + init_iss, + scratch.name, + retrieved_str, + prev_convo_insert, + curr_location, + curr_context, + scratch.name, + target_scratch.name, + convo_str, + scratch.name, + target_scratch.name, + scratch.name, + scratch.name, + scratch.name, + ] + return prompt_input + + access_tile = await init_role.rc.env.observe( + EnvAPIAbstract(api_name="access_tile", kwargs={"tile": init_role.scratch.curr_tile}) + ) + prompt_input = create_prompt_input(access_tile, init_role, target_role, retrieved, curr_context, curr_chat) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "iterative_convo_v1.txt") + # original using `ChatGPT_safe_generate_response_OLD` + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_wo_extra_prompt(prompt) + logger.info(f"Role: {init_role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/inner_voice_action.py b/examples/st_game/actions/inner_voice_action.py new file mode 100644 index 000000000..cfbdf0303 --- /dev/null +++ b/examples/st_game/actions/inner_voice_action.py @@ -0,0 +1,31 @@ +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +class AgentWhisperThoughtAction(STAction): + name: str = "AgentWhisperThoughtAction" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> list: + return llm_resp.split('"')[0].strip() + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, test_input=None, verbose=False) -> str: + def create_prompt_input(role: "STRole", statements, test_input=None): + prompt_input = [role.scratch.name, statements] + return prompt_input + + prompt_input = create_prompt_input(role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "whisper_inner_thought_v1.txt") + + output = await self._run_gpt35_max_tokens(prompt, max_tokens=50) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/new_decomp_schedule.py b/examples/st_game/actions/new_decomp_schedule.py new file mode 100644 index 000000000..8e5c79401 --- /dev/null +++ b/examples/st_game/actions/new_decomp_schedule.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : new_decomp_schedule + +import datetime + +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +class NewDecompSchedule(STAction): + name: str = "NewDecompSchedule" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + resp = False + try: + llm_resp = self._func_cleanup(llm_resp, prompt) + dur_sum = 0 + for act, dur in llm_resp: + dur_sum += dur + if isinstance(act, str): + return False + if isinstance(dur, int): + return False + x = prompt.split("\n")[0].split("originally planned schedule from")[-1].strip()[:-1] + x = [datetime.datetime.strptime(i.strip(), "%H:%M %p") for i in x.split(" to ")] + delta_min = int((x[1] - x[0]).total_seconds() / 60) + + if int(dur_sum) != int(delta_min): + return False + except Exception: + pass + return resp + + def _func_cleanup(self, llm_resp: str, prompt: str) -> list: + new_schedule = prompt + " " + llm_resp.strip() + new_schedule = new_schedule.split("The revised schedule:")[-1].strip() + new_schedule = new_schedule.split("\n") + + ret_temp = [] + for i in new_schedule: + ret_temp += [i.split(" -- ")] + + ret = [] + for time_str, action in ret_temp: + start_time = time_str.split(" ~ ")[0].strip() + end_time = time_str.split(" ~ ")[1].strip() + delta = datetime.datetime.strptime(end_time, "%H:%M") - datetime.datetime.strptime(start_time, "%H:%M") + delta_min = int(delta.total_seconds() / 60) + if delta_min < 0: + delta_min = 0 + ret += [[action, delta_min]] + + return ret + + def _func_fail_default_resp(self, main_act_dur: int, truncated_act_dur: int) -> int: + dur_sum = 0 + for act, dur in main_act_dur: + dur_sum += dur + + ret = truncated_act_dur[:] + ret += main_act_dur[len(ret) - 1 :] + + # If there are access, we need to trim... + ret_dur_sum = 0 + count = 0 + over = None + for act, dur in ret: + ret_dur_sum += dur + if ret_dur_sum == dur_sum: + break + if ret_dur_sum > dur_sum: + over = ret_dur_sum - dur_sum + break + count += 1 + + if over: + ret = ret[: count + 1] + ret[-1][1] -= over + + return ret + + async def run( + self, + role: "STRole", + main_act_dur: int, + truncated_act_dur: int, + start_time_hour: datetime, + end_time_hour: datetime, + inserted_act: str, + inserted_act_dur: int, + *args, + **kwargs, + ): + def create_prompt_input( + role: "STRole", + main_act_dur: int, + truncated_act_dur: int, + start_time_hour: datetime, + end_time_hour: datetime, + inserted_act: str, + inserted_act_dur: int, + ): + persona_name = role.name + start_hour_str = start_time_hour.strftime("%H:%M %p") + end_hour_str = end_time_hour.strftime("%H:%M %p") + + original_plan = "" + for_time = start_time_hour + for i in main_act_dur: + original_plan += ( + f'{for_time.strftime("%H:%M")} ~ ' + f'{(for_time + datetime.timedelta(minutes=int(i[1]))).strftime("%H:%M")} -- ' + i[0] + ) + original_plan += "\n" + for_time += datetime.timedelta(minutes=int(i[1])) + + new_plan_init = "" + for_time = start_time_hour + for count, i in enumerate(truncated_act_dur): + new_plan_init += ( + f'{for_time.strftime("%H:%M")} ~ ' + f'{(for_time + datetime.timedelta(minutes=int(i[1]))).strftime("%H:%M")} -- ' + i[0] + ) + new_plan_init += "\n" + if count < len(truncated_act_dur) - 1: + for_time += datetime.timedelta(minutes=int(i[1])) + + new_plan_init += (for_time + datetime.timedelta(minutes=int(i[1]))).strftime("%H:%M") + " ~" + + prompt_input = [ + persona_name, + start_hour_str, + end_hour_str, + original_plan, + persona_name, + inserted_act, + inserted_act_dur, + persona_name, + start_hour_str, + end_hour_str, + end_hour_str, + new_plan_init, + ] + return prompt_input + + prompt_input = create_prompt_input( + role, main_act_dur, truncated_act_dur, start_time_hour, end_time_hour, inserted_act, inserted_act_dur + ) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "new_decomp_schedule_v1.txt") + self.fail_default_resp = self._func_fail_default_resp(main_act_dur, truncated_act_dur) + output = await self._run_gpt35_max_tokens(prompt, max_tokens=1000) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/run_reflect_action.py b/examples/st_game/actions/run_reflect_action.py new file mode 100644 index 000000000..d3bf008cb --- /dev/null +++ b/examples/st_game/actions/run_reflect_action.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : Integration Reflect Action + +import re + +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +# Run GPT Prompt Focal Point method +class AgentFocusPt(STAction): + name: str = "AgentFocusPt" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> str: + try: + """ + Cleanup handling has been completed for run_v2 + """ + return llm_resp + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, n: int, test_input=None) -> str: + def create_prompt_input(role: "STRole", statements, n, test_input=None): + prompt_input = [statements, str(n)] + return prompt_input + + prompt_input = create_prompt_input(role, statements, n) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "generate_focal_pt_v1.txt") + + example_output = '["What should Jane do for lunch", "Does Jane like strawberry", "Who is Jane"]' + special_instruction = "Output must be a list of str." + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Insight and Guidance +class AgentInsightAndGuidance(STAction): + name: str = "AgentInsightAndGuidance" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> dict: + try: + llm_resp = "1. " + llm_resp.strip() + ret = dict() + for i in llm_resp.split("\n"): + row = " ".join(i.split(". ")[1:]) + if "(because of " not in row: + continue + thought = row.split("(because of ")[0].strip() + if ")" not in row.split("(because of ")[1]: + continue + evi_raw = row.split("(because of ")[1].split(")")[0].strip() + evi_raw = re.findall(r"\d+", evi_raw) + evi_raw = [int(i.strip()) for i in evi_raw] + ret[thought] = evi_raw + return ret + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self, n: int) -> str: + return ["I am hungry"] * n + + async def run(self, role: "STRole", statements: str, n: int, test_input=None) -> dict: + def create_prompt_input(role, statements, n, test_input=None): + prompt_input = [statements, str(n)] + return prompt_input + + prompt_input = create_prompt_input(role, statements, n) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "insight_and_evidence_v1.txt") + + self.fail_default_resp = self._func_fail_default_resp(n) + output = await self._run_gpt35_max_tokens(prompt, max_tokens=150) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Event Triple +class AgentEventTriple(STAction): + name: str = "AgentEventTriple" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + llm_resp = self._func_cleanup(llm_resp, prompt="") + if len(llm_resp) != 2: + return False + except Exception: + return False + return True + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> list: + try: + cr = llm_resp.strip() + cr = [i.strip() for i in cr.split(")")[0].split(",")] + if len(cr) != 2: + return cr[-2:] + return cr + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, statements: str, role: "STRole", verbose=False) -> tuple: + def create_prompt_input(statements, role): + if "(" in statements: + statements = statements.split("(")[-1].split(")")[0] + prompt_input = [role.scratch.name, statements, role.scratch.name] + return prompt_input + + prompt_input = create_prompt_input(statements, role) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "generate_event_triple_v1.txt") + + output = await self._run_gpt35_max_tokens(prompt, max_tokens=30) + output = (role.scratch.name, output[0], output[1]) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Event Poignancy +class AgentEventPoignancy(STAction): + name: str = "AgentEventPoignancy" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> int: + try: + llm_resp = int(llm_resp.strip()) + return llm_resp + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, test_input=None, verbose=False) -> str: + def create_prompt_input(role: "STRole", statements: str, test_input=None): + prompt_input = [role.scratch.name, role.scratch.get_str_iss(), role.scratch.name, statements] + return prompt_input + + prompt_input = create_prompt_input(role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "poignancy_event_v1.txt") + + example_output = "5" # ######## + special_instruction = "The output should ONLY contain ONE integer value on the scale of 1 to 10." + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Chat Poignancy +class AgentChatPoignancy(STAction): + name: str = "AgentChatPoignancy" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> int: + try: + llm_resp = int(llm_resp.strip()) + return llm_resp + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, test_input=None, verbose=False) -> str: + def create_prompt_input(role: "STRole", statements, test_input=None): + prompt_input = [role.scratch.name, role.scratch.get_str_iss(), role.scratch.name, statements] + return prompt_input + + prompt_input = create_prompt_input(role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "poignancy_chat_v1.txt") + + example_output = "5" # ######## + special_instruction = "The output should ONLY contain ONE integer value on the scale of 1 to 10." + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Planning Thought on Convo +class AgentPlanThoughtOnConvo(STAction): + name: str = "AgentPlanThoughtOnConvo" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> str: + try: + return llm_resp.split('"')[0].strip() + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, test_input=None, verbose=False) -> str: + def create_prompt_input(role, statements, test_input=None): + prompt_input = [statements, role.scratch.name, role.scratch.name, role.scratch.name] + return prompt_input + + prompt_input = create_prompt_input(role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "planning_thought_on_convo_v1.txt") + + output = await self._run_gpt35_max_tokens(prompt, max_tokens=50) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Memory on Convo +class AgentMemoryOnConvo(STAction): + name: str = "AgentMemoryOnConvo" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> str: + try: + return llm_resp.split('"')[0].strip() + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, test_input=None, verbose=False) -> str: + def create_prompt_input(role, statements, test_input=None): + prompt_input = [statements, role.scratch.name, role.scratch.name, role.scratch.name] + return prompt_input + + prompt_input = create_prompt_input(role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "memo_on_convo_v1.txt") + example_output = "Jane Doe was interesting to talk to." + special_instruction = ( + "The output should ONLY contain a string that summarizes anything interesting " + "that the agent may have noticed" + ) + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/st_action.py b/examples/st_game/actions/st_action.py new file mode 100644 index 000000000..0796b5c8d --- /dev/null +++ b/examples/st_game/actions/st_action.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : StanfordTown Action +import json +import time +from abc import abstractmethod +from pathlib import Path +from typing import Any, Optional, Union + +from examples.st_game.utils.const import PROMPTS_DIR +from metagpt.actions.action import Action +from metagpt.config2 import config +from metagpt.logs import logger + + +class STAction(Action): + name: str = "STAction" + prompt_dir: Path = PROMPTS_DIR + fail_default_resp: Optional[str] = None + + @property + def cls_name(self): + return self.__class__.__name__ + + @abstractmethod + def _func_validate(self, llm_resp: str, prompt: str): + raise NotImplementedError + + @abstractmethod + def _func_cleanup(self, llm_resp: str, prompt: str): + raise NotImplementedError + + @abstractmethod + def _func_fail_default_resp(self): + raise NotImplementedError + + def generate_prompt_with_tmpl_filename(self, prompt_input: Union[str, list], tmpl_filename) -> str: + """ + same with `generate_prompt` + Args: + prompt_input: the input we want to feed in (IF THERE ARE MORE THAN ONE INPUT, THIS CAN BE A LIST.) + tmpl_filename: prompt template filename + Returns: + a str prompt that will be sent to LLM server. + """ + if isinstance(prompt_input, str): + prompt_input = [prompt_input] + prompt_input = [str(i) for i in prompt_input] + + f = open(str(self.prompt_dir.joinpath(tmpl_filename)), "r") + prompt = f.read() + f.close() + for count, i in enumerate(prompt_input): + prompt = prompt.replace(f"!!", i) + if "###" in prompt: + prompt = prompt.split("###")[1] + return prompt.strip() + + async def _aask(self, prompt: str) -> str: + return await self.llm.aask(prompt) + + async def _run_gpt35_max_tokens(self, prompt: str, max_tokens: int = 50, retry: int = 3): + for idx in range(retry): + try: + tmp_max_tokens_rsp = getattr(config.llm, "max_token", 1500) + setattr(config.llm, "max_token", max_tokens) + self.llm.use_system_prompt = False # to make it behave like a non-chat completions + + llm_resp = await self._aask(prompt) + + setattr(config.llm, "max_token", tmp_max_tokens_rsp) + logger.info(f"Action: {self.cls_name} llm _run_gpt35_max_tokens raw resp: {llm_resp}") + if self._func_validate(llm_resp, prompt): + return self._func_cleanup(llm_resp, prompt) + except Exception as exp: + logger.warning(f"Action: {self.cls_name} _run_gpt35_max_tokens exp: {exp}") + time.sleep(5) + return self.fail_default_resp + + async def _run_gpt35( + self, prompt: str, example_output: str, special_instruction: str, retry: int = 3 + ) -> Union[bool, Any]: + """same with `gpt_structure.ChatGPT_safe_generate_response`""" + prompt = '"""\n' + prompt + '\n"""\n' + prompt += f"Output the response to the prompt above in json. {special_instruction}\n" + prompt += "Example output json:\n" + prompt += '{"output": "' + str(example_output) + '"}' + + for idx in range(retry): + try: + llm_resp = await self._aask(prompt) + logger.info(f"Action: {self.cls_name} llm _run_gpt35 raw resp: {llm_resp}") + end_idx = llm_resp.strip().rfind("}") + 1 + llm_resp = llm_resp[:end_idx] + llm_resp = json.loads(llm_resp)["output"] + + if self._func_validate(llm_resp, prompt): + return self._func_cleanup(llm_resp, prompt) + except Exception as exp: + logger.warning(f"Action: {self.cls_name} _run_gpt35 exp: {exp}") + time.sleep(5) # usually avoid `Rate limit` + return False + + async def _run_gpt35_wo_extra_prompt(self, prompt: str, retry: int = 3) -> str: + for idx in range(retry): + try: + llm_resp = await self._aask(prompt) + llm_resp = llm_resp.strip() + logger.info(f"Action: {self.cls_name} llm _run_gpt35_wo_extra_prompt raw resp: {llm_resp}") + if self._func_validate(llm_resp, prompt): + return self._func_cleanup(llm_resp, prompt) + except Exception as exp: + logger.warning(f"Action: {self.cls_name} _run_gpt35_wo_extra_prompt exp: {exp}") + time.sleep(5) # usually avoid `Rate limit` + return self.fail_default_resp + + async def run(self, *args, **kwargs): + """Run action""" + raise NotImplementedError("The run method should be implemented in a subclass.") diff --git a/examples/st_game/actions/summarize_conv.py b/examples/st_game/actions/summarize_conv.py new file mode 100644 index 000000000..f35651b35 --- /dev/null +++ b/examples/st_game/actions/summarize_conv.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : summarize the content of agents' conversation + +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +class SummarizeConv(STAction): + name: str = "SummarizeConv" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + resp = False + try: + _ = self._func_cleanup(llm_resp, prompt) + resp = True + except Exception: + pass + return resp + + def _func_cleanup(self, llm_resp: str, prompt: str) -> str: + ret = "conversing about " + llm_resp.strip() + return ret + + def _func_fail_default_resp(self) -> str: + return "conversing with a housemate about morning greetings" + + async def run(self, conv: list): + def create_prompt_input(conversation: list): + convo_str = "" + for row in conversation: + convo_str += f'{row[0]}: "{row[1]}"\n' + prompt_input = [convo_str] + return prompt_input + + prompt_input = create_prompt_input(conv) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "summarize_conversation_v1.txt") + + example_output = "conversing about what to eat for lunch" + special_instruction = ( + "The output must continue the sentence above by filling in the tag. " + "Don't start with 'this is a conversation about...' Just finish the sentence " + "but do not miss any important details (including who are chatting)." + ) + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/task_decomp.py b/examples/st_game/actions/task_decomp.py new file mode 100644 index 000000000..d9d5ec9fa --- /dev/null +++ b/examples/st_game/actions/task_decomp.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : task_decomp + +import datetime + +from metagpt.logs import logger + +from ..actions.st_action import STAction + + +class TaskDecomp(STAction): + name: str = "TaskDecomp" + + def _func_cleanup(self, llm_resp: str, prompt: str) -> list: + # TODO SOMETHING HERE sometimes fails... See screenshot + temp = [i.strip() for i in llm_resp.split("\n")] + _cr = [] + cr = [] + for count, i in enumerate(temp): + if count != 0: + _cr += [" ".join([j.strip() for j in i.split(" ")][3:])] + else: + _cr += [i] + for count, i in enumerate(_cr): + k = [j.strip() for j in i.split("(duration in minutes:")] + task = k[0] + if task[-1] == ".": + task = task[:-1] + duration = int(k[1].split(",")[0].strip()) + cr += [[task, duration]] + + total_expected_min = int(prompt.split("(total duration in minutes")[-1].split("):")[0].strip()) + + # TODO -- now, you need to make sure that this is the same as the sum of + # the current action sequence. + curr_min_slot = [ + ["dummy", -1], + ] # (task_name, task_index) + for count, i in enumerate(cr): + i_task = i[0] + i_duration = i[1] + + i_duration -= i_duration % 5 + if i_duration > 0: + for j in range(i_duration): + curr_min_slot += [(i_task, count)] + curr_min_slot = curr_min_slot[1:] + + if len(curr_min_slot) > total_expected_min: + last_task = curr_min_slot[60] + for i in range(1, 6): + curr_min_slot[-1 * i] = last_task + elif len(curr_min_slot) < total_expected_min: + last_task = curr_min_slot[-1] + for i in range(total_expected_min - len(curr_min_slot)): + curr_min_slot += [last_task] + + cr_ret = [ + ["dummy", -1], + ] + for task, task_index in curr_min_slot: + if task != cr_ret[-1][0]: + cr_ret += [[task, 1]] + else: + cr_ret[-1][1] += 1 + cr = cr_ret[1:] + + return cr + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + # TODO -- this sometimes generates error + try: + self._func_cleanup(llm_resp, prompt) + except Exception: + return False + return True + + def _func_fail_default_resp(self) -> int: + fs = [["asleep", 0]] + return fs + + async def run(self, role: "STRole", task_desc: int, truncated_act_dur: int, *args, **kwargs): + def create_prompt_input(role, task, duration): + """ + Today is Saturday June 25. From 00:00 ~ 06:00am, Maeve is + planning on sleeping, 06:00 ~ 07:00am, Maeve is + planning on waking up and doing her morning routine, + and from 07:00am ~08:00am, Maeve is planning on having breakfast. + """ + + curr_f_org_index = role.scratch.get_f_daily_schedule_hourly_org_index() + all_indices = [] + # if curr_f_org_index > 0: + # all_indices += [curr_f_org_index-1] + all_indices += [curr_f_org_index] + if curr_f_org_index + 1 <= len(role.scratch.f_daily_schedule_hourly_org): + all_indices += [curr_f_org_index + 1] + if curr_f_org_index + 2 <= len(role.scratch.f_daily_schedule_hourly_org): + all_indices += [curr_f_org_index + 2] + + curr_time_range = "" + + print("DEBUG") + print(role.scratch.f_daily_schedule_hourly_org) + print(all_indices) + + summ_str = f'Today is {role.scratch.curr_time.strftime("%B %d, %Y")}. ' + summ_str += "From " + for index in all_indices: + print("index", index) + if index < len(role.scratch.f_daily_schedule_hourly_org): + start_min = 0 + for i in range(index): + start_min += role.scratch.f_daily_schedule_hourly_org[i][1] + end_min = start_min + role.scratch.f_daily_schedule_hourly_org[index][1] + start_time = datetime.datetime.strptime("00:00:00", "%H:%M:%S") + datetime.timedelta( + minutes=start_min + ) + end_time = datetime.datetime.strptime("00:00:00", "%H:%M:%S") + datetime.timedelta( + minutes=end_min + ) + start_time_str = start_time.strftime("%H:%M%p") + end_time_str = end_time.strftime("%H:%M%p") + summ_str += ( + f"{start_time_str} ~ {end_time_str}, {role.name} is planning " + f"on {role.scratch.f_daily_schedule_hourly_org[index][0]}, " + ) + if curr_f_org_index + 1 == index: + curr_time_range = f"{start_time_str} ~ {end_time_str}" + summ_str = summ_str[:-2] + "." + + prompt_input = [] + prompt_input += [role.scratch.get_str_iss()] + prompt_input += [summ_str] + # prompt_input += [role.scratch.get_str_curr_date_str()] + prompt_input += [role.scratch.get_str_firstname()] + prompt_input += [role.scratch.get_str_firstname()] + prompt_input += [task] + prompt_input += [curr_time_range] + prompt_input += [duration] + prompt_input += [role.scratch.get_str_firstname()] + return prompt_input + + prompt_input = create_prompt_input(role, task_desc, truncated_act_dur) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "task_decomp_v3.txt") + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=1000) + logger.info(f"Role: {role.name} {self.cls_name} output: {output}") + + fin_output = [] + time_sum = 0 + for i_task, i_duration in output: + time_sum += i_duration + # HM????????? + # if time_sum < duration: + if time_sum <= truncated_act_dur: + fin_output += [[i_task, i_duration]] + else: + break + ftime_sum = 0 + for fi_task, fi_duration in fin_output: + ftime_sum += fi_duration + + # print ("for debugging... line 365", fin_output) + fin_output[-1][1] += truncated_act_dur - ftime_sum + output = fin_output + + task_decomp = output + ret = [] + for decomp_task, duration in task_decomp: + ret += [[f"{task_desc} ({decomp_task})", duration]] + output = ret + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/wake_up.py b/examples/st_game/actions/wake_up.py new file mode 100644 index 000000000..d39115854 --- /dev/null +++ b/examples/st_game/actions/wake_up.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : wake_up + + +from metagpt.logs import logger + +from ..actions.st_action import STAction + + +class WakeUp(STAction): + name: str = "WakeUp" + + def _func_validate(self, llm_resp: str, prompt: str = None) -> bool: + try: + self._func_cleanup(llm_resp, prompt="") + except Exception: + return False + return True + + def _func_cleanup(self, llm_resp: str, prompt: str) -> int: + cr = int(llm_resp.strip().lower().split("am")[0]) + return cr + + def _func_fail_default_resp(self) -> int: + fs = 8 + return fs + + async def run(self, role: "STRole"): + def create_prompt_input(role): + prompt_input = [ + role.scratch.get_str_iss(), + role.scratch.get_str_lifestyle(), + role.scratch.get_str_firstname(), + ] + return prompt_input + + prompt_input = create_prompt_input(role) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "wake_up_hour_v1.txt") + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=5) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/memory/__init__.py b/examples/st_game/memory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/st_game/memory/agent_memory.py b/examples/st_game/memory/agent_memory.py new file mode 100644 index 000000000..a66a40efd --- /dev/null +++ b/examples/st_game/memory/agent_memory.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : BasicMemory,AgentMemory实现 + +from datetime import datetime +from pathlib import Path +from typing import Optional + +from pydantic import Field, field_serializer, model_validator + +from metagpt.logs import logger +from metagpt.memory.memory import Memory +from metagpt.schema import Message +from metagpt.utils.common import read_json_file, write_json_file + + +class BasicMemory(Message): + """ + BasicMemory继承于MG的Message类，其中content属性替代description属性 + Message类中对于Chat类型支持的非常好，对于Agent个体的Perceive,Reflection,Plan支持的并不多 + 在Type设计上，我们延续GA的三个种类，但是对于Chat种类的对话进行特别设计（具体怎么设计还没想好） + """ + + memory_id: Optional[str] = Field(default=None) # 记忆ID + memory_count: int = -1 # 第几个记忆，实际数值与Memory相等 + type_count: int = -1 # 第几种记忆，类型为整数 + memory_type: Optional[str] = Field(default=None) # 记忆类型，包含 event,thought,chat三种类型 + depth: int = -1 # 记忆深度，类型为整数 + created: Optional[datetime] = Field(default=None) # 创建时间 + expiration: Optional[datetime] = Field(default=None) # 记忆失效时间，默认为空（） + last_accessed: Optional[datetime] = Field(default=None) # 上一次调用的时间，初始化时候与self.created一致 + subject: Optional[str] = Field(default=None) # 主语 + predicate: Optional[str] = Field(default=None) # 谓语 + object: Optional[str] = Field(default=None) # 宾语 + + description: Optional[str] = Field(default=None) + embedding_key: Optional[str] = Field(default=None) # 内容与self.content一致 + poignancy: int = -1 # importance值 + keywords: list[str] = Field(default=[]) # keywords + filling: list = Field(default=[]) # 装的与之相关联的memory_id的列表 + + @model_validator(mode="before") + @classmethod + def check_values(cls, values): + if "created" in values: + values["last_accessed"] = values["created"] + if "content" in values: + values["description"] = values["content"] + return values + + @field_serializer("created", "expiration") + def transform_time_field(self, time_field: Optional[datetime]) -> str: + if time_field: + time_field = time_field.strftime("%Y-%m-%d %H:%M:%S") + return time_field + + def summary(self): + return self.subject, self.predicate, self.object + + def save_to_dict(self) -> dict: + """ + 将MemoryBasic类转化为字典，用于存储json文件 + 这里需要注意，cause_by跟GA不兼容，所以需要做一个格式转换 + """ + memory_dict = dict() + node_id = self.memory_id + basic_mem_obj = self.model_dump( + include=[ + "node_count", + "type_count", + "type", + "depth", + "created", + "expiration", + "subject", + "predicate", + "object", + "description", + "embedding_key", + "poignancy", + "keywords", + "filling", + "cause_by", + ] + ) + + memory_dict[node_id] = basic_mem_obj + return memory_dict + + +class AgentMemory(Memory): + """ + GA中主要存储三种JSON + 1. embedding.json (Dict embedding_key:embedding) + 2. Node.json (Dict Node_id:Node) + 3. kw_strength.json + """ + + storage: list[BasicMemory] = [] # 重写Storage，存储BasicMemory所有节点 + event_list: list[BasicMemory] = [] # 存储event记忆 + thought_list: list[BasicMemory] = [] # 存储thought记忆 + chat_list: list[BasicMemory] = [] # chat-related memory + + event_keywords: dict[str, list[BasicMemory]] = dict() # 存储keywords + thought_keywords: dict[str, list[BasicMemory]] = dict() + chat_keywords: dict[str, list[BasicMemory]] = dict() + + kw_strength_event: dict[str, int] = dict() + kw_strength_thought: dict[str, int] = dict() + + memory_saved: Optional[Path] = Field(default=None) + embeddings: dict[str, list[float]] = dict() + + def set_mem_path(self, memory_saved: Path): + self.memory_saved = memory_saved + self.load(memory_saved) + + def save(self, memory_saved: Path): + """ + 将MemoryBasic类存储为Nodes.json形式。复现GA中的Kw Strength.json形式 + 这里添加一个路径即可 + TODO 这里在存储时候进行倒序存储，之后需要验证（test_memory通过） + """ + memory_json = dict() + for i in range(len(self.storage)): + memory_node = self.storage[len(self.storage) - i - 1] + memory_node = memory_node.save_to_dict() + memory_json.update(memory_node) + write_json_file(memory_saved.joinpath("nodes.json"), memory_json) + write_json_file(memory_saved.joinpath("embeddings.json"), self.embeddings) + + strength_json = dict() + strength_json["kw_strength_event"] = self.kw_strength_event + strength_json["kw_strength_thought"] = self.kw_strength_thought + write_json_file(memory_saved.joinpath("kw_strength.json"), strength_json) + + def load(self, memory_saved: Path): + """ + 将GA的JSON解析，填充到AgentMemory类之中 + """ + self.embeddings = read_json_file(memory_saved.joinpath("embeddings.json")) + memory_load = read_json_file(memory_saved.joinpath("nodes.json")) + for count in range(len(memory_load.keys())): + node_id = f"node_{str(count + 1)}" + node_details = memory_load[node_id] + node_type = node_details["type"] + created = datetime.strptime(node_details["created"], "%Y-%m-%d %H:%M:%S") + expiration = None + if node_details["expiration"]: + expiration = datetime.strptime(node_details["expiration"], "%Y-%m-%d %H:%M:%S") + + s = node_details["subject"] + p = node_details["predicate"] + o = node_details["object"] + + description = node_details["description"] + embedding_pair = (node_details["embedding_key"], self.embeddings[node_details["embedding_key"]]) + poignancy = node_details["poignancy"] + keywords = set(node_details["keywords"]) + filling = node_details["filling"] + if node_type == "thought": + self.add_thought( + created, expiration, s, p, o, description, keywords, poignancy, embedding_pair, filling + ) + if node_type == "event": + self.add_event(created, expiration, s, p, o, description, keywords, poignancy, embedding_pair, filling) + if node_type == "chat": + self.add_chat(created, expiration, s, p, o, description, keywords, poignancy, embedding_pair, filling) + + strength_keywords_load = read_json_file(memory_saved.joinpath("kw_strength.json")) + if strength_keywords_load["kw_strength_event"]: + self.kw_strength_event = strength_keywords_load["kw_strength_event"] + if strength_keywords_load["kw_strength_thought"]: + self.kw_strength_thought = strength_keywords_load["kw_strength_thought"] + + def add(self, memory_basic: BasicMemory): + """ + Add a new message to storage, while updating the index + 重写add方法，修改原有的Message类为BasicMemory类，并添加不同的记忆类型添加方式 + """ + if memory_basic.memory_id in self.storage: + return + self.storage.append(memory_basic) + if memory_basic.memory_type == "chat": + self.chat_list[0:0] = [memory_basic] + return + if memory_basic.memory_type == "thought": + self.thought_list[0:0] = [memory_basic] + return + if memory_basic.memory_type == "event": + self.event_list[0:0] = [memory_basic] + return + + def add_chat( + self, created, expiration, s, p, o, content, keywords, poignancy, embedding_pair, filling, cause_by="" + ): + """ + 调用add方法，初始化chat，在创建的时候就需要调用embedding函数 + """ + memory_count = len(self.storage) + 1 + type_count = len(self.thought_list) + 1 + memory_type = "chat" + memory_id = f"node_{str(memory_count)}" + depth = 1 + + memory_node = BasicMemory( + memory_id=memory_id, + memory_count=memory_count, + type_count=type_count, + memory_type=memory_type, + depth=depth, + created=created, + expiration=expiration, + subject=s, + predicate=p, + object=o, + description=content, + embedding_key=embedding_pair[0], + poignancy=poignancy, + keywords=keywords, + filling=filling, + cause_by=cause_by + ) + + keywords = [i.lower() for i in keywords] + for kw in keywords: + if kw in self.chat_keywords: + self.chat_keywords[kw][0:0] = [memory_node] + else: + self.chat_keywords[kw] = [memory_node] + + self.add(memory_node) + + self.embeddings[embedding_pair[0]] = embedding_pair[1] + return memory_node + + def add_thought(self, created, expiration, s, p, o, content, keywords, poignancy, embedding_pair, filling): + """ + 调用add方法，初始化thought + """ + memory_count = len(self.storage) + 1 + type_count = len(self.thought_list) + 1 + memory_type = "thought" + memory_id = f"node_{str(memory_count)}" + depth = 1 + + try: + if filling: + depth_list = [memory_node.depth for memory_node in self.storage if memory_node.memory_id in filling] + depth += max(depth_list) + except Exception as exp: + logger.warning(f"filling init occur {exp}") + pass + + memory_node = BasicMemory( + memory_id=memory_id, + memory_count=memory_count, + type_count=type_count, + memory_type=memory_type, + depth=depth, + created=created, + expiration=expiration, + subject=s, + predicate=p, + object=o, + description=content, + embedding_key=embedding_pair[0], + poignancy=poignancy, + keywords=keywords, + filling=filling + ) + + keywords = [i.lower() for i in keywords] + for kw in keywords: + if kw in self.thought_keywords: + self.thought_keywords[kw][0:0] = [memory_node] + else: + self.thought_keywords[kw] = [memory_node] + + self.add(memory_node) + + if f"{p} {o}" != "is idle": + for kw in keywords: + if kw in self.kw_strength_thought: + self.kw_strength_thought[kw] += 1 + else: + self.kw_strength_thought[kw] = 1 + + self.embeddings[embedding_pair[0]] = embedding_pair[1] + return memory_node + + def add_event(self, created, expiration, s, p, o, content, keywords, poignancy, embedding_pair, filling): + """ + 调用add方法，初始化event + """ + memory_count = len(self.storage) + 1 + type_count = len(self.event_list) + 1 + memory_type = "event" + memory_id = f"node_{str(memory_count)}" + depth = 0 + + if "(" in content: + content = " ".join(content.split()[:3]) + " " + content.split("(")[-1][:-1] + + memory_node = BasicMemory( + memory_id=memory_id, + memory_count=memory_count, + type_count=type_count, + memory_type=memory_type, + depth=depth, + created=created, + expiration=expiration, + subject=s, + predicate=p, + object=o, + description=content, + embedding_key=embedding_pair[0], + poignancy=poignancy, + keywords=keywords, + filling=filling + ) + + keywords = [i.lower() for i in keywords] + for kw in keywords: + if kw in self.event_keywords: + self.event_keywords[kw][0:0] = [memory_node] + else: + self.event_keywords[kw] = [memory_node] + + self.add(memory_node) + + if f"{p} {o}" != "is idle": + for kw in keywords: + if kw in self.kw_strength_event: + self.kw_strength_event[kw] += 1 + else: + self.kw_strength_event[kw] = 1 + + self.embeddings[embedding_pair[0]] = embedding_pair[1] + return memory_node + + def get_summarized_latest_events(self, retention): + ret_set = set() + for e_node in self.event_list[:retention]: + ret_set.add(e_node.summary()) + return ret_set + + def get_last_chat(self, target_role_name: str): + if target_role_name.lower() in self.chat_keywords: + return self.chat_keywords[target_role_name.lower()][0] + else: + return False + + def retrieve_relevant_thoughts(self, s_content: str, p_content: str, o_content: str) -> set: + contents = [s_content, p_content, o_content] + + ret = [] + for i in contents: + if i in self.thought_keywords: + ret += self.thought_keywords[i.lower()] + + ret = set(ret) + return ret + + def retrieve_relevant_events(self, s_content: str, p_content: str, o_content: str) -> set: + contents = [s_content, p_content, o_content] + + ret = [] + for i in contents: + if i in self.event_keywords: + ret += self.event_keywords[i] + + ret = set(ret) + return ret diff --git a/examples/st_game/memory/retrieve.py b/examples/st_game/memory/retrieve.py new file mode 100644 index 000000000..eeeecf85a --- /dev/null +++ b/examples/st_game/memory/retrieve.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : Retrieve函数实现 + +import datetime + +from numpy import dot +from numpy.linalg import norm + +from examples.st_game.memory.agent_memory import BasicMemory +from examples.st_game.utils.utils import get_embedding + + +def agent_retrieve( + agent_memory, + curr_time: datetime.datetime, + memory_forget: float, + query: str, + nodes: list[BasicMemory], + topk: int = 4, +) -> list[BasicMemory]: + """ + Retrieve需要集合Role使用,原因在于Role才具有AgentMemory,scratch + 逻辑:Role调用该函数,self.rc.AgentMemory,self.rc.scratch.curr_time,self.rc.scratch.memory_forget + 输入希望查询的内容与希望回顾的条数,返回TopK条高分记忆，即List[BasicMemory] + + Score_lists示例 + { + "memory": memories[i], BasicMemory类 + "importance": memories[i].poignancy + "recency": 衰减因子计算结果 + "relevance": 搜索结果 + } + """ + memories = nodes + agent_memory_embedding = agent_memory.embeddings + memories = sorted(memories, key=lambda memory_node: memory_node.last_accessed, reverse=True) + + score_list = [] + score_list = extract_importance(memories, score_list) + score_list = extract_recency(curr_time, memory_forget, score_list) + score_list = extract_relevance(agent_memory_embedding, query, score_list) + score_list = normalize_score_floats(score_list, 0, 1) + + total_dict = {} + gw = [1, 1, 1] # 三个因素的权重,重要性,近因性,相关性, + for i in range(len(score_list)): + total_score = ( + score_list[i]["importance"] * gw[0] + score_list[i]["recency"] * gw[1] + score_list[i]["relevance"] * gw[2] + ) + total_dict[score_list[i]["memory"].memory_id] = total_score + + result = top_highest_x_values(total_dict, topk) + + return result # 返回的是一个BasicMemory列表 + + +def new_agent_retrieve(role, focus_points: list, n_count=30) -> dict: + """ + 输入为role，关注点列表,返回记忆数量 + 输出为字典，键为focus_point，值为对应的记忆列表 + """ + retrieved = dict() + for focal_pt in focus_points: + nodes = [ + [i.last_accessed, i] + for i in role.memory.event_list + role.memory.thought_list + if "idle" not in i.embedding_key + ] + nodes = sorted(nodes, key=lambda x: x[0]) + nodes = [i for created, i in nodes] + results = agent_retrieve( + role.memory, role.scratch.curr_time, role.scratch.recency_decay, focal_pt, nodes, n_count + ) + final_result = [] + for n in results: + for i in role.memory.storage: + if i.memory_id == n: + i.last_accessed = role.scratch.curr_time + final_result.append(i) + + retrieved[focal_pt] = final_result + + return retrieved + + +def top_highest_x_values(d, x): + """ + 输入字典，Topx + 返回以字典值排序，字典键组成的List[BasicMemory] + """ + top_v = [item[0] for item in sorted(d.items(), key=lambda item: item[1], reverse=True)[:x]] + return top_v + + +def extract_importance(memories, score_list): + """ + 抽取重要性 + """ + for i in range(len(memories)): + score = {"memory": memories[i], "importance": memories[i].poignancy} + score_list.append(score) + return score_list + + +def extract_relevance(agent_memory_embedding, query, score_list): + """ + 抽取相关性 + """ + query_embedding = get_embedding(query) + # 进行 + for i in range(len(score_list)): + node_embedding = agent_memory_embedding[score_list[i]["memory"].embedding_key] + result = cos_sim(node_embedding, query_embedding) + score_list[i]["relevance"] = result + + return score_list + + +def extract_recency(curr_time, memory_forget, score_list): + """ + 抽取近因性，目前使用的现实世界过一天走一个衰减因子 + """ + for i in range(len(score_list)): + day_count = (curr_time - score_list[i]["memory"].created).days + score_list[i]["recency"] = memory_forget**day_count + return score_list + + +def cos_sim(a, b): + """ + 计算余弦相似度 + """ + return dot(a, b) / (norm(a) * norm(b)) + + +def normalize_list_floats(single_list, target_min, target_max): + """ + 单个列表归一化 + """ + if len(single_list) == 0: + return [] + + min_val = min(single_list) + max_val = max(single_list) + range_val = max_val - min_val + + if range_val == 0: + for i in range(len(single_list)): + single_list[i] = (target_max - target_min) / 2 + else: + for i in range(len(single_list)): + single_list[i] = (single_list[i] - min_val) * (target_max - target_min) / range_val + target_min + return single_list + + +def normalize_score_floats(score_list, target_min, target_max): + """ + 整体归一化 + """ + importance_list = [] + relevance_list = [] + recency_list = [] + + for i in range(len(score_list)): + importance_list.append(score_list[i]["importance"]) + relevance_list.append(score_list[i]["relevance"]) + recency_list.append(score_list[i]["recency"]) + + # 进行归一化操作 + importance_list = normalize_list_floats(importance_list, target_min, target_max) + relevance_list = normalize_list_floats(relevance_list, target_min, target_max) + recency_list = normalize_list_floats(recency_list, target_min, target_max) + + for i in range(len(score_list)): + score_list[i]["importance"] = importance_list[i] + score_list[i]["relevance"] = relevance_list[i] + score_list[i]["recency"] = recency_list[i] + + return score_list diff --git a/examples/st_game/memory/scratch.py b/examples/st_game/memory/scratch.py new file mode 100644 index 000000000..b7040ce08 --- /dev/null +++ b/examples/st_game/memory/scratch.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : Scratch类实现（角色信息类） + +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +from pydantic import ( + BaseModel, + Field, + field_serializer, + field_validator, + model_validator, +) + +from metagpt.utils.common import read_json_file, write_json_file + + +class Scratch(BaseModel): + # 类别1:人物超参 + vision_r: int = 4 + att_bandwidth: int = 3 + retention: int = 5 + + # 类别2:世界信息 + curr_time: Optional[datetime] = Field(default=None) + curr_tile: Optional[str] = Field(default=None) + daily_plan_req: Optional[str] = Field(default=None) + + # 类别3:人物角色的核心身份 + name: Optional[str] = Field(default=None) + first_name: Optional[str] = Field(default=None) + last_name: Optional[str] = Field(default=None) + age: Optional[int] = Field(default=None) + innate: Optional[str] = Field(default=None) # L0 permanent core traits. + learned: Optional[str] = Field(default=None) # L1 stable traits. + currently: Optional[str] = Field(default=None) # L2 external implementation. + lifestyle: Optional[str] = Field(default=None) + living_area: Optional[str] = Field(default=None) + + # 类别4:旧反思变量 + concept_forget: int = 100 + daily_reflection_time: int = 60 * 3 + daily_reflection_size: int = 5 + overlap_reflect_th: int = 2 + kw_strg_event_reflect_th: int = 4 + kw_strg_thought_reflect_th: int = 4 + + # 类别5:新反思变量 + recency_w: int = 1 + relevance_w: int = 1 + importance_w: int = 1 + recency_decay: float = 0.99 + importance_trigger_max: int = 150 + importance_trigger_curr: int = 150 + importance_ele_n: int = 0 + thought_count: int = 5 + + # 类别6:个人计划 + daily_req: list[str] = Field(default=[]) + f_daily_schedule: list[list[str]] = Field(default=[]) + f_daily_schedule_hourly_org: list[list[str]] = Field(default=[]) + + # 类别7:当前动作 + act_address: Optional[str] = Field(default=None) + act_start_time: Optional[datetime] = Field(default=None) + act_duration: Optional[int] = Field(default=None) + act_description: Optional[str] = Field(default=None) + act_pronunciatio: Optional[str] = Field(default=None) + act_event: tuple[str, Optional[str], Optional[str]] = (None, None, None) + + act_obj_description: Optional[str] = Field(default=None) + act_obj_pronunciatio: Optional[str] = Field(default=None) + act_obj_event: tuple[Optional[str], Optional[str], Optional[str]] = (None, None, None) + + chatting_with: Optional[str] = Field(default=None) + chat: Optional[str] = Field(default=None) + chatting_with_buffer: dict = dict() + chatting_end_time: Optional[datetime] = Field(default=None) + + act_path_set: bool = False + planned_path: list[str] = Field(default=[]) + + @model_validator(mode="after") + @classmethod + def check_values(cls, values): + if "name" in values: + values["act_event"] = (values["name"], None, None) + values["act_obj_event"] = (values["name"], None, None) + return values + + @field_validator("curr_time", "act_start_time", "chatting_end_time") + @classmethod + def check_time_filed(cls, time_filed): + val = datetime.strptime(time_filed, "%B %d, %Y, %H:%M:%S") if time_filed else None + return val + + @field_serializer("curr_time", "act_start_time", "chatting_end_time") + def transform_time_field(self, time_filed: Optional[datetime]) -> str: + if time_filed: + time_filed = time_filed.strftime("%B %d, %Y, %H:%M:%S") + return time_filed + + @classmethod + def set_scratch_path(cls, f_saved: Path): + scratch_load = read_json_file(f_saved) + scratch = Scratch(**scratch_load) + return scratch + + def save(self, out_json: Path): + """ + Save persona's scratch. + + INPUT: + out_json: The file where we wil be saving our persona's state. + OUTPUT: + None + """ + scratch = self.model_dump() + write_json_file(out_json, scratch, encoding="utf-8") + + def get_f_daily_schedule_index(self, advance=0): + """ + We get the current index of self.f_daily_schedule. + + Recall that self.f_daily_schedule stores the decomposed action sequences + up until now, and the hourly sequences of the future action for the rest + of today. Given that self.f_daily_schedule is a list of list where the + inner list is composed of [task, duration], we continue to add up the + duration until we reach "if elapsed > today_min_elapsed" condition. The + index where we stop is the index we will return. + + INPUT + advance: Integer value of the number minutes we want to look into the + future. This allows us to get the index of a future timeframe. + OUTPUT + an integer value for the current index of f_daily_schedule. + """ + # We first calculate teh number of minutes elapsed today. + today_min_elapsed = 0 + today_min_elapsed += self.curr_time.hour * 60 + today_min_elapsed += self.curr_time.minute + today_min_elapsed += advance + + x = 0 + for task, duration in self.f_daily_schedule: + x += duration + x = 0 + for task, duration in self.f_daily_schedule_hourly_org: + x += duration + + # We then calculate the current index based on that. + curr_index = 0 + elapsed = 0 + for task, duration in self.f_daily_schedule: + elapsed += duration + if elapsed > today_min_elapsed: + return curr_index + curr_index += 1 + + return curr_index + + def get_f_daily_schedule_hourly_org_index(self, advance=0): + """ + We get the current index of self.f_daily_schedule_hourly_org. + It is otherwise the same as get_f_daily_schedule_index. + + INPUT + advance: Integer value of the number minutes we want to look into the + future. This allows us to get the index of a future timeframe. + OUTPUT + an integer value for the current index of f_daily_schedule. + """ + # We first calculate teh number of minutes elapsed today. + today_min_elapsed = 0 + today_min_elapsed += self.curr_time.hour * 60 + today_min_elapsed += self.curr_time.minute + today_min_elapsed += advance + # We then calculate the current index based on that. + curr_index = 0 + elapsed = 0 + for task, duration in self.f_daily_schedule_hourly_org: + elapsed += duration + if elapsed > today_min_elapsed: + return curr_index + curr_index += 1 + return curr_index + + def get_str_iss(self): + """ + ISS stands for "identity stable set." This describes the commonset summary + of this persona -- basically, the bare minimum description of the persona + that gets used in almost all prompts that need to call on the persona. + + INPUT + None + OUTPUT + the identity stable set summary of the persona in a string form. + EXAMPLE STR OUTPUT + "Name: Dolores Heitmiller + Age: 28 + Innate traits: hard-edged, independent, loyal + Learned traits: Dolores is a painter who wants live quietly and paint + while enjoying her everyday life. + Currently: Dolores is preparing for her first solo show. She mostly + works from home. + Lifestyle: Dolores goes to bed around 11pm, sleeps for 7 hours, eats + dinner around 6pm. + Daily plan requirement: Dolores is planning to stay at home all day and + never go out." + """ + commonset = "" + commonset += f"Name: {self.name}\n" + commonset += f"Age: {self.age}\n" + commonset += f"Innate traits: {self.innate}\n" + commonset += f"Learned traits: {self.learned}\n" + commonset += f"Currently: {self.currently}\n" + commonset += f"Lifestyle: {self.lifestyle}\n" + commonset += f"Daily plan requirement: {self.daily_plan_req}\n" + commonset += f"Current Date: {self.curr_time.strftime('%A %B %d') if self.curr_time else ''}\n" + return commonset + + def get_str_name(self): + return self.name + + def get_str_firstname(self): + return self.first_name + + def get_str_lastname(self): + return self.last_name + + def get_str_age(self): + return str(self.age) + + def get_str_innate(self): + return self.innate + + def get_str_learned(self): + return self.learned + + def get_str_currently(self): + return self.currently + + def get_str_lifestyle(self): + return self.lifestyle + + def get_str_daily_plan_req(self): + return self.daily_plan_req + + def get_str_curr_date_str(self): + return self.curr_time.strftime("%A %B %d") + + def get_curr_event(self): + if not self.act_address: + return (self.name, None, None) + else: + return self.act_event + + def get_curr_event_and_desc(self): + if not self.act_address: + return (self.name, None, None, None) + else: + return (self.act_event[0], self.act_event[1], self.act_event[2], self.act_description) + + def get_curr_obj_event_and_desc(self): + if not self.act_address: + return ("", None, None, None) + else: + return (self.act_address, self.act_obj_event[1], self.act_obj_event[2], self.act_obj_description) + + def add_new_action( + self, + action_address, + action_duration, + action_description, + action_pronunciatio, + action_event, + chatting_with, + chat, + chatting_with_buffer, + chatting_end_time, + act_obj_description, + act_obj_pronunciatio, + act_obj_event, + act_start_time=None, + ): + self.act_address = action_address + self.act_duration = action_duration + self.act_description = action_description + self.act_pronunciatio = action_pronunciatio + self.act_event = action_event + + self.chatting_with = chatting_with + self.chat = chat + if chatting_with_buffer: + self.chatting_with_buffer.update(chatting_with_buffer) + self.chatting_end_time = chatting_end_time + + self.act_obj_description = act_obj_description + self.act_obj_pronunciatio = act_obj_pronunciatio + self.act_obj_event = act_obj_event + + self.act_start_time = self.curr_time + + self.act_path_set = False + + def act_time_str(self): + """ + Returns a string output of the current time. + + INPUT + None + OUTPUT + A string output of the current time. + EXAMPLE STR OUTPUT + "14:05 P.M." + """ + return self.act_start_time.strftime("%H:%M %p") + + def act_check_finished(self): + """ + Checks whether the self.Action instance has finished. + + INPUT + curr_datetime: Current time. If current time is later than the action's + start time + its duration, then the action has finished. + OUTPUT + Boolean [True]: Action has finished. + Boolean [False]: Action has not finished and is still ongoing. + """ + if not self.act_address: + return True + + if self.chatting_with: + end_time = self.chatting_end_time + else: + x = self.act_start_time + if x.second != 0: + x = x.replace(second=0) + x = x + timedelta(minutes=1) + end_time = x + timedelta(minutes=self.act_duration) + + if end_time.strftime("%H:%M:%S") == self.curr_time.strftime("%H:%M:%S"): + return True + return False + + def act_summarize(self): + """ + Summarize the current action as a dictionary. + + INPUT + None + OUTPUT + ret: A human readable summary of the action. + """ + exp = dict() + exp["persona"] = self.name + exp["address"] = self.act_address + exp["start_datetime"] = self.act_start_time + exp["duration"] = self.act_duration + exp["description"] = self.act_description + exp["pronunciatio"] = self.act_pronunciatio + return exp + + def act_summary_str(self): + """ + Returns a string summary of the current action. Meant to be + human-readable. + + INPUT + None + OUTPUT + ret: A human readable summary of the action. + """ + start_datetime_str = self.act_start_time.strftime("%A %B %d -- %H:%M %p") + ret = f"[{start_datetime_str}]\n" + ret += f"Activity: {self.name} is {self.act_description}\n" + ret += f"Address: {self.act_address}\n" + ret += f"Duration in minutes (e.g., x min): {str(self.act_duration)} min\n" + return ret + + def get_daily_schedule(self, daily_schedule: list[list[str]]): + ret = "" + curr_min_sum = 0 + for row in daily_schedule: + curr_min_sum += row[1] + hour = int(curr_min_sum / 60) + minute = curr_min_sum % 60 + ret += f"{hour:02}:{minute:02} || {row[0]}\n" + return ret + + def get_str_daily_schedule_summary(self): + return self.get_daily_schedule(self.f_daily_schedule) + + def get_str_daily_schedule_hourly_org_summary(self): + return self.get_daily_schedule(self.f_daily_schedule_hourly_org) diff --git a/examples/st_game/memory/spatial_memory.py b/examples/st_game/memory/spatial_memory.py new file mode 100644 index 000000000..c1030cdbb --- /dev/null +++ b/examples/st_game/memory/spatial_memory.py @@ -0,0 +1,115 @@ +""" +Author: Joon Sung Park (joonspk@stanford.edu) + +File: spatial_memory.py +Description: Defines the MemoryTree class that serves as the agents' spatial +memory that aids in grounding their behavior in the game world. +""" +from pathlib import Path + +from pydantic import BaseModel, Field + +from metagpt.utils.common import read_json_file, write_json_file + + +class MemoryTree(BaseModel): + tree: dict = Field(default=dict) + + def set_mem_path(self, f_saved: Path): + self.tree = read_json_file(f_saved) + + def print_tree(self) -> None: + def _print_tree(tree, depth): + dash = " >" * depth + if isinstance(tree, list): + if tree: + print(dash, tree) + return + + for key, val in tree.items(): + if key: + print(dash, key) + _print_tree(val, depth + 1) + + _print_tree(self.tree, 0) + + def save(self, out_json: Path) -> None: + write_json_file(out_json, self.tree) + + def get_str_accessible_sectors(self, curr_world: str) -> str: + """ + Returns a summary string of all the arenas that the persona can access + within the current sector. + + Note that there are places a given persona cannot enter. This information + is provided in the persona sheet. We account for this in this function. + + INPUT + None + OUTPUT + A summary string of all the arenas that the persona can access. + EXAMPLE STR OUTPUT + "bedroom, kitchen, dining room, office, bathroom" + """ + x = ", ".join(list(self.tree[curr_world].keys())) + return x + + def get_str_accessible_sector_arenas(self, sector: str) -> str: + """ + Returns a summary string of all the arenas that the persona can access + within the current sector. + + Note that there are places a given persona cannot enter. This information + is provided in the persona sheet. We account for this in this function. + + INPUT + None + OUTPUT + A summary string of all the arenas that the persona can access. + EXAMPLE STR OUTPUT + "bedroom, kitchen, dining room, office, bathroom" + """ + curr_world, curr_sector = sector.split(":") + if not curr_sector: + return "" + x = ", ".join(list(self.tree[curr_world][curr_sector].keys())) + return x + + def get_str_accessible_arena_game_objects(self, arena: str) -> str: + """ + Get a str list of all accessible game objects that are in the arena. If + temp_address is specified, we return the objects that are available in + that arena, and if not, we return the objects that are in the arena our + persona is currently in. + + INPUT + temp_address: optional arena address + OUTPUT + str list of all accessible game objects in the gmae arena. + EXAMPLE STR OUTPUT + "phone, charger, bed, nightstand" + """ + curr_world, curr_sector, curr_arena = arena.split(":") + + if not curr_arena: + return "" + + try: + x = ", ".join(list(self.tree[curr_world][curr_sector][curr_arena])) + except Exception: + x = ", ".join(list(self.tree[curr_world][curr_sector][curr_arena.lower()])) + return x + + def add_tile_info(self, tile_info: dict) -> None: + if tile_info["world"]: + if tile_info["world"] not in self.tree: + self.tree[tile_info["world"]] = {} + if tile_info["sector"]: + if tile_info["sector"] not in self.tree[tile_info["world"]]: + self.tree[tile_info["world"]][tile_info["sector"]] = {} + if tile_info["arena"]: + if tile_info["arena"] not in self.tree[tile_info["world"]][tile_info["sector"]]: + self.tree[tile_info["world"]][tile_info["sector"]][tile_info["arena"]] = [] + if tile_info["game_object"]: + if tile_info["game_object"] not in self.tree[tile_info["world"]][tile_info["sector"]][tile_info["arena"]]: + self.tree[tile_info["world"]][tile_info["sector"]][tile_info["arena"]] += [tile_info["game_object"]] diff --git a/examples/st_game/plan/__init__.py b/examples/st_game/plan/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/examples/st_game/plan/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/examples/st_game/plan/converse.py b/examples/st_game/plan/converse.py new file mode 100644 index 000000000..0371b3d44 --- /dev/null +++ b/examples/st_game/plan/converse.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : conversation between two agents + +from typing import Tuple + +from examples.st_game.actions.agent_chat_sum_rel import AgentChatSumRel +from examples.st_game.actions.gen_iter_chat_utt import GenIterChatUTT +from examples.st_game.memory.retrieve import new_agent_retrieve +from metagpt.logs import logger + + +async def agent_conversation(init_role: "STRole", target_role: "STRole", conv_rounds: int = 8) -> list[list[str]]: + curr_chat = [] + logger.info(f"Role: {init_role.name} starts a conversation with Role: {target_role.name}") + + for idx in range(conv_rounds): + logger.info(f"Conv round: {idx} between {init_role.name} and {target_role.name}") + scratch = init_role.rc.scratch + target_scratch = target_role.rc.scratch + + focal_points = [f"{target_scratch.name}"] + retrieved = new_agent_retrieve(init_role, focal_points, 50) + relationship = await generate_summarize_agent_relationship(init_role, target_role, retrieved) + logger.info(f"The relationship between {init_role.name} and {target_role.name}: {relationship}") + last_chat = "" + for i in curr_chat[-4:]: + last_chat += ": ".join(i) + "\n" + if last_chat: + focal_points = [f"{relationship}", f"{target_scratch.name} is {target_scratch.act_description}", last_chat] + else: + focal_points = [f"{relationship}", f"{target_scratch.name} is {target_scratch.act_description}"] + retrieved = new_agent_retrieve(init_role, focal_points, 15) + utt, end = await generate_one_utterance(init_role, target_role, retrieved, curr_chat) + + curr_chat += [[scratch.name, utt]] + if end: + break + + focal_points = [f"{scratch.name}"] + retrieved = new_agent_retrieve(target_role, focal_points, 50) + relationship = await generate_summarize_agent_relationship(target_role, init_role, retrieved) + logger.info(f"The relationship between {target_role.name} and {init_role.name}: {relationship}") + last_chat = "" + for i in curr_chat[-4:]: + last_chat += ": ".join(i) + "\n" + if last_chat: + focal_points = [f"{relationship}", f"{scratch.name} is {scratch.act_description}", last_chat] + else: + focal_points = [f"{relationship}", f"{scratch.name} is {scratch.act_description}"] + retrieved = new_agent_retrieve(target_role, focal_points, 15) + utt, end = await generate_one_utterance(target_role, init_role, retrieved, curr_chat) + + curr_chat += [[target_scratch.name, utt]] + if end: + break + + logger.warning(f"Conversations between {target_role.name} and {init_role.name}:") + for row in curr_chat: + logger.info(row) + + return curr_chat + + +async def generate_summarize_agent_relationship(init_role: "STRole", target_role: "STRole", retrieved: dict) -> str: + all_embedding_keys = list() + for key, val in retrieved.items(): + for i in val: + all_embedding_keys += [i.embedding_key] + all_embedding_key_str = "" + for i in all_embedding_keys: + all_embedding_key_str += f"{i}\n" + + summarized_relationship = await AgentChatSumRel().run(init_role, target_role, all_embedding_key_str) + return summarized_relationship + + +async def generate_one_utterance(init_role, target_role, retrieved: dict, curr_chat: list) -> Tuple[str, str]: + # Chat version optimized for speed via batch generation + scratch = init_role.rc.scratch + target_scratch = target_role.rc.scratch + curr_context = ( + f"{scratch.name} " + + f"was {scratch.act_description} " + + f"when {scratch.name} " + + f"saw {target_scratch.name} " + + f"in the middle of {target_scratch.act_description}.\n" + ) + curr_context += f"{scratch.name} " + "is initiating a conversation with " + f"{target_scratch.name}." + + x = await GenIterChatUTT().run(init_role, target_role, retrieved, curr_context, curr_chat) + + return x["utterance"], x["end"] diff --git a/examples/st_game/plan/st_plan.py b/examples/st_game/plan/st_plan.py new file mode 100644 index 000000000..017b3e1fb --- /dev/null +++ b/examples/st_game/plan/st_plan.py @@ -0,0 +1,719 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : st' planning execution + +import datetime +import math +import random +from typing import Tuple, Union + +from metagpt.llm import LLM +from metagpt.logs import logger + +from ..actions.decide_to_talk import DecideToTalk +from ..actions.gen_action_details import GenActionDetails +from ..actions.gen_daily_schedule import GenDailySchedule +from ..actions.gen_hourly_schedule import GenHourlySchedule +from ..actions.new_decomp_schedule import NewDecompSchedule +from ..actions.summarize_conv import SummarizeConv +from ..actions.task_decomp import TaskDecomp +from ..actions.wake_up import WakeUp +from ..memory.retrieve import new_agent_retrieve +from ..plan.converse import agent_conversation +from ..utils.utils import get_embedding + + +async def plan(role: "STRole", roles: dict["STRole"], new_day: bool, retrieved: dict) -> str: + # PART 1: Generate the hourly schedule. + if new_day: + await _long_term_planning(role, new_day) + + # PART 2: If the current action has expired, we want to create a new plan. + act_check_finished = role.scratch.act_check_finished() + logger.info(f"Role: {role.name} act_check_finished is {act_check_finished}") + if act_check_finished: + await _determine_action(role) + + # PART 3: If you perceived an event that needs to be responded to (saw + # another role), and retrieved relevant information. + # Step 1: Retrieved may have multiple events represented in it. The first + # job here is to determine which of the events we want to focus + # on for the role. + # takes the form of a dictionary like this: + # dictionary {["curr_event"] = , + # ["events"] = [, ...], + # ["thoughts"] = [, ...]} + focused_event = False + if retrieved.keys(): + focused_event = _choose_retrieved(role.name, retrieved) + + # Step 2: Once we choose an event, we need to determine whether the + # role will take any actions for the perceived event. There are + # three possible modes of reaction returned by _should_react. + # a) "chat with {target_role.name}" + # b) "react" + # c) False + logger.info(f"Role: {role.name} focused_event: {focused_event}") + if focused_event: + reaction_mode = await _should_react(role, focused_event, roles) + logger.info(f"Role: {role.name} reaction_mode: {reaction_mode}") + if reaction_mode: + # If we do want to chat, then we generate conversation + if reaction_mode[:9] == "chat with": + await _chat_react(role, reaction_mode, roles) + elif reaction_mode[:4] == "wait": + await _wait_react(role, reaction_mode) + + # Step 3: Chat-related state clean up. + # If the persona is not chatting with anyone, we clean up any of the + # chat-related states here. + if role.rc.scratch.act_event[1] != "chat with": + role.rc.scratch.chatting_with = None + role.rc.scratch.chat = None + role.rc.scratch.chatting_end_time = None + # We want to make sure that the persona does not keep conversing with each + # other in an infinite loop. So, chatting_with_buffer maintains a form of + # buffer that makes the persona wait from talking to the same target + # immediately after chatting once. We keep track of the buffer value here. + curr_persona_chat_buffer = role.rc.scratch.chatting_with_buffer + for persona_name, buffer_count in curr_persona_chat_buffer.items(): + if persona_name != role.rc.scratch.chatting_with: + role.rc.scratch.chatting_with_buffer[persona_name] -= 1 + + return role.rc.scratch.act_address + + +def _choose_retrieved(role_name: str, retrieved: dict) -> Union[None, dict]: + """ + Retrieved elements have multiple core "curr_events". We need to choose one + event to which we are going to react to. We pick that event here. + Args: + role_name: Current role instance's name whose action we are determining. + retrieved: A dictionary of that were retrieved from the + the role's associative memory. This dictionary takes the + following form: + dictionary[event.description] = + {["curr_event"] = , + ["events"] = [, ...], + ["thoughts"] = [, ...] } + """ + # Once we are done with the reflection, we might want to build a more + # complex structure here. + + # We do not want to take self events... for now + copy_retrieved = retrieved.copy() + for event_desc, rel_ctx in copy_retrieved.items(): + curr_event = rel_ctx["curr_event"] + if curr_event.subject == role_name: + del retrieved[event_desc] + + # Always choose role first. + priority = [] + for event_desc, rel_ctx in retrieved.items(): + curr_event = rel_ctx["curr_event"] + if ":" not in curr_event.subject and curr_event.subject != role_name: + priority += [rel_ctx] + if priority: + return random.choice(priority) + + # Skip idle. + for event_desc, rel_ctx in retrieved.items(): + if "is idle" not in event_desc: + priority += [rel_ctx] + if priority: + return random.choice(priority) + return None + + +async def _should_react(role: "STRole", retrieved: dict, roles: dict): + """ + Determines what form of reaction the role should exihibit given the + retrieved values. + INPUT + role: Current <"STRole"> instance whose action we are determining. + retrieved: A dictionary of that were retrieved from the + the role's associative memory. This dictionary takes the + following form: + dictionary[event.description] = + {["curr_event"] = , + ["events"] = [, ...], + ["thoughts"] = [, ...] } + roles: A dictionary that contains all role names as keys, and the + <"STRole"> instance as values. + """ + + async def lets_talk(init_role: "STRole", target_role: "STRole", retrieved: dict): + if init_role.name == target_role.name: + logger.info(f"Role: {role.name} _should_react lets_talk meet same role, return False") + return False + + scratch = init_role.rc.scratch + target_scratch = target_role.rc.scratch + if ( + not target_scratch.act_address + or not target_scratch.act_description + or not scratch.act_address + or not scratch.act_description + ): + return False + + if "sleeping" in target_scratch.act_description or "sleeping" in scratch.act_description: + return False + + if scratch.curr_time.hour == 23: + return False + + if "" in target_scratch.act_address: + return False + + if target_scratch.chatting_with or scratch.chatting_with: + return False + + if target_role.name in scratch.chatting_with_buffer: + if scratch.chatting_with_buffer[target_role.name] > 0: + return False + + if await DecideToTalk().run(init_role, target_role, retrieved): + return True + + return False + + async def lets_react(init_role: "STRole", target_role: "STRole", retrieved: dict): + if init_role.name == target_role.name: + logger.info(f"Role: {role.name} _should_react lets_react meet same role, return False") + return False + + scratch = init_role.rc.scratch + target_scratch = target_role.rc.scratch + if ( + not target_scratch.act_address + or not target_scratch.act_description + or not scratch.act_address + or not scratch.act_description + ): + return False + + if "sleeping" in target_scratch.act_description or "sleeping" in scratch.act_description: + return False + + # return False + if scratch.curr_time.hour == 23: + return False + + if "waiting" in target_scratch.act_description: + return False + if scratch.planned_path == []: + return False + + if scratch.act_address != target_scratch.act_address: + return False + + react_mode = await DecideToTalk().run(init_role, target_role, retrieved) + + if react_mode == "1": + wait_until = ( + target_scratch.act_start_time + datetime.timedelta(minutes=target_scratch.act_duration - 1) + ).strftime("%B %d, %Y, %H:%M:%S") + return f"wait: {wait_until}" + elif react_mode == "2": + return False + return "do other things" + else: + return False # "keep" + + # If the role is chatting right now, default to no reaction + scratch = role.rc.scratch + if scratch.chatting_with: + return False + if "" in scratch.act_address: + return False + + # Recall that retrieved takes the following form: + # dictionary {["curr_event"] = } + curr_event = retrieved["curr_event"] + logger.info(f"Role: {role.name} _should_react curr_event.subject: {curr_event.subject}") + + if ":" not in curr_event.subject: + # this is a role event. + if await lets_talk(role, roles[curr_event.subject], retrieved): + return f"chat with {curr_event.subject}" + react_mode = await lets_react(role, roles[curr_event.subject], retrieved) + return react_mode + return False + + +async def _chat_react(role: "STRole", reaction_mode: str, roles: dict["STRole"]): + # There are two roles -- the role who is initiating the conversation + # and the role who is the target. We get the role instances here. + init_role = role + target_role = roles[reaction_mode[9:].strip()] + + # Actually creating the conversation here. + convo, duration_min = await generate_convo(init_role, target_role) # 2222 + convo_summary = await generate_convo_summary(convo) + inserted_act = convo_summary + inserted_act_dur = duration_min + + act_start_time = target_role.rc.scratch.act_start_time + + curr_time = target_role.rc.scratch.curr_time + if curr_time.second != 0: + temp_curr_time = curr_time + datetime.timedelta(seconds=60 - curr_time.second) + chatting_end_time = temp_curr_time + datetime.timedelta(minutes=inserted_act_dur) + else: + chatting_end_time = curr_time + datetime.timedelta(minutes=inserted_act_dur) + + for role, p in [("init", init_role), ("target", target_role)]: + if role == "init": + act_address = f" {target_role.name}" + act_event = (p.name, "chat with", target_role.name) + chatting_with = target_role.name + chatting_with_buffer = {} + chatting_with_buffer[target_role.name] = 800 + elif role == "target": + act_address = f" {init_role.name}" + act_event = (p.name, "chat with", init_role.name) + chatting_with = init_role.name + chatting_with_buffer = {} + chatting_with_buffer[init_role.name] = 800 + + act_pronunciatio = "💬" + act_obj_description = None + act_obj_pronunciatio = None + act_obj_event = (None, None, None) + + await _create_react( + p, + inserted_act, + inserted_act_dur, + act_address, + act_event, + chatting_with, + convo, + chatting_with_buffer, + chatting_end_time, + act_pronunciatio, + act_obj_description, + act_obj_pronunciatio, + act_obj_event, + act_start_time, + ) + + +async def _create_react( + role: "STRole", + inserted_act: str, + inserted_act_dur: int, + act_address: str, + act_event: Tuple, + chatting_with: str, + chat: list, + chatting_with_buffer: dict, + chatting_end_time: datetime, + act_pronunciatio: str, + act_obj_description: str, + act_obj_pronunciatio: str, + act_obj_event: Tuple, + act_start_time=None, +): + p = role + scratch = role.rc.scratch + + min_sum = 0 + for i in range(scratch.get_f_daily_schedule_hourly_org_index()): + min_sum += scratch.f_daily_schedule_hourly_org[i][1] + start_hour = int(min_sum / 60) + + if scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index()][1] >= 120: + end_hour = ( + start_hour + scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index()][1] / 60 + ) + + elif ( + scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index()][1] + + scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index() + 1][1] + ): + end_hour = start_hour + ( + ( + scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index()][1] + + scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index() + 1][1] + ) + / 60 + ) + + else: + end_hour = start_hour + 2 + end_hour = int(end_hour) + + dur_sum = 0 + count = 0 + start_index = None + end_index = None + for act, dur in scratch.f_daily_schedule: + if dur_sum >= start_hour * 60 and start_index is None: + start_index = count + if dur_sum >= end_hour * 60 and end_index is None: + end_index = count + dur_sum += dur + count += 1 + + ret = await generate_new_decomp_schedule(p, inserted_act, inserted_act_dur, start_hour, end_hour) + scratch.f_daily_schedule[start_index:end_index] = ret + scratch.add_new_action( + act_address, + inserted_act_dur, + inserted_act, + act_pronunciatio, + act_event, + chatting_with, + chat, + chatting_with_buffer, + chatting_end_time, + act_obj_description, + act_obj_pronunciatio, + act_obj_event, + act_start_time, + ) + + +async def _wait_react(role: "STRole", reaction_mode: str): + scratch = role.rc.scratch + + inserted_act = f'waiting to start {scratch.act_description.split("(")[-1][:-1]}' + end_time = datetime.datetime.strptime(reaction_mode[6:].strip(), "%B %d, %Y, %H:%M:%S") + inserted_act_dur = ( + (end_time.minute + end_time.hour * 60) - (scratch.curr_time.minute + scratch.curr_time.hour * 60) + 1 + ) + + act_address = f" {scratch.curr_tile[0]} {scratch.curr_tile[1]}" + act_event = (role.name, "waiting to start", scratch.act_description.split("(")[-1][:-1]) + chatting_with = None + chat = None + chatting_with_buffer = None + chatting_end_time = None + + act_pronunciatio = "⌛" + act_obj_description = None + act_obj_pronunciatio = None + act_obj_event = (None, None, None) + + await _create_react( + role, + inserted_act, + inserted_act_dur, + act_address, + act_event, + chatting_with, + chat, + chatting_with_buffer, + chatting_end_time, + act_pronunciatio, + act_obj_description, + act_obj_pronunciatio, + act_obj_event, + ) + + +async def generate_convo(init_role: "STRole", target_role: "STRole") -> Union[list, int]: + convo = await agent_conversation(init_role, target_role) + all_utt = "" + + for row in convo: + speaker = row[0] + utt = row[1] + all_utt += f"{speaker}: {utt}\n" + + convo_length = math.ceil(int(len(all_utt) / 8) / 30) + + return convo, convo_length + + +async def generate_convo_summary(conv: list[list[str]]) -> str: + conv_summary = await SummarizeConv().run(conv) + return conv_summary + + +async def generate_new_decomp_schedule( + role: "STRole", inserted_act: str, inserted_act_dur: int, start_hour: int, end_hour: int +): + # Step 1: Setting up the core variables for the function. + #

is the role whose schedule we are editing right now. + scratch = role.rc.scratch + # indicates the number of minutes that have passed today. + today_min_pass = int(scratch.curr_time.hour) * 60 + int(scratch.curr_time.minute) + 1 + + # Step 2: We need to create and . + main_act_dur = [] + truncated_act_dur = [] + dur_sum = 0 # duration sum + count = 0 # enumerate count + truncated_fin = False + + print("DEBUG::: ", scratch.name) + for act, dur in scratch.f_daily_schedule: + if (dur_sum >= start_hour * 60) and (dur_sum < end_hour * 60): + main_act_dur += [[act, dur]] + if dur_sum <= today_min_pass: + truncated_act_dur += [[act, dur]] + elif dur_sum > today_min_pass and not truncated_fin: + # We need to insert that last act, duration list like this one: + # e.g., ['wakes up and completes her morning routine (wakes up...)', 2] + truncated_act_dur += [[scratch.f_daily_schedule[count][0], dur_sum - today_min_pass]] + truncated_act_dur[-1][-1] -= ( + dur_sum - today_min_pass + ) # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass + 1) + print("DEBUG::: ", truncated_act_dur) + + # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass) + truncated_fin = True + dur_sum += dur + count += 1 + + main_act_dur = main_act_dur + + x = ( + truncated_act_dur[-1][0].split("(")[0].strip() + + " (on the way to " + + truncated_act_dur[-1][0].split("(")[-1][:-1] + + ")" + ) + truncated_act_dur[-1][0] = x + + if "(" in truncated_act_dur[-1][0]: + inserted_act = truncated_act_dur[-1][0].split("(")[0].strip() + " (" + inserted_act + ")" + + # To do inserted_act_dur+1 below is an important decision but I'm not sure + # if I understand the full extent of its implications. Might want to + # revisit. + truncated_act_dur += [[inserted_act, inserted_act_dur]] + start_time_hour = datetime.datetime(2022, 10, 31, 0, 0) + datetime.timedelta(hours=start_hour) + end_time_hour = datetime.datetime(2022, 10, 31, 0, 0) + datetime.timedelta(hours=end_hour) + + return await NewDecompSchedule().run( + role, main_act_dur, truncated_act_dur, start_time_hour, end_time_hour, inserted_act, inserted_act_dur + ) + + +async def _long_term_planning(role: "STRole", new_day: bool): + """ + Formulates the role's daily long-term plan if it is the start of a new + day. This basically has two components: first, we create the wake-up hour, + and second, we create the hourly schedule based on it. + INPUT + new_day: Indicates whether the current time signals a "First day", + "New day", or False (for neither). This is important because we + create the roles' long term planning on the new day. + """ + # We start by creating the wake up hour for the role. + wake_up_hour = await WakeUp().run(role) + wake_up_hour = int(wake_up_hour) + logger.info(f"Role: {role.name} long_term_planning, wake_up_hour: {wake_up_hour}") + + # When it is a new day, we start by creating the daily_req of the role. + # Note that the daily_req is a list of strings that describe the role's + # day in broad strokes. + if new_day == "First day": + # Bootstrapping the daily plan for the start of then generation: + # if this is the start of generation (so there is no previous day's + # daily requirement, or if we are on a new day, we want to create a new + # set of daily requirements. + role.scratch.daily_req = await GenDailySchedule().run(role, wake_up_hour) + logger.info(f"Role: {role.name} daily requirements: {role.scratch.daily_req}") + elif new_day == "New day": + revise_identity(role) + + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - TODO + # We need to create a new daily_req here... + role.scratch.daily_req = role.scratch.daily_req + + # Based on the daily_req, we create an hourly schedule for the role, + # which is a list of todo items with a time duration (in minutes) that + # add up to 24 hours. + role.scratch.f_daily_schedule = await GenHourlySchedule().run(role, wake_up_hour) + logger.info(f"Role: {role.name} f_daily_schedule: {role.scratch.f_daily_schedule}") + role.scratch.f_daily_schedule_hourly_org = role.scratch.f_daily_schedule[:] + + # Added March 4 -- adding plan to the memory. + thought = f"This is {role.scratch.name}'s plan for {role.scratch.curr_time.strftime('%A %B %d')}:" + for i in role.scratch.daily_req: + thought += f" {i}," + thought = thought[:-1] + "." + created = role.scratch.curr_time + expiration = role.scratch.curr_time + datetime.timedelta(days=30) + s, p, o = (role.scratch.name, "plan", role.scratch.curr_time.strftime("%A %B %d")) + keywords = set(["plan"]) + thought_poignancy = 5 + thought_embedding_pair = (thought, get_embedding(thought)) + role.a_mem.add_thought( + created, expiration, s, p, o, thought, keywords, thought_poignancy, thought_embedding_pair, None + ) + + # print("Sleeping for 20 seconds...") + # time.sleep(10) + # print("Done sleeping!") + + +async def _determine_action(role: "STRole"): + """ + Creates the next action sequence for the role. + The main goal of this function is to run "add_new_action" on the role's + scratch space, which sets up all the action related variables for the next + action. + As a part of this, the role may need to decompose its hourly schedule as + needed. + INPUT + role: Current instance whose action we are determining. + """ + + def determine_decomp(act_desp, act_dura): + """ + Given an action description and its duration, we determine whether we need + to decompose it. If the action is about the agent sleeping, we generally + do not want to decompose it, so that's what we catch here. + + INPUT: + act_desp: the description of the action (e.g., "sleeping") + act_dura: the duration of the action in minutes. + OUTPUT: + a boolean. True if we need to decompose, False otherwise. + """ + if "sleep" not in act_desp and "bed" not in act_desp: + return True + elif "sleeping" in act_desp or "asleep" in act_desp or "in bed" in act_desp: + return False + elif "sleep" in act_desp or "bed" in act_desp: + if act_dura > 60: + return False + return True + + # The goal of this function is to get us the action associated with + # . As a part of this, we may need to decompose some large + # chunk actions. + # Importantly, we try to decompose at least two hours worth of schedule at + # any given point. + curr_index = role.scratch.get_f_daily_schedule_index() + curr_index_60 = role.scratch.get_f_daily_schedule_index(advance=60) + + logger.info(f"f_daily_schedule: {role.scratch.f_daily_schedule}") + # * Decompose * + # During the first hour of the day, we need to decompose two hours + # sequence. We do that here. + if curr_index == 0: + # This portion is invoked if it is the first hour of the day. + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] + if act_dura >= 60: + # We decompose if the next action is longer than an hour, and fits the + # criteria described in determine_decomp. + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index : curr_index + 1] = await TaskDecomp().run(role, act_desp, act_dura) + if curr_index_60 + 1 < len(role.scratch.f_daily_schedule): + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60 + 1] + if act_dura >= 60: + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index_60 + 1 : curr_index_60 + 2] = await TaskDecomp().run( + role, act_desp, act_dura + ) + + if curr_index_60 < len(role.scratch.f_daily_schedule): + # If it is not the first hour of the day, this is always invoked (it is + # also invoked during the first hour of the day -- to double up so we can + # decompose two hours in one go). Of course, we need to have something to + # decompose as well, so we check for that too. + if role.scratch.curr_time.hour < 23: + # And we don't want to decompose after 11 pm. + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60] + if act_dura >= 60: + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index_60 : curr_index_60 + 1] = await TaskDecomp().run( + role, act_desp, act_dura + ) + # * End of Decompose * + + # Generate an instance from the action description and duration. By + # this point, we assume that all the relevant actions are decomposed and + # ready in f_daily_schedule. + print("DEBUG LJSDLFSKJF") + for i in role.scratch.f_daily_schedule: + print(i) + print(curr_index) + print(len(role.scratch.f_daily_schedule)) + print(role.scratch.name) + print("------") + + # 1440 + x_emergency = 0 + for i in role.scratch.f_daily_schedule: + x_emergency += i[1] + # print ("x_emergency", x_emergency) + + if 1440 - x_emergency > 0: + print("x_emergency__AAA", x_emergency) + role.scratch.f_daily_schedule += [["sleeping", 1440 - x_emergency]] + + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] + + new_action_details = await GenActionDetails().run(role, act_desp, act_dura) + # Adding the action to role's queue. + role.scratch.add_new_action(**new_action_details) + + +def revise_identity(role: "STRole"): + p_name = role.scratch.name + + focal_points = [ + f"{p_name}'s plan for {role.scratch.get_str_curr_date_str()}.", + f"Important recent events for {p_name}'s life.", + ] + retrieved = new_agent_retrieve(role, focal_points) + + statements = "[Statements]\n" + for key, val in retrieved.items(): + for i in val: + statements += f"{i.created.strftime('%A %B %d -- %H:%M %p')}: {i.embedding_key}\n" + + # print (";adjhfno;asdjao;idfjo;af", p_name) + plan_prompt = statements + "\n" + plan_prompt += f"Given the statements above, is there anything that {p_name} should remember as they plan for" + plan_prompt += f" *{role.scratch.curr_time.strftime('%A %B %d')}*? " + plan_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement)\n\n" + plan_prompt += f"Write the response from {p_name}'s perspective." + plan_note = LLM().ask(plan_prompt) + # print (plan_note) + + thought_prompt = statements + "\n" + thought_prompt += ( + f"Given the statements above, how might we summarize {p_name}'s feelings about their days up to now?\n\n" + ) + thought_prompt += f"Write the response from {p_name}'s perspective." + thought_note = LLM().ask(thought_prompt) + # print (thought_note) + + currently_prompt = ( + f"{p_name}'s status from {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" + ) + currently_prompt += f"{role.scratch.currently}\n\n" + currently_prompt += f"{p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" + currently_prompt += (plan_note + thought_note).replace("\n", "") + "\n\n" + currently_prompt += f"It is now {role.scratch.curr_time.strftime('%A %B %d')}. Given the above, write {p_name}'s status for {role.scratch.curr_time.strftime('%A %B %d')} that reflects {p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}. Write this in third-person talking about {p_name}." + currently_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement).\n\n" + currently_prompt += "Follow this format below:\nStatus: " + # print ("DEBUG ;adjhfno;asdjao;asdfsidfjo;af", p_name) + # print (currently_prompt) + new_currently = LLM().ask(currently_prompt) + # print (new_currently) + # print (new_currently[10:]) + + role.scratch.currently = new_currently + + daily_req_prompt = role.scratch.get_str_iss() + "\n" + daily_req_prompt += f"Today is {role.scratch.curr_time.strftime('%A %B %d')}. Here is {role.scratch.name}'s plan today in broad-strokes (with the time of the day. e.g., have a lunch at 12:00 pm, watch TV from 7 to 8 pm).\n\n" + daily_req_prompt += "Follow this format (the list should have 4~6 items but no more):\n" + daily_req_prompt += "1. wake up and complete the morning routine at

## News -🚀 Feb. 08, 2024: [v0.7.0](https://github.com/geekan/MetaGPT/releases/tag/v0.7.0) released, supporting assigning different LLMs to different Roles. We also introduced [Interpreter](https://github.com/geekan/MetaGPT/blob/main/examples/mi/README.md), a powerful agent capable of solving a wide range of real-world problems. +🚀 March. 01, 2024: Our Data Interpreter paper is on arxiv. Find all design and benchmark details [here](https://arxiv.org/abs/2402.18679)! + +🚀 Feb. 08, 2024: [v0.7.0](https://github.com/geekan/MetaGPT/releases/tag/v0.7.0) released, supporting assigning different LLMs to different Roles. We also introduced [Data Interpreter](https://github.com/geekan/MetaGPT/blob/main/examples/di/README.md), a powerful agent capable of solving a wide range of real-world problems. 🚀 Jan. 16, 2024: Our paper [MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework ](https://arxiv.org/abs/2308.00352) accepted for oral presentation **(top 1.2%)** at ICLR 2024, **ranking #1** in the LLM-based Agent category. diff --git a/examples/di/README.md b/examples/di/README.md new file mode 100644 index 000000000..fb4e8044b --- /dev/null +++ b/examples/di/README.md @@ -0,0 +1,18 @@ +# Data Interpreter (DI) + +## What is Data Interpreter +Data Interpreter is an agent who solves problems through codes. It understands user requirements, makes plans, writes codes for execution, and uses tools if necessary. These capabilities enable it to tackle a wide range of scenarios, please check out the examples below. + +## Example List +- Data visualization +- Machine learning modeling +- Image background removal +- Solve math problems +- Receipt OCR +- Tool usage: web page imitation +- Tool usage: web crawling +- Tool usage: text2image +- Tool usage: email summarization and response +- More on the way! + +Please see [here](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/intro.html) for detailed explanation. \ No newline at end of file diff --git a/examples/mi/crawl_webpage.py b/examples/di/crawl_webpage.py similarity index 78% rename from examples/mi/crawl_webpage.py rename to examples/di/crawl_webpage.py index b5d2fb3d0..f06b85d9b 100644 --- a/examples/mi/crawl_webpage.py +++ b/examples/di/crawl_webpage.py @@ -5,15 +5,15 @@ @File : crawl_webpage.py """ -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): prompt = """Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/, and save it to a csv file. paper title must include `multiagent` or `large language model`. *notice: print key variables*""" - mi = Interpreter(use_tools=True) + di = DataInterpreter(use_tools=True) - await mi.run(prompt) + await di.run(prompt) if __name__ == "__main__": diff --git a/examples/mi/data_visualization.py b/examples/di/data_visualization.py similarity index 59% rename from examples/mi/data_visualization.py rename to examples/di/data_visualization.py index 2e4acc9b4..9af72dc42 100644 --- a/examples/mi/data_visualization.py +++ b/examples/di/data_visualization.py @@ -1,11 +1,11 @@ import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=False) - await mi.run(requirement) + di = DataInterpreter(use_tools=False) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/email_summary.py b/examples/di/email_summary.py similarity index 90% rename from examples/mi/email_summary.py rename to examples/di/email_summary.py index e1511c5b0..af081fee2 100644 --- a/examples/mi/email_summary.py +++ b/examples/di/email_summary.py @@ -6,7 +6,7 @@ """ import os -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): @@ -22,9 +22,9 @@ async def main(): Firstly, Please help me fetch the latest 5 senders and full letter contents. Then, summarize each of the 5 emails into one sentence (you can do this by yourself, no need to import other models to do this) and output them in a markdown format.""" - mi = Interpreter(use_tools=True) + di = DataInterpreter(use_tools=True) - await mi.run(prompt) + await di.run(prompt) if __name__ == "__main__": diff --git a/examples/mi/imitate_webpage.py b/examples/di/imitate_webpage.py similarity index 81% rename from examples/mi/imitate_webpage.py rename to examples/di/imitate_webpage.py index 0e9ca731d..e2c99b874 100644 --- a/examples/mi/imitate_webpage.py +++ b/examples/di/imitate_webpage.py @@ -5,7 +5,7 @@ @Author : mannaandpoem @File : imitate_webpage.py """ -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): @@ -15,9 +15,9 @@ Firstly, utilize Selenium and WebDriver for rendering. Secondly, convert image to a webpage including HTML, CSS and JS in one go. Finally, save webpage in a text file. Note: All required dependencies and environments have been fully installed and configured.""" - mi = Interpreter(use_tools=True) + di = DataInterpreter(use_tools=True) - await mi.run(prompt) + await di.run(prompt) if __name__ == "__main__": diff --git a/examples/mi/machine_learning.py b/examples/di/machine_learning.py similarity index 67% rename from examples/mi/machine_learning.py rename to examples/di/machine_learning.py index a8ab5051e..a58735831 100644 --- a/examples/mi/machine_learning.py +++ b/examples/di/machine_learning.py @@ -1,12 +1,12 @@ import fire -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(auto_run: bool = True): requirement = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy." - mi = Interpreter(auto_run=auto_run) - await mi.run(requirement) + di = DataInterpreter(auto_run=auto_run) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/ml_engineer_with_tools.py b/examples/di/ml_engineer_with_tools.py similarity index 94% rename from examples/mi/ml_engineer_with_tools.py rename to examples/di/ml_engineer_with_tools.py index 9d0e7f951..6119ad843 100644 --- a/examples/mi/ml_engineer_with_tools.py +++ b/examples/di/ml_engineer_with_tools.py @@ -1,6 +1,6 @@ import asyncio -from metagpt.roles.mi.ml_engineer import MLEngineer +from metagpt.roles.di.ml_engineer import MLEngineer async def main(requirement: str): diff --git a/examples/mi/ocr_receipt.py b/examples/di/ocr_receipt.py similarity index 81% rename from examples/mi/ocr_receipt.py rename to examples/di/ocr_receipt.py index ffa5cff05..8b48be4f1 100644 --- a/examples/mi/ocr_receipt.py +++ b/examples/di/ocr_receipt.py @@ -1,4 +1,4 @@ -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(): @@ -8,9 +8,9 @@ async def main(): requirement = f"""This is a {language} receipt image. Your goal is to perform OCR on images using PaddleOCR, then extract the total amount from ocr text results, and finally save as table. Image path: {image_path}. NOTE: The environments for Paddle and PaddleOCR are all ready and has been fully installed.""" - mi = Interpreter() + di = DataInterpreter() - await mi.run(requirement) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/rm_image_background.py b/examples/di/rm_image_background.py similarity index 75% rename from examples/mi/rm_image_background.py rename to examples/di/rm_image_background.py index 57e89b103..b74a79eeb 100644 --- a/examples/mi/rm_image_background.py +++ b/examples/di/rm_image_background.py @@ -1,11 +1,11 @@ import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=False) - await mi.run(requirement) + di = DataInterpreter(use_tools=False) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/sd_tool_usage.py b/examples/di/sd_tool_usage.py similarity index 73% rename from examples/mi/sd_tool_usage.py rename to examples/di/sd_tool_usage.py index f8507a411..69c7df5bd 100644 --- a/examples/mi/sd_tool_usage.py +++ b/examples/di/sd_tool_usage.py @@ -4,12 +4,12 @@ # @Desc : import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=True, goal=requirement) - await mi.run(requirement) + di = DataInterpreter(use_tools=True, goal=requirement) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/solve_math_problems.py b/examples/di/solve_math_problems.py similarity index 70% rename from examples/mi/solve_math_problems.py rename to examples/di/solve_math_problems.py index dce2edb00..35a69c953 100644 --- a/examples/mi/solve_math_problems.py +++ b/examples/di/solve_math_problems.py @@ -1,11 +1,11 @@ import asyncio -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter async def main(requirement: str = ""): - mi = Interpreter(use_tools=False) - await mi.run(requirement) + di = DataInterpreter(use_tools=False) + await di.run(requirement) if __name__ == "__main__": diff --git a/examples/mi/README.md b/examples/mi/README.md deleted file mode 100644 index 1734ba388..000000000 --- a/examples/mi/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# MetaGPT Interpreter (MI) - -## What is Interpreter -Interpreter is an agent who solves problems through codes. It understands user requirements, makes plans, writes codes for execution, and uses tools if necessary. These capabilities enable it to tackle a wide range of scenarios, please check out the examples below. - -## Example List -- Data visualization -- Machine learning modeling -- Image background removal -- Solve math problems -- Receipt OCR -- Tool usage: web page imitation -- Tool usage: web crawling -- Tool usage: text2image -- Tool usage: email summarization and response -- More on the way! - -Please see [here](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/mi_intro.html) for detailed explanation. \ No newline at end of file diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index 19a7c10d5..29af1dad1 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -22,9 +22,9 @@ from metagpt.actions.write_code_review import WriteCodeReview from metagpt.actions.write_prd import WritePRD from metagpt.actions.write_prd_review import WritePRDReview from metagpt.actions.write_test import WriteTest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.write_analysis_code import WriteCodeWithoutTools, WriteCodeWithTools -from metagpt.actions.mi.write_plan import WritePlan +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.write_analysis_code import WriteCodeWithoutTools, WriteCodeWithTools +from metagpt.actions.di.write_plan import WritePlan class ActionType(Enum): diff --git a/metagpt/actions/mi/__init__.py b/metagpt/actions/di/__init__.py similarity index 100% rename from metagpt/actions/mi/__init__.py rename to metagpt/actions/di/__init__.py diff --git a/metagpt/actions/mi/ask_review.py b/metagpt/actions/di/ask_review.py similarity index 100% rename from metagpt/actions/mi/ask_review.py rename to metagpt/actions/di/ask_review.py diff --git a/metagpt/actions/mi/debug_code.py b/metagpt/actions/di/debug_code.py similarity index 98% rename from metagpt/actions/mi/debug_code.py rename to metagpt/actions/di/debug_code.py index db3b7a9db..f589436f5 100644 --- a/metagpt/actions/mi/debug_code.py +++ b/metagpt/actions/di/debug_code.py @@ -1,6 +1,6 @@ from __future__ import annotations -from metagpt.actions.mi.write_analysis_code import BaseWriteAnalysisCode +from metagpt.actions.di.write_analysis_code import BaseWriteAnalysisCode from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import create_func_call_config diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/di/execute_nb_code.py similarity index 100% rename from metagpt/actions/mi/execute_nb_code.py rename to metagpt/actions/di/execute_nb_code.py diff --git a/metagpt/actions/mi/ml_action.py b/metagpt/actions/di/ml_action.py similarity index 93% rename from metagpt/actions/mi/ml_action.py rename to metagpt/actions/di/ml_action.py index 60b2fb547..d49b7b67d 100644 --- a/metagpt/actions/mi/ml_action.py +++ b/metagpt/actions/di/ml_action.py @@ -3,14 +3,14 @@ from __future__ import annotations from typing import Tuple from metagpt.actions import Action -from metagpt.actions.mi.write_analysis_code import WriteCodeWithTools -from metagpt.prompts.mi.ml_action import ( +from metagpt.actions.di.write_analysis_code import WriteCodeWithTools +from metagpt.prompts.di.ml_action import ( ML_GENERATE_CODE_PROMPT, ML_TOOL_USAGE_PROMPT, PRINT_DATA_COLUMNS, UPDATE_DATA_COLUMNS, ) -from metagpt.prompts.mi.write_analysis_code import CODE_GENERATOR_WITH_TOOLS +from metagpt.prompts.di.write_analysis_code import CODE_GENERATOR_WITH_TOOLS from metagpt.schema import Message, Plan from metagpt.utils.common import create_func_call_config, remove_comments diff --git a/metagpt/actions/mi/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py similarity index 99% rename from metagpt/actions/mi/write_analysis_code.py rename to metagpt/actions/di/write_analysis_code.py index b3d0632b6..0c4980c2b 100644 --- a/metagpt/actions/mi/write_analysis_code.py +++ b/metagpt/actions/di/write_analysis_code.py @@ -10,7 +10,7 @@ from typing import Tuple from metagpt.actions import Action from metagpt.logs import logger -from metagpt.prompts.mi.write_analysis_code import ( +from metagpt.prompts.di.write_analysis_code import ( CODE_GENERATOR_WITH_TOOLS, SELECT_FUNCTION_TOOLS, TOOL_RECOMMENDATION_PROMPT, diff --git a/metagpt/actions/mi/write_plan.py b/metagpt/actions/di/write_plan.py similarity index 98% rename from metagpt/actions/mi/write_plan.py rename to metagpt/actions/di/write_plan.py index 8067d7b87..518dfb6c6 100644 --- a/metagpt/actions/mi/write_plan.py +++ b/metagpt/actions/di/write_plan.py @@ -12,7 +12,7 @@ from typing import Tuple from metagpt.actions import Action from metagpt.logs import logger -from metagpt.prompts.mi.write_analysis_code import ( +from metagpt.prompts.di.write_analysis_code import ( ASSIGN_TASK_TYPE_CONFIG, ASSIGN_TASK_TYPE_PROMPT, ) diff --git a/metagpt/prompts/mi/__init__.py b/metagpt/prompts/di/__init__.py similarity index 100% rename from metagpt/prompts/mi/__init__.py rename to metagpt/prompts/di/__init__.py diff --git a/metagpt/prompts/mi/ml_action.py b/metagpt/prompts/di/ml_action.py similarity index 100% rename from metagpt/prompts/mi/ml_action.py rename to metagpt/prompts/di/ml_action.py diff --git a/metagpt/prompts/mi/write_analysis_code.py b/metagpt/prompts/di/write_analysis_code.py similarity index 100% rename from metagpt/prompts/mi/write_analysis_code.py rename to metagpt/prompts/di/write_analysis_code.py diff --git a/metagpt/roles/mi/__init__.py b/metagpt/roles/di/__init__.py similarity index 100% rename from metagpt/roles/mi/__init__.py rename to metagpt/roles/di/__init__.py diff --git a/metagpt/roles/mi/interpreter.py b/metagpt/roles/di/data_interpreter.py similarity index 91% rename from metagpt/roles/mi/interpreter.py rename to metagpt/roles/di/data_interpreter.py index fa50098e9..b3a2e789f 100644 --- a/metagpt/roles/mi/interpreter.py +++ b/metagpt/roles/di/data_interpreter.py @@ -2,9 +2,9 @@ from __future__ import annotations from pydantic import Field -from metagpt.actions.mi.ask_review import ReviewConst -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.write_analysis_code import ( +from metagpt.actions.di.ask_review import ReviewConst +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.write_analysis_code import ( WriteCodeWithoutTools, WriteCodeWithTools, ) @@ -13,9 +13,9 @@ from metagpt.roles import Role from metagpt.schema import Message, Task, TaskResult -class Interpreter(Role): - name: str = "Ivy" - profile: str = "Interpreter" +class DataInterpreter(Role): + name: str = "David" + profile: str = "DataInterpreter" auto_run: bool = True use_tools: bool = False execute_code: ExecuteNbCode = Field(default_factory=ExecuteNbCode, exclude=True) diff --git a/metagpt/roles/mi/ml_engineer.py b/metagpt/roles/di/ml_engineer.py similarity index 89% rename from metagpt/roles/mi/ml_engineer.py rename to metagpt/roles/di/ml_engineer.py index 78d605d3e..b33b166cf 100644 --- a/metagpt/roles/mi/ml_engineer.py +++ b/metagpt/roles/di/ml_engineer.py @@ -1,13 +1,13 @@ -from metagpt.actions.mi.debug_code import DebugCode -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.ml_action import UpdateDataColumns, WriteCodeWithToolsML +from metagpt.actions.di.debug_code import DebugCode +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.ml_action import UpdateDataColumns, WriteCodeWithToolsML from metagpt.logs import logger -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter from metagpt.tools.tool_type import ToolType from metagpt.utils.common import any_to_str -class MLEngineer(Interpreter): +class MLEngineer(DataInterpreter): name: str = "Mark" profile: str = "MLEngineer" debug_context: list = [] diff --git a/metagpt/strategy/planner.py b/metagpt/strategy/planner.py index 99d16f78b..44294be00 100644 --- a/metagpt/strategy/planner.py +++ b/metagpt/strategy/planner.py @@ -4,8 +4,8 @@ import json from pydantic import BaseModel, Field -from metagpt.actions.mi.ask_review import AskReview, ReviewConst -from metagpt.actions.mi.write_plan import ( +from metagpt.actions.di.ask_review import AskReview, ReviewConst +from metagpt.actions.di.write_plan import ( WritePlan, precheck_update_plan_from_rsp, update_plan_from_rsp, diff --git a/metagpt/strategy/solver.py b/metagpt/strategy/solver.py index ab16d82bd..e532f736b 100644 --- a/metagpt/strategy/solver.py +++ b/metagpt/strategy/solver.py @@ -49,8 +49,8 @@ class TOTSolver(BaseSolver): raise NotImplementedError -class InterpreterSolver(BaseSolver): - """InterpreterSolver: Write&Run code in the graph""" +class DataInterpreterSolver(BaseSolver): + """DataInterpreterSolver: Write&Run code in the graph""" async def solve(self): raise NotImplementedError diff --git a/setup.py b/setup.py index f5d880ac9..2044e577a 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.3", + version="0.7.4", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/metagpt/actions/mi/test_ask_review.py b/tests/metagpt/actions/di/test_ask_review.py similarity index 84% rename from tests/metagpt/actions/mi/test_ask_review.py rename to tests/metagpt/actions/di/test_ask_review.py index 92e8bd046..6bb1accf5 100644 --- a/tests/metagpt/actions/mi/test_ask_review.py +++ b/tests/metagpt/actions/di/test_ask_review.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.ask_review import AskReview +from metagpt.actions.di.ask_review import AskReview @pytest.mark.asyncio diff --git a/tests/metagpt/actions/mi/test_debug_code.py b/tests/metagpt/actions/di/test_debug_code.py similarity index 96% rename from tests/metagpt/actions/mi/test_debug_code.py rename to tests/metagpt/actions/di/test_debug_code.py index 24cfcef10..67f72ad63 100644 --- a/tests/metagpt/actions/mi/test_debug_code.py +++ b/tests/metagpt/actions/di/test_debug_code.py @@ -5,7 +5,7 @@ import pytest -from metagpt.actions.mi.debug_code import DebugCode +from metagpt.actions.di.debug_code import DebugCode from metagpt.schema import Message ErrorStr = """Tested passed: diff --git a/tests/metagpt/actions/mi/test_execute_nb_code.py b/tests/metagpt/actions/di/test_execute_nb_code.py similarity index 98% rename from tests/metagpt/actions/mi/test_execute_nb_code.py rename to tests/metagpt/actions/di/test_execute_nb_code.py index 59a814054..b491dd212 100644 --- a/tests/metagpt/actions/mi/test_execute_nb_code.py +++ b/tests/metagpt/actions/di/test_execute_nb_code.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode, truncate +from metagpt.actions.di.execute_nb_code import ExecuteNbCode, truncate @pytest.mark.asyncio diff --git a/tests/metagpt/actions/mi/test_ml_action.py b/tests/metagpt/actions/di/test_ml_action.py similarity index 95% rename from tests/metagpt/actions/mi/test_ml_action.py rename to tests/metagpt/actions/di/test_ml_action.py index 27d47b0e3..826a7fcf2 100644 --- a/tests/metagpt/actions/mi/test_ml_action.py +++ b/tests/metagpt/actions/di/test_ml_action.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.ml_action import WriteCodeWithToolsML +from metagpt.actions.di.ml_action import WriteCodeWithToolsML from metagpt.schema import Plan, Task diff --git a/tests/metagpt/actions/mi/test_write_analysis_code.py b/tests/metagpt/actions/di/test_write_analysis_code.py similarity index 99% rename from tests/metagpt/actions/mi/test_write_analysis_code.py rename to tests/metagpt/actions/di/test_write_analysis_code.py index 6c2228222..d6bbae1c6 100644 --- a/tests/metagpt/actions/mi/test_write_analysis_code.py +++ b/tests/metagpt/actions/di/test_write_analysis_code.py @@ -2,8 +2,8 @@ import asyncio import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode -from metagpt.actions.mi.write_analysis_code import ( +from metagpt.actions.di.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.write_analysis_code import ( WriteCodeWithoutTools, WriteCodeWithTools, ) diff --git a/tests/metagpt/actions/mi/test_write_plan.py b/tests/metagpt/actions/di/test_write_plan.py similarity index 95% rename from tests/metagpt/actions/mi/test_write_plan.py rename to tests/metagpt/actions/di/test_write_plan.py index 97632ea44..80b3399b8 100644 --- a/tests/metagpt/actions/mi/test_write_plan.py +++ b/tests/metagpt/actions/di/test_write_plan.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.write_plan import ( +from metagpt.actions.di.write_plan import ( Plan, Task, WritePlan, diff --git a/tests/metagpt/roles/mi/test_interpreter.py b/tests/metagpt/roles/di/test_data_interpreter.py similarity index 65% rename from tests/metagpt/roles/mi/test_interpreter.py rename to tests/metagpt/roles/di/test_data_interpreter.py index 3bae4a1ac..ba50f473b 100644 --- a/tests/metagpt/roles/mi/test_interpreter.py +++ b/tests/metagpt/roles/di/test_data_interpreter.py @@ -1,23 +1,23 @@ import pytest from metagpt.logs import logger -from metagpt.roles.mi.interpreter import Interpreter +from metagpt.roles.di.data_interpreter import DataInterpreter @pytest.mark.asyncio @pytest.mark.parametrize("auto_run", [(True), (False)]) async def test_interpreter(mocker, auto_run): - mocker.patch("metagpt.actions.mi.execute_nb_code.ExecuteNbCode.run", return_value=("a successful run", True)) + mocker.patch("metagpt.actions.di.execute_nb_code.ExecuteNbCode.run", return_value=("a successful run", True)) mocker.patch("builtins.input", return_value="confirm") requirement = "Run data analysis on sklearn Iris dataset, include a plot" tools = [] - mi = Interpreter(auto_run=auto_run, use_tools=True, tools=tools) - rsp = await mi.run(requirement) + di = DataInterpreter(auto_run=auto_run, use_tools=True, tools=tools) + rsp = await di.run(requirement) logger.info(rsp) assert len(rsp.content) > 0 - finished_tasks = mi.planner.plan.get_finished_tasks() + finished_tasks = di.planner.plan.get_finished_tasks() assert len(finished_tasks) > 0 assert len(finished_tasks[0].code) > 0 # check one task to see if code is recorded diff --git a/tests/metagpt/roles/mi/test_ml_engineer.py b/tests/metagpt/roles/di/test_ml_engineer.py similarity index 94% rename from tests/metagpt/roles/mi/test_ml_engineer.py rename to tests/metagpt/roles/di/test_ml_engineer.py index 921ac8822..08b92cd27 100644 --- a/tests/metagpt/roles/mi/test_ml_engineer.py +++ b/tests/metagpt/roles/di/test_ml_engineer.py @@ -1,11 +1,11 @@ import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.execute_nb_code import ExecuteNbCode from metagpt.logs import logger -from metagpt.roles.mi.ml_engineer import MLEngineer +from metagpt.roles.di.ml_engineer import MLEngineer from metagpt.schema import Message, Plan, Task from metagpt.tools.tool_type import ToolType -from tests.metagpt.actions.mi.test_debug_code import CODE, DebugContext, ErrorStr +from tests.metagpt.actions.di.test_debug_code import CODE, DebugContext, ErrorStr def test_mle_init(): diff --git a/tests/metagpt/utils/test_save_code.py b/tests/metagpt/utils/test_save_code.py index 9df2650f3..aceecec3b 100644 --- a/tests/metagpt/utils/test_save_code.py +++ b/tests/metagpt/utils/test_save_code.py @@ -6,7 +6,7 @@ import nbformat import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode +from metagpt.actions.di.execute_nb_code import ExecuteNbCode from metagpt.utils.common import read_json_file from metagpt.utils.save_code import DATA_PATH, save_code_file From 37221780eadaca700c2564b645e31b2c238806ef Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 4 Mar 2024 11:18:06 +0800 Subject: [PATCH 006/127] add temp_storage_path --- examples/st_game/README.md | 10 +++++-- examples/st_game/README_CN.md | 10 +++++-- .../st_game/actions/gen_action_details.py | 2 +- examples/st_game/actions/st_action.py | 2 +- examples/st_game/memory/agent_memory.py | 6 ++-- examples/st_game/plan/st_plan.py | 4 ++- examples/st_game/run_st_game.py | 28 +++++++++++++++---- examples/st_game/stanford_town.py | 5 ++-- .../tests/actions/test_gen_action_details.py | 4 ++- .../st_game/tests/plan/test_conversation.py | 3 +- examples/st_game/tests/roles/test_st_role.py | 5 ++-- examples/st_game/tests/test_reflect.py | 2 +- examples/st_game/utils/mg_ga_transform.py | 12 +++++--- 13 files changed, 66 insertions(+), 27 deletions(-) diff --git a/examples/st_game/README.md b/examples/st_game/README.md index cc4ada230..ea6547f42 100644 --- a/examples/st_game/README.md +++ b/examples/st_game/README.md @@ -1,7 +1,11 @@ ## Stanford Town Game ### Pre-Description -The path configured in `examples/st_game/utils/const.py` is the storage path of the current project. In order to facilitate GA(generative_agents)'s frontend docking data (to avoid changing its code), you can change the path under `const.py` like beflow +The path configured in `examples/st_game/utils/const.py` is the storage path of the current project. In order to facilitate GA( [generative_agents](https://github.com/joonspk-research/generative_agents) )'s frontend docking data (to avoid changing its code), you can set the value `temp_storage_path` to `temp_storage` of `generative_agents` when start `run_st_game.py`. like + +`python3 run_st_game.py --temp_storage_path path/to/ga/temp_storage xxx` + +Or change the path under `const.py` like beflow ``` STORAGE_PATH = ROOT_PATH.joinpath("storage") @@ -19,7 +23,9 @@ ### Backend service startup `idea` is the user's voice to the first Agent, and it is disseminated through this voice to see whether the final multi-agents achieve the goal of hosting or participating in the event. ### Frontend service startup -Enter `generative_agents/environment/frontend_server` and use `python manage.py runserver` to start the front-end service. +Enter project folder `generative_agents` + +Enter `environment/frontend_server` and use `python3 manage.py runserver` to start the front-end service. Visit `http://localhost:8000/simulator_home` to enter the current simulation interface. ## Appreciation diff --git a/examples/st_game/README_CN.md b/examples/st_game/README_CN.md index 9780b3ed8..4c4c2a41b 100644 --- a/examples/st_game/README_CN.md +++ b/examples/st_game/README_CN.md @@ -1,7 +1,11 @@ ## Stanford Town Game ### 前置 -`examples/st_game/utils/const.py`配置的路径为当前项目的存储路径，为了方便GA（generative_agents）的前端对接数据（避免改动它那块的代码），可将`const.py`下的 +`examples/st_game/utils/const.py`配置的路径为当前项目的存储路径，为了方便GA（ [generative_agents](https://github.com/joonspk-research/generative_agents) ）的前端对接数据（避免改动它那块的代码），可在启动`run_st_game.py`加上`temp_storage_path`指向`generative_agents`对应的`temp_storage`路径。比如 + +`python3 run_st_game.py --temp_storage_path path/to/ga/temp_storage xxx` + +或将`const.py`下的 ``` STORAGE_PATH = ROOT_PATH.joinpath("storage") @@ -18,7 +22,9 @@ ### 后端服务启动 `idea`为用户给第一个Agent的用户心声，并通过这个心声进行传播，看最后多智能体是否达到举办、参加活动的目标。 ### 前端服务启动 -进入`generative_agents/environment/frontend_server`，使用`python manage.py runserver`启动前端服务。 +进入`generative_agents`项目目录 + +进入`environment/frontend_server`，使用`python3 manage.py runserver`启动前端服务。 访问`http://localhost:8000/simulator_home` 进入当前的仿真界面。 ## Appreciation diff --git a/examples/st_game/actions/gen_action_details.py b/examples/st_game/actions/gen_action_details.py index aacc6852a..92a53087a 100644 --- a/examples/st_game/actions/gen_action_details.py +++ b/examples/st_game/actions/gen_action_details.py @@ -154,7 +154,7 @@ class GenActionArena(STAction): prompt_input = create_prompt_input(role, act_desp, act_world, act_sector) prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) self.fail_default_resp = self._func_fail_default_resp() - print('prompt ', prompt) + print("prompt ", prompt) output = await self._run_gpt35_max_tokens(prompt, max_tokens=15) logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") return output diff --git a/examples/st_game/actions/st_action.py b/examples/st_game/actions/st_action.py index 0796b5c8d..81f66c080 100644 --- a/examples/st_game/actions/st_action.py +++ b/examples/st_game/actions/st_action.py @@ -58,7 +58,7 @@ class STAction(Action): async def _aask(self, prompt: str) -> str: return await self.llm.aask(prompt) - + async def _run_gpt35_max_tokens(self, prompt: str, max_tokens: int = 50, retry: int = 3): for idx in range(retry): try: diff --git a/examples/st_game/memory/agent_memory.py b/examples/st_game/memory/agent_memory.py index a66a40efd..d4f346fe3 100644 --- a/examples/st_game/memory/agent_memory.py +++ b/examples/st_game/memory/agent_memory.py @@ -219,7 +219,7 @@ class AgentMemory(Memory): poignancy=poignancy, keywords=keywords, filling=filling, - cause_by=cause_by + cause_by=cause_by, ) keywords = [i.lower() for i in keywords] @@ -267,7 +267,7 @@ class AgentMemory(Memory): embedding_key=embedding_pair[0], poignancy=poignancy, keywords=keywords, - filling=filling + filling=filling, ) keywords = [i.lower() for i in keywords] @@ -317,7 +317,7 @@ class AgentMemory(Memory): embedding_key=embedding_pair[0], poignancy=poignancy, keywords=keywords, - filling=filling + filling=filling, ) keywords = [i.lower() for i in keywords] diff --git a/examples/st_game/plan/st_plan.py b/examples/st_game/plan/st_plan.py index 017b3e1fb..080d87de1 100644 --- a/examples/st_game/plan/st_plan.py +++ b/examples/st_game/plan/st_plan.py @@ -608,7 +608,9 @@ async def _determine_action(role: "STRole"): # We decompose if the next action is longer than an hour, and fits the # criteria described in determine_decomp. if determine_decomp(act_desp, act_dura): - role.scratch.f_daily_schedule[curr_index : curr_index + 1] = await TaskDecomp().run(role, act_desp, act_dura) + role.scratch.f_daily_schedule[curr_index : curr_index + 1] = await TaskDecomp().run( + role, act_desp, act_dura + ) if curr_index_60 + 1 < len(role.scratch.f_daily_schedule): act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60 + 1] if act_dura >= 60: diff --git a/examples/st_game/run_st_game.py b/examples/st_game/run_st_game.py index a62767f07..523cfabaa 100644 --- a/examples/st_game/run_st_game.py +++ b/examples/st_game/run_st_game.py @@ -3,6 +3,7 @@ # @Desc : entry of Stanford Town(ST/st) game import asyncio +from typing import Optional import fire @@ -18,7 +19,9 @@ from examples.st_game.utils.utils import copy_folder from metagpt.logs import logger -async def startup(idea: str, fork_sim_code: str, sim_code: str, investment: float = 30.0, n_round: int = 500): +async def startup( + idea: str, fork_sim_code: str, sim_code: str, temp_storage_path: str, investment: float = 30.0, n_round: int = 500 +): town = StanfordTown() logger.info("StanfordTown init environment") @@ -45,8 +48,8 @@ async def startup(idea: str, fork_sim_code: str, sim_code: str, investment: floa roles.append(role) # init temp_storage - write_curr_sim_code({"sim_code": sim_code}) - write_curr_step({"step": reverie_meta.get("step", 0)}) + write_curr_sim_code({"sim_code": sim_code}, temp_storage_path) + write_curr_step({"step": reverie_meta.get("step", 0)}, temp_storage_path) await town.hire(roles) @@ -56,18 +59,33 @@ async def startup(idea: str, fork_sim_code: str, sim_code: str, investment: floa await town.run(n_round) -def main(idea: str, fork_sim_code: str, sim_code: str, investment: float = 30.0, n_round: int = 500): +def main( + idea: str, + fork_sim_code: str, + sim_code: str, + temp_storage_path: Optional[str] = None, + investment: float = 30.0, + n_round: int = 500, +): """ Args: idea: idea works as an `inner voice` to the first agent. fork_sim_code: old simulation name to start with sim_code: new simulation name to save simulation result + temp_storage_path: generative_agents storage path inside `environment/frontend_server` to investment: the investment of running agents n_round: rounds to run agents """ asyncio.run( - startup(idea=idea, fork_sim_code=fork_sim_code, sim_code=sim_code, investment=investment, n_round=n_round) + startup( + idea=idea, + fork_sim_code=fork_sim_code, + sim_code=sim_code, + temp_storage_path=temp_storage_path, + investment=investment, + n_round=n_round, + ) ) diff --git a/examples/st_game/stanford_town.py b/examples/st_game/stanford_town.py index c05a1287e..799113512 100644 --- a/examples/st_game/stanford_town.py +++ b/examples/st_game/stanford_town.py @@ -6,14 +6,13 @@ from typing import Any from pydantic import Field +from examples.st_game.roles.st_role import STRole +from examples.st_game.utils.const import MAZE_ASSET_PATH from metagpt.context import Context from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv from metagpt.logs import logger from metagpt.team import Team -from examples.st_game.roles.st_role import STRole -from examples.st_game.utils.const import MAZE_ASSET_PATH - class StanfordTown(Team): env: StanfordTownEnv = Field(default=None) diff --git a/examples/st_game/tests/actions/test_gen_action_details.py b/examples/st_game/tests/actions/test_gen_action_details.py index a82478607..3edf9b116 100644 --- a/examples/st_game/tests/actions/test_gen_action_details.py +++ b/examples/st_game/tests/actions/test_gen_action_details.py @@ -31,7 +31,9 @@ async def test_gen_action_details(): act_desp = "sleeping" act_dura = "120" - access_tile = await role.rc.env.observe(EnvAPIAbstract(api_name="access_tile", kwargs={"tile": role.scratch.curr_tile})) + access_tile = await role.rc.env.observe( + EnvAPIAbstract(api_name="access_tile", kwargs={"tile": role.scratch.curr_tile}) + ) act_world = access_tile["world"] assert act_world == "the Ville" diff --git a/examples/st_game/tests/plan/test_conversation.py b/examples/st_game/tests/plan/test_conversation.py index caf2e69bd..3a158fc5f 100644 --- a/examples/st_game/tests/plan/test_conversation.py +++ b/examples/st_game/tests/plan/test_conversation.py @@ -3,11 +3,12 @@ # @Desc : unittest of roles conversation from typing import Tuple + import pytest from examples.st_game.plan.converse import agent_conversation from examples.st_game.roles.st_role import STRole -from examples.st_game.utils.const import STORAGE_PATH, MAZE_ASSET_PATH +from examples.st_game.utils.const import MAZE_ASSET_PATH, STORAGE_PATH from examples.st_game.utils.mg_ga_transform import get_reverie_meta from examples.st_game.utils.utils import copy_folder from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv diff --git a/examples/st_game/tests/roles/test_st_role.py b/examples/st_game/tests/roles/test_st_role.py index 5efc19a93..c0844d10c 100644 --- a/examples/st_game/tests/roles/test_st_role.py +++ b/examples/st_game/tests/roles/test_st_role.py @@ -13,8 +13,9 @@ from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTown @pytest.mark.asyncio async def test_observe(): role = STRole( - sim_code="base_the_ville_isabella_maria_klaus", start_time="February 13, 2023", - curr_time="February 13, 2023, 00:00:00" + sim_code="base_the_ville_isabella_maria_klaus", + start_time="February 13, 2023", + curr_time="February 13, 2023, 00:00:00", ) role.set_env(StanfordTownEnv(maze_asset_path=MAZE_ASSET_PATH)) await role.init_curr_tile() diff --git a/examples/st_game/tests/test_reflect.py b/examples/st_game/tests/test_reflect.py index 310104aee..6672b0545 100644 --- a/examples/st_game/tests/test_reflect.py +++ b/examples/st_game/tests/test_reflect.py @@ -9,8 +9,8 @@ from examples.st_game.actions.run_reflect_action import ( AgentFocusPt, AgentInsightAndGuidance, ) -from examples.st_game.utils.const import MAZE_ASSET_PATH from examples.st_game.roles.st_role import STRole +from examples.st_game.utils.const import MAZE_ASSET_PATH from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv diff --git a/examples/st_game/utils/mg_ga_transform.py b/examples/st_game/utils/mg_ga_transform.py index a9e2e1f24..9ff57a3b1 100644 --- a/examples/st_game/utils/mg_ga_transform.py +++ b/examples/st_game/utils/mg_ga_transform.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- # @Desc : data transform of mg <-> ga under storage +from pathlib import Path +from typing import Optional from examples.st_game.utils.const import STORAGE_PATH, TEMP_STORAGE_PATH from metagpt.logs import logger @@ -53,9 +55,11 @@ def get_role_environment(sim_code: str, role_name: str, step: int = 0) -> dict: return role_env -def write_curr_sim_code(curr_sim_code: dict): - write_json_file(TEMP_STORAGE_PATH.joinpath("curr_sim_code.json"), curr_sim_code) +def write_curr_sim_code(curr_sim_code: dict, temp_storage_path: Optional[Path] = None): + temp_storage_path = Path(temp_storage_path) or TEMP_STORAGE_PATH + write_json_file(temp_storage_path.joinpath("curr_sim_code.json"), curr_sim_code) -def write_curr_step(curr_step: dict): - write_json_file(TEMP_STORAGE_PATH.joinpath("curr_step.json"), curr_step) +def write_curr_step(curr_step: dict, temp_storage_path: Optional[Path] = None): + temp_storage_path = Path(temp_storage_path) or TEMP_STORAGE_PATH + write_json_file(temp_storage_path.joinpath("curr_step.json"), curr_step) From e238a6f658a95595838b30c2925aebd77a2d2377 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 4 Mar 2024 13:46:46 +0800 Subject: [PATCH 007/127] update --- examples/st_game/README.md | 2 ++ examples/st_game/README_CN.md | 4 +++- examples/st_game/memory/agent_memory.py | 4 ++++ examples/st_game/roles/st_role.py | 19 +++++++++++++------ examples/st_game/stanford_town.py | 7 +++---- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/examples/st_game/README.md b/examples/st_game/README.md index ea6547f42..28c4efec4 100644 --- a/examples/st_game/README.md +++ b/examples/st_game/README.md @@ -19,6 +19,8 @@ # updated ### Backend service startup The execution entry is `python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` +or +`python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10 --temp_storage_path path/to/ga/temp_storage` `idea` is the user's voice to the first Agent, and it is disseminated through this voice to see whether the final multi-agents achieve the goal of hosting or participating in the event. diff --git a/examples/st_game/README_CN.md b/examples/st_game/README_CN.md index 4c4c2a41b..4e1b3f439 100644 --- a/examples/st_game/README_CN.md +++ b/examples/st_game/README_CN.md @@ -17,7 +17,9 @@ # 更新为 这样可用实现不改变GA代码情况下，实现仿真数据的对接。不然得修改GA的代码来适配MG的输出路径。 ### 后端服务启动 -执行入口为：`python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` +执行入口为：`python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` +或者 +`python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10 --temp_storage_path path/to/ga/temp_storage` `idea`为用户给第一个Agent的用户心声，并通过这个心声进行传播，看最后多智能体是否达到举办、参加活动的目标。 diff --git a/examples/st_game/memory/agent_memory.py b/examples/st_game/memory/agent_memory.py index d4f346fe3..d212232f4 100644 --- a/examples/st_game/memory/agent_memory.py +++ b/examples/st_game/memory/agent_memory.py @@ -39,6 +39,8 @@ class BasicMemory(Message): keywords: list[str] = Field(default=[]) # keywords filling: list = Field(default=[]) # 装的与之相关联的memory_id的列表 + __hash__ = object.__hash__ # support hash in AgentMemory + @model_validator(mode="before") @classmethod def check_values(cls, values): @@ -46,6 +48,8 @@ class BasicMemory(Message): values["last_accessed"] = values["created"] if "content" in values: values["description"] = values["content"] + if "filling" in values: + values["filling"] = values["filling"] or [] return values @field_serializer("created", "expiration") diff --git a/examples/st_game/roles/st_role.py b/examples/st_game/roles/st_role.py index 2a9722a8f..af4188733 100644 --- a/examples/st_game/roles/st_role.py +++ b/examples/st_game/roles/st_role.py @@ -40,6 +40,7 @@ from metagpt.environment.api.env_api import EnvAPIAbstract from metagpt.logs import logger from metagpt.roles.role import Role, RoleContext from metagpt.schema import Message +from metagpt.utils.common import any_to_str if TYPE_CHECKING: from metagpt.environment.stanford_town_env.stanford_town_env import ( # noqa: F401 @@ -171,13 +172,19 @@ class STRole(Role): logger.info(f"Role: {self.name} saved role's memory into {str(self.role_storage_path)}") - async def _observe(self) -> int: + async def _observe(self, ignore_memory=False) -> int: if not self.rc.env: return 0 + news = [] + if not news: + news = self.rc.msg_buffer.pop_all() + old_messages = [] if ignore_memory else self.rc.memory.get() + # Filter out messages of interest. + self.rc.news = [ + n for n in news if (n.cause_by in self.rc.watch or self.name in n.send_to) and n not in old_messages + ] - observed = self.rc.env.memory.get_by_actions(self.rc.watch) - self.rc.news = self.rc.memory.remember(observed) - if len(self.rc.news) == 1 and self.rc.news[0].cause_by == UserRequirement: + if len(self.rc.news) == 1 and self.rc.news[0].cause_by == any_to_str(UserRequirement): logger.warning(f"Role: {self.name} add inner voice: {self.rc.news[0].content}") await self.add_inner_voice(self.rc.news[0].content) @@ -318,7 +325,7 @@ class STRole(Role): # Get event poignancy. event_poignancy = await generate_poig_score(self, "event", desc_embedding_in) - logger.info(f"Role {self.name} event_poignancy: {event_poignancy}") + logger.debug(f"Role {self.name} event_poignancy: {event_poignancy}") # If we observe the persona's self chat, we include that in the memory # of the persona here. @@ -579,7 +586,7 @@ class STRole(Role): async def _react(self) -> Message: # update role env - ret = self.update_role_env() + ret = await self.update_role_env() if not ret: # TODO add message logger.info(f"Role: {self.name} update_role_env return False") diff --git a/examples/st_game/stanford_town.py b/examples/st_game/stanford_town.py index 799113512..446b60ce0 100644 --- a/examples/st_game/stanford_town.py +++ b/examples/st_game/stanford_town.py @@ -2,9 +2,8 @@ # -*- coding: utf-8 -*- # @Desc : StanfordTown to works like SoftwareCompany -from typing import Any +from typing import Any, Optional -from pydantic import Field from examples.st_game.roles.st_role import STRole from examples.st_game.utils.const import MAZE_ASSET_PATH @@ -15,10 +14,10 @@ from metagpt.team import Team class StanfordTown(Team): - env: StanfordTownEnv = Field(default=None) + env: Optional[StanfordTownEnv] = None def __init__(self, context: Context = None, **data: Any): - super(StanfordTown, self).__init__(**data) + super(Team, self).__init__(**data) ctx = context or Context() if not self.env: self.env = StanfordTownEnv(context=ctx, maze_asset_path=MAZE_ASSET_PATH) From 729094173a0bcc310091246467d5f991fcbd42ac Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 4 Mar 2024 13:47:13 +0800 Subject: [PATCH 008/127] update --- examples/st_game/stanford_town.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/st_game/stanford_town.py b/examples/st_game/stanford_town.py index 446b60ce0..f10332b60 100644 --- a/examples/st_game/stanford_town.py +++ b/examples/st_game/stanford_town.py @@ -4,7 +4,6 @@ from typing import Any, Optional - from examples.st_game.roles.st_role import STRole from examples.st_game.utils.const import MAZE_ASSET_PATH from metagpt.context import Context From cf476cd3225f3ae9e5bd5ee20ab004d138b4b521 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 4 Mar 2024 14:07:41 +0800 Subject: [PATCH 009/127] update field definition of scratch --- examples/st_game/actions/dummy_action.py | 3 --- examples/st_game/memory/scratch.py | 10 +++++----- examples/st_game/run_st_game.py | 1 + 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/examples/st_game/actions/dummy_action.py b/examples/st_game/actions/dummy_action.py index ba87d8516..a5004d5ef 100644 --- a/examples/st_game/actions/dummy_action.py +++ b/examples/st_game/actions/dummy_action.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- # @Desc : dummy action to make every STRole can deal DummyMessage which is caused by DummyAction -from dataclasses import dataclass - from metagpt.actions import Action from metagpt.schema import Message @@ -13,7 +11,6 @@ class DummyAction(Action): raise NotImplementedError -@dataclass class DummyMessage(Message): """ dummy message to pass to role and make them to have a execution every round diff --git a/examples/st_game/memory/scratch.py b/examples/st_game/memory/scratch.py index b7040ce08..1c30f1829 100644 --- a/examples/st_game/memory/scratch.py +++ b/examples/st_game/memory/scratch.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta from pathlib import Path -from typing import Optional +from typing import Optional, Union from pydantic import ( BaseModel, @@ -25,7 +25,7 @@ class Scratch(BaseModel): # 类别2:世界信息 curr_time: Optional[datetime] = Field(default=None) - curr_tile: Optional[str] = Field(default=None) + curr_tile: Optional[list[int]] = Field(default=None) daily_plan_req: Optional[str] = Field(default=None) # 类别3:人物角色的核心身份 @@ -59,8 +59,8 @@ class Scratch(BaseModel): # 类别6:个人计划 daily_req: list[str] = Field(default=[]) - f_daily_schedule: list[list[str]] = Field(default=[]) - f_daily_schedule_hourly_org: list[list[str]] = Field(default=[]) + f_daily_schedule: list[list[Union[int, str]]] = Field(default=[]) + f_daily_schedule_hourly_org: list[list[Union[int, str]]] = Field(default=[]) # 类别7:当前动作 act_address: Optional[str] = Field(default=None) @@ -90,7 +90,7 @@ class Scratch(BaseModel): values["act_obj_event"] = (values["name"], None, None) return values - @field_validator("curr_time", "act_start_time", "chatting_end_time") + @field_validator("curr_time", "act_start_time", "chatting_end_time", mode="before") @classmethod def check_time_filed(cls, time_filed): val = datetime.strptime(time_filed, "%B %d, %Y, %H:%M:%S") if time_filed else None diff --git a/examples/st_game/run_st_game.py b/examples/st_game/run_st_game.py index 523cfabaa..09ae8f0ec 100644 --- a/examples/st_game/run_st_game.py +++ b/examples/st_game/run_st_game.py @@ -24,6 +24,7 @@ async def startup( ): town = StanfordTown() logger.info("StanfordTown init environment") + print(f"{fork_sim_code}\t{sim_code}{temp_storage_path}\t{investment}\t{n_round}") # copy `storage/{fork_sim_code}` to `storage/{sim_code}` copy_folder(str(STORAGE_PATH.joinpath(fork_sim_code)), str(STORAGE_PATH.joinpath(sim_code))) From fa721dc95f219f26e581681cc1dd5425c48958b9 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 4 Mar 2024 14:35:04 +0800 Subject: [PATCH 010/127] rm act_event validation --- examples/st_game/README.md | 2 ++ examples/st_game/README_CN.md | 4 +++- examples/st_game/memory/scratch.py | 27 +++++++++------------------ examples/st_game/roles/st_role.py | 2 +- examples/st_game/run_st_game.py | 5 ++--- 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/examples/st_game/README.md b/examples/st_game/README.md index 28c4efec4..114f7f672 100644 --- a/examples/st_game/README.md +++ b/examples/st_game/README.md @@ -17,6 +17,8 @@ # updated This can be used to achieve docking of simulation data without changing the GA code. Otherwise, the GA code must be modified to adapt to the MG output path. +If you don't want to start from 0, copy other simulation directories under `generative_agents/environment/frontend_server/storage/` to `examples/st_game/storage`, and select a directory named `fork_sim_code`. + ### Backend service startup The execution entry is `python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` or diff --git a/examples/st_game/README_CN.md b/examples/st_game/README_CN.md index 4e1b3f439..84470f5c2 100644 --- a/examples/st_game/README_CN.md +++ b/examples/st_game/README_CN.md @@ -14,7 +14,9 @@ # 更新为 STORAGE_PATH = Path("{path/to/ga/storage}") TEMP_STORAGE_PATH = Path("{path/to/ga/temp_storage}") ``` -这样可用实现不改变GA代码情况下，实现仿真数据的对接。不然得修改GA的代码来适配MG的输出路径。 +这样可用实现不改变GA代码情况下，实现仿真数据的对接。不然得修改GA的代码来适配MG的输出路径。 + +如果你不想从0开始启动，拷贝`generative_agents/environment/frontend_server/storage/`下的其他仿真目录到`examples/st_game/storage`，并选择一个目录名作为`fork_sim_code`。 ### 后端服务启动 执行入口为：`python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` diff --git a/examples/st_game/memory/scratch.py b/examples/st_game/memory/scratch.py index 1c30f1829..6fc687901 100644 --- a/examples/st_game/memory/scratch.py +++ b/examples/st_game/memory/scratch.py @@ -11,7 +11,6 @@ from pydantic import ( Field, field_serializer, field_validator, - model_validator, ) from metagpt.utils.common import read_json_file, write_json_file @@ -68,11 +67,11 @@ class Scratch(BaseModel): act_duration: Optional[int] = Field(default=None) act_description: Optional[str] = Field(default=None) act_pronunciatio: Optional[str] = Field(default=None) - act_event: tuple[str, Optional[str], Optional[str]] = (None, None, None) + act_event: list[Optional[str]] = [None, None, None] act_obj_description: Optional[str] = Field(default=None) act_obj_pronunciatio: Optional[str] = Field(default=None) - act_obj_event: tuple[Optional[str], Optional[str], Optional[str]] = (None, None, None) + act_obj_event: list[Optional[str]] = [None, None, None] chatting_with: Optional[str] = Field(default=None) chat: Optional[str] = Field(default=None) @@ -80,15 +79,7 @@ class Scratch(BaseModel): chatting_end_time: Optional[datetime] = Field(default=None) act_path_set: bool = False - planned_path: list[str] = Field(default=[]) - - @model_validator(mode="after") - @classmethod - def check_values(cls, values): - if "name" in values: - values["act_event"] = (values["name"], None, None) - values["act_obj_event"] = (values["name"], None, None) - return values + planned_path: list[list[int]] = Field(default=[]) @field_validator("curr_time", "act_start_time", "chatting_end_time", mode="before") @classmethod @@ -103,7 +94,7 @@ class Scratch(BaseModel): return time_filed @classmethod - def set_scratch_path(cls, f_saved: Path): + def init_scratch_from_path(cls, f_saved: Path): scratch_load = read_json_file(f_saved) scratch = Scratch(**scratch_load) return scratch @@ -253,21 +244,21 @@ class Scratch(BaseModel): def get_curr_event(self): if not self.act_address: - return (self.name, None, None) + return self.name, None, None else: return self.act_event def get_curr_event_and_desc(self): if not self.act_address: - return (self.name, None, None, None) + return self.name, None, None, None else: - return (self.act_event[0], self.act_event[1], self.act_event[2], self.act_description) + return self.act_event[0], self.act_event[1], self.act_event[2], self.act_description def get_curr_obj_event_and_desc(self): if not self.act_address: - return ("", None, None, None) + return "", None, None, None else: - return (self.act_address, self.act_obj_event[1], self.act_obj_event[2], self.act_obj_description) + return self.act_address, self.act_obj_event[1], self.act_obj_event[2], self.act_obj_description def add_new_action( self, diff --git a/examples/st_game/roles/st_role.py b/examples/st_game/roles/st_role.py index af4188733..d5dd994f9 100644 --- a/examples/st_game/roles/st_role.py +++ b/examples/st_game/roles/st_role.py @@ -153,7 +153,7 @@ class STRole(Role): self.rc.spatial_memory.set_mem_path(f_saved=sp_mem_saved) scratch_f_saved = self.role_storage_path.joinpath("bootstrap_memory/scratch.json") - self.rc.scratch = Scratch.set_scratch_path(f_saved=scratch_f_saved) + self.rc.scratch = Scratch.init_scratch_from_path(f_saved=scratch_f_saved) logger.info(f"Role: {self.name} loaded role's memory from {str(self.role_storage_path)}") diff --git a/examples/st_game/run_st_game.py b/examples/st_game/run_st_game.py index 09ae8f0ec..304bf24ba 100644 --- a/examples/st_game/run_st_game.py +++ b/examples/st_game/run_st_game.py @@ -24,7 +24,6 @@ async def startup( ): town = StanfordTown() logger.info("StanfordTown init environment") - print(f"{fork_sim_code}\t{sim_code}{temp_storage_path}\t{investment}\t{n_round}") # copy `storage/{fork_sim_code}` to `storage/{sim_code}` copy_folder(str(STORAGE_PATH.joinpath(fork_sim_code)), str(STORAGE_PATH.joinpath(sim_code))) @@ -71,9 +70,9 @@ def main( """ Args: idea: idea works as an `inner voice` to the first agent. - fork_sim_code: old simulation name to start with + fork_sim_code: old simulation name to start with, choose one inside `generative_agents/environment/frontend_server/storage/` sim_code: new simulation name to save simulation result - temp_storage_path: generative_agents storage path inside `environment/frontend_server` to + temp_storage_path: generative_agents temp_storage path inside `environment/frontend_server` to interact. investment: the investment of running agents n_round: rounds to run agents """ From fac1d35223ccf1732ab74dbdff0fb2c9f45c5e8c Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 4 Mar 2024 14:36:47 +0800 Subject: [PATCH 011/127] update --- examples/st_game/memory/scratch.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/examples/st_game/memory/scratch.py b/examples/st_game/memory/scratch.py index 6fc687901..b4036f839 100644 --- a/examples/st_game/memory/scratch.py +++ b/examples/st_game/memory/scratch.py @@ -6,12 +6,7 @@ from datetime import datetime, timedelta from pathlib import Path from typing import Optional, Union -from pydantic import ( - BaseModel, - Field, - field_serializer, - field_validator, -) +from pydantic import BaseModel, Field, field_serializer, field_validator from metagpt.utils.common import read_json_file, write_json_file From 97bf135da6d78eed0a740beb8a87ca6a6fadc58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Mar 2024 12:20:58 +0800 Subject: [PATCH 012/127] feat: gemini + proxy --- metagpt/provider/google_gemini_api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 09e554205..8f8f5f707 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # @Desc : Google Gemini LLM from https://ai.google.dev/tutorials/python_quickstart +import os from typing import Optional, Union import google.generativeai as genai @@ -15,7 +16,7 @@ from google.generativeai.types.generation_types import ( ) from metagpt.configs.llm_config import LLMConfig, LLMType -from metagpt.logs import log_llm_stream +from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -51,6 +52,10 @@ class GeminiLLM(BaseLLM): self.llm = GeminiGenerativeModel(model_name=self.model) def __init_gemini(self, config: LLMConfig): + if config.proxy: + logger.info(f"Use proxy: {config.proxy}") + os.environ["HTTP_PROXY"] = config.proxy + os.environ["HTTP_PROXYS"] = config.proxy genai.configure(api_key=config.api_key) def _user_msg(self, msg: str, images: Optional[Union[str, list[str]]] = None) -> dict[str, str]: From f6260ec084443d9ae96dd6aec02afc595c709ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Mar 2024 12:27:28 +0800 Subject: [PATCH 013/127] feat: gemini + proxy --- metagpt/provider/google_gemini_api.py | 6 +++++- setup.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 2647ab16b..bdbf7acd6 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # @Desc : Google Gemini LLM from https://ai.google.dev/tutorials/python_quickstart - +import os from typing import Optional, Union import google.generativeai as genai @@ -58,6 +58,10 @@ class GeminiLLM(BaseLLM): self.llm = GeminiGenerativeModel(model_name=self.model) def __init_gemini(self, config: LLMConfig): + if config.proxy: + logger.info(f"Use proxy: {config.proxy}") + os.environ["HTTP_PROXY"] = config.proxy + os.environ["HTTP_PROXYS"] = config.proxy genai.configure(api_key=config.api_key) def _user_msg(self, msg: str, images: Optional[Union[str, list[str]]] = None) -> dict[str, str]: diff --git a/setup.py b/setup.py index 2044e577a..072237e8c 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.4", + version="0.7.5", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", From 0867dad4d7b66ad098934dd9d39b3cdb261c0c00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Mar 2024 17:04:44 +0800 Subject: [PATCH 014/127] fixbug: gbk UnicodeEncodeError --- metagpt/learn/skill_loader.py | 5 ++--- metagpt/utils/common.py | 16 +++++++++++---- metagpt/utils/dependency_file.py | 7 ++----- metagpt/utils/file_repository.py | 7 ++----- metagpt/utils/mermaid.py | 8 ++------ setup.py | 2 +- .../metagpt/roles/test_tutorial_assistant.py | 7 +++---- tests/metagpt/utils/test_common.py | 20 ++++++++++++++----- tests/metagpt/utils/test_git_repository.py | 5 ++--- tests/metagpt/utils/test_s3.py | 8 ++------ 10 files changed, 43 insertions(+), 42 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index bcf28bb87..e98f73cf9 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -9,11 +9,11 @@ from pathlib import Path from typing import Dict, List, Optional -import aiofiles import yaml from pydantic import BaseModel, Field from metagpt.context import Context +from metagpt.utils.common import aread class Example(BaseModel): @@ -68,8 +68,7 @@ class SkillsDeclaration(BaseModel): async def load(skill_yaml_file_name: Path = None) -> "SkillsDeclaration": if not skill_yaml_file_name: skill_yaml_file_name = Path(__file__).parent.parent.parent / "docs/.well-known/skills.yaml" - async with aiofiles.open(str(skill_yaml_file_name), mode="r") as reader: - data = await reader.read(-1) + data = await aread(filename=skill_yaml_file_name) skill_data = yaml.safe_load(data) return SkillsDeclaration(**skill_data) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 015902c3d..aba75fbec 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -29,6 +29,7 @@ from pathlib import Path from typing import Any, Callable, List, Tuple, Union import aiofiles +import chardet import loguru import requests from PIL import Image @@ -587,14 +588,21 @@ def role_raise_decorator(func): @handle_exception -async def aread(filename: str | Path, encoding=None) -> str: +async def aread(filename: str | Path, encoding="utf-8") -> str: """Read file asynchronously.""" - async with aiofiles.open(str(filename), mode="r", encoding=encoding) as reader: - content = await reader.read() + try: + async with aiofiles.open(str(filename), mode="r", encoding=encoding) as reader: + content = await reader.read() + except UnicodeDecodeError: + async with aiofiles.open(str(filename), mode="rb") as reader: + raw = await reader.read() + result = chardet.detect(raw) + detected_encoding = result["encoding"] + content = raw.decode(detected_encoding) return content -async def awrite(filename: str | Path, data: str, encoding=None): +async def awrite(filename: str | Path, data: str, encoding="utf-8"): """Write file asynchronously.""" pathname = Path(filename) pathname.parent.mkdir(parents=True, exist_ok=True) diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py index d3add1171..0a375051c 100644 --- a/metagpt/utils/dependency_file.py +++ b/metagpt/utils/dependency_file.py @@ -13,9 +13,7 @@ import re from pathlib import Path from typing import Set -import aiofiles - -from metagpt.utils.common import aread +from metagpt.utils.common import aread, awrite from metagpt.utils.exceptions import handle_exception @@ -45,8 +43,7 @@ class DependencyFile: async def save(self): """Save dependencies to the file asynchronously.""" data = json.dumps(self._dependencies) - async with aiofiles.open(str(self._filename), mode="w") as writer: - await writer.write(data) + await awrite(filename=self._filename, data=data) async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True): """Update dependencies for a file asynchronously. diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index d2a06963a..d19f2b705 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -14,11 +14,9 @@ from datetime import datetime from pathlib import Path from typing import Dict, List, Set -import aiofiles - from metagpt.logs import logger from metagpt.schema import Document -from metagpt.utils.common import aread +from metagpt.utils.common import aread, awrite from metagpt.utils.json_to_markdown import json_to_markdown @@ -55,8 +53,7 @@ class FileRepository: pathname = self.workdir / filename pathname.parent.mkdir(parents=True, exist_ok=True) content = content if content else "" # avoid `argument must be str, not None` to make it continue - async with aiofiles.open(str(pathname), mode="w") as writer: - await writer.write(content) + await awrite(filename=str(pathname), data=content) logger.info(f"save to: {str(pathname)}") if dependencies is not None: diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index ae3c5118f..e1d140e84 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -9,11 +9,9 @@ import asyncio import os from pathlib import Path -import aiofiles - from metagpt.config2 import config from metagpt.logs import logger -from metagpt.utils.common import check_cmd_exists +from metagpt.utils.common import awrite, check_cmd_exists async def mermaid_to_file(engine, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: @@ -30,9 +28,7 @@ async def mermaid_to_file(engine, mermaid_code, output_file_without_suffix, widt if dir_name and not os.path.exists(dir_name): os.makedirs(dir_name) tmp = Path(f"{output_file_without_suffix}.mmd") - async with aiofiles.open(tmp, "w", encoding="utf-8") as f: - await f.write(mermaid_code) - # tmp.write_text(mermaid_code, encoding="utf-8") + await awrite(filename=tmp, data=mermaid_code) if engine == "nodejs": if check_cmd_exists(config.mermaid.path) != 0: diff --git a/setup.py b/setup.py index 072237e8c..813d2410c 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pr setup( name="metagpt", - version="0.7.5", + version="0.7.6", description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/metagpt/roles/test_tutorial_assistant.py b/tests/metagpt/roles/test_tutorial_assistant.py index c12c2b26e..732f346fd 100644 --- a/tests/metagpt/roles/test_tutorial_assistant.py +++ b/tests/metagpt/roles/test_tutorial_assistant.py @@ -6,11 +6,11 @@ @File : test_tutorial_assistant.py """ -import aiofiles import pytest from metagpt.const import TUTORIAL_PATH from metagpt.roles.tutorial_assistant import TutorialAssistant +from metagpt.utils.common import aread @pytest.mark.asyncio @@ -20,9 +20,8 @@ async def test_tutorial_assistant(language: str, topic: str, context): msg = await role.run(topic) assert TUTORIAL_PATH.exists() filename = msg.content - async with aiofiles.open(filename, mode="r", encoding="utf-8") as reader: - content = await reader.read() - assert "pip" in content + content = await aread(filename=filename) + assert "pip" in content if __name__ == "__main__": diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index 9b1fa878e..7c59b8072 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -13,7 +13,6 @@ import uuid from pathlib import Path from typing import Any, Set -import aiofiles import pytest from pydantic import BaseModel @@ -125,9 +124,7 @@ class TestGetProjectRoot: async def test_parse_data_exception(self, filename, want): pathname = Path(__file__).parent.parent.parent / "data/output_parser" / filename assert pathname.exists() - async with aiofiles.open(str(pathname), mode="r") as reader: - data = await reader.read() - + data = await aread(filename=pathname) result = OutputParser.parse_data(data=data) assert want in result @@ -198,12 +195,25 @@ class TestGetProjectRoot: @pytest.mark.asyncio async def test_read_write(self): - pathname = Path(__file__).parent / uuid.uuid4().hex / "test.tmp" + pathname = Path(__file__).parent / f"../../../workspace/unittest/{uuid.uuid4().hex}" / "test.tmp" await awrite(pathname, "ABC") data = await aread(pathname) assert data == "ABC" pathname.unlink(missing_ok=True) + @pytest.mark.asyncio + async def test_read_write_error_charset(self): + pathname = Path(__file__).parent / f"../../../workspace/unittest/{uuid.uuid4().hex}" / "test.txt" + content = "中国abc123\u27f6" + await awrite(filename=pathname, data=content) + data = await aread(filename=pathname) + assert data == content + + content = "GB18030 是中国国家标准局发布的新一代中文字符集标准，是 GBK 的升级版，支持更广泛的字符范围。" + await awrite(filename=pathname, data=content, encoding="gb2312") + data = await aread(filename=pathname, encoding="utf-8") + assert data == content + if __name__ == "__main__": pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index ea28b8f0b..480a22e24 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -10,15 +10,14 @@ import shutil from pathlib import Path -import aiofiles import pytest +from metagpt.utils.common import awrite from metagpt.utils.git_repository import GitRepository async def mock_file(filename, content=""): - async with aiofiles.open(str(filename), mode="w") as file: - await file.write(content) + await awrite(filename=filename, data=content) async def mock_repo(local_path) -> (GitRepository, Path): diff --git a/tests/metagpt/utils/test_s3.py b/tests/metagpt/utils/test_s3.py index b26ebe94d..c1a85f4ff 100644 --- a/tests/metagpt/utils/test_s3.py +++ b/tests/metagpt/utils/test_s3.py @@ -9,7 +9,6 @@ import uuid from pathlib import Path import aioboto3 -import aiofiles import pytest from metagpt.config2 import Config @@ -37,7 +36,7 @@ async def test_s3(mocker): conn = S3(s3) object_name = "unittest.bak" await conn.upload_file(bucket=s3.bucket, local_path=__file__, object_name=object_name) - pathname = (Path(__file__).parent / uuid.uuid4().hex).with_suffix(".bak") + pathname = (Path(__file__).parent / "../../../workspace/unittest" / uuid.uuid4().hex).with_suffix(".bak") pathname.unlink(missing_ok=True) await conn.download_file(bucket=s3.bucket, object_name=object_name, local_path=str(pathname)) assert pathname.exists() @@ -45,8 +44,7 @@ async def test_s3(mocker): assert url bin_data = await conn.get_object(bucket=s3.bucket, object_name=object_name) assert bin_data - async with aiofiles.open(__file__, mode="r", encoding="utf-8") as reader: - data = await reader.read() + data = await aread(filename=__file__) res = await conn.cache(data, ".bak", "script") assert "http" in res @@ -60,8 +58,6 @@ async def test_s3(mocker): except Exception: pass - await reader.close() - if __name__ == "__main__": pytest.main([__file__, "-s"]) From e39cafdd580ebcf1b587b6de40a51b5deae3fef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 11 Mar 2024 22:25:38 +0800 Subject: [PATCH 015/127] feat: + tree command --- metagpt/utils/tree.py | 99 ++++++++++++++++++++++++++++++++ tests/metagpt/utils/test_tree.py | 54 +++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 metagpt/utils/tree.py create mode 100644 tests/metagpt/utils/test_tree.py diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py new file mode 100644 index 000000000..49b5634c6 --- /dev/null +++ b/metagpt/utils/tree.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/3/11 +@Author : mashenquan +@File : tree.py +@Desc : Implement the same functionality as the `tree` command. +Example: + root + +-- dir1 + | +-- file1.txt + | +-- file2.txt + +-- dir2 + | +-- subdir1 + | | +-- file1.txt + | | +-- file2.txt + | +-- subdir2 + | +-- file1.txt + | +-- file2.txt + +-- file.txt +""" +from __future__ import annotations + +from pathlib import Path +from typing import Callable, Dict, List + +from anthropic import BaseModel +from pydantic import Field + + +class Tree(BaseModel): + """ + Represents a directory tree structure. + + Attributes: + root (str): The root directory of the tree. + tree (Dict[str, Dict]): The tree structure as a dictionary. + + Methods: + print: Print the directory tree structure. + + """ + + root: str + tree: Dict[str, Dict] = Field(default_factory=dict) + + def print(self, git_ignore_rules: Callable = None) -> str: + """ + Recursively traverses the directory structure and prints it out in a tree-like format. + + Args: + git_ignore_rules (Callable): Optional. A function to filter files to ignore. + + Returns: + str: A string representation of the directory tree. + + """ + root = Path(self.root).resolve() + self.tree[root.name] = self._list_children(root=root, git_ignore_rules=git_ignore_rules) + v = self._print_tree(self.tree) + return "\n".join(v) + + @staticmethod + def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: + tree = {} + for i in root.iterdir(): + if git_ignore_rules and git_ignore_rules(str(i)): + continue + if i.is_file(): + tree[i.name] = {} + else: + tree[i.name] = Tree._list_children(root=i, git_ignore_rules=git_ignore_rules) + return tree + + @staticmethod + def _print_tree(tree: Dict[str:Dict], indent: int = 0) -> List[str]: + ret = [] + for name, children in tree.items(): + ret.append(name) + if not children: + continue + lines = Tree._print_tree(tree=children, indent=indent + 1) + for j, v in enumerate(lines): + if v[0] not in ["+", " ", "|"]: + ret = Tree._add_line(ret) + row = f"+-- {v}" + else: + row = f" {v}" + ret.append(row) + return ret + + @staticmethod + def _add_line(rows: List[str]) -> List[str]: + for i in range(len(rows) - 1, -1, -1): + v = rows[i] + if v[0] != " ": + return rows + rows[i] = "|" + v[1:] + return rows diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py new file mode 100644 index 000000000..0d48f7ce3 --- /dev/null +++ b/tests/metagpt/utils/test_tree.py @@ -0,0 +1,54 @@ +from pathlib import Path +from typing import List + +import pytest +from gitignore_parser import parse_gitignore + +from metagpt.utils.tree import Tree + + +@pytest.mark.parametrize( + ("root", "rules"), + [ + (str(Path(__file__).parent / "../.."), None), + (str(Path(__file__).parent / "../.."), str(Path(__file__).parent / "../../../.gitignore")), + ], +) +def test_tree(root: str, rules: str): + gitignore_rules = parse_gitignore(full_path=rules) if rules else None + tree = Tree(root=root).print(git_ignore_rules=gitignore_rules) + assert tree + + +@pytest.mark.parametrize( + ("tree", "want"), + [ + ({"a": {"b": {}, "c": {}}}, ["a", "+-- b", "+-- c"]), + ({"a": {"b": {}, "c": {"d": {}}}}, ["a", "+-- b", "+-- c", " +-- d"]), + ( + {"a": {"b": {"e": {"f": {}, "g": {}}}, "c": {"d": {}}}}, + ["a", "+-- b", "| +-- e", "| +-- f", "| +-- g", "+-- c", " +-- d"], + ), + ( + {"h": {"a": {"b": {"e": {"f": {}, "g": {}}}, "c": {"d": {}}}, "i": {}}}, + [ + "h", + "+-- a", + "| +-- b", + "| | +-- e", + "| | +-- f", + "| | +-- g", + "| +-- c", + "| +-- d", + "+-- i", + ], + ), + ], +) +def test__print_tree(tree: dict, want: List[str]): + v = Tree._print_tree(tree) + assert v == want + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 7d32f9efe6958bf9b76b4309d426c590e3268e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 12 Mar 2024 11:10:09 +0800 Subject: [PATCH 016/127] refactor: Replace Tree class with tree() --- metagpt/utils/tree.py | 218 ++++++++++++++++++++----------- tests/metagpt/utils/test_tree.py | 8 +- 2 files changed, 149 insertions(+), 77 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index 49b5634c6..ad3373f5f 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -6,94 +6,166 @@ @File : tree.py @Desc : Implement the same functionality as the `tree` command. Example: - root - +-- dir1 - | +-- file1.txt - | +-- file2.txt - +-- dir2 - | +-- subdir1 - | | +-- file1.txt - | | +-- file2.txt - | +-- subdir2 - | +-- file1.txt - | +-- file2.txt - +-- file.txt + Usage: + >>> print_tree(".") + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- dependency_file.py + +-- index.html + +-- make_sk_kernel.py + +-- token_counter.py + +-- embedding.py + +-- repair_llm_raw_output.py + +-- mermaid.py + +-- parse_html.py + +-- visual_graph_repo.py + +-- special_tokens.py + +-- ahttp_client.py + +-- __init__.py + +-- mmdc_ink.py + +-- di_graph_repository.py + +-- yaml_model.py + +-- cost_manager.py + +-- __pycache__ + | +-- __init__.cpython-39.pyc + | +-- redis.cpython-39.pyc + | +-- singleton.cpython-39.pyc + | +-- mmdc_ink.cpython-39.pyc + | +-- read_document.cpython-39.pyc + | +-- mermaid.cpython-39.pyc + | +-- parse_html.cpython-39.pyc + | +-- human_interaction.cpython-39.pyc + | +-- cost_manager.cpython-39.pyc + | +-- json_to_markdown.cpython-39.pyc + | +-- graph_repository.cpython-39.pyc + | +-- ahttp_client.cpython-39.pyc + | +-- visual_graph_repo.cpython-39.pyc + | +-- file.cpython-39.pyc + | +-- di_graph_repository.cpython-39.pyc + | +-- pycst.cpython-39.pyc + | +-- save_code.cpython-39.pyc + | +-- dependency_file.cpython-39.pyc + | +-- text.cpython-39.pyc + | +-- token_counter.cpython-39.pyc + | +-- project_repo.cpython-39.pyc + | +-- yaml_model.cpython-39.pyc + | +-- serialize.cpython-39.pyc + | +-- git_repository.cpython-39.pyc + | +-- custom_decoder.cpython-39.pyc + | +-- parse_docstring.cpython-39.pyc + | +-- common.cpython-39.pyc + | +-- exceptions.cpython-39.pyc + | +-- repair_llm_raw_output.cpython-39.pyc + | +-- s3.cpython-39.pyc + | +-- embedding.cpython-39.pyc + | +-- make_sk_kernel.cpython-39.pyc + | +-- file_repository.cpython-39.pyc + +-- file.py + +-- save_code.py + +-- common.py + +-- redis.py + +-- text.py + +-- graph_repository.py + +-- singleton.py + +-- recovery_util.py + +-- file_repository.py + +-- pycst.py + +-- exceptions.py + +-- human_interaction.py + +-- highlight.py + +-- mmdc_pyppeteer.py + +-- s3.py + +-- json_to_markdown.py + +-- custom_decoder.py + +-- git_repository.py + +-- read_document.py + +-- parse_docstring.py """ from __future__ import annotations from pathlib import Path from typing import Callable, Dict, List -from anthropic import BaseModel -from pydantic import Field - -class Tree(BaseModel): +def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: """ - Represents a directory tree structure. + Recursively traverses the directory structure and prints it out in a tree-like format. - Attributes: - root (str): The root directory of the tree. - tree (Dict[str, Dict]): The tree structure as a dictionary. + Args: + root (str or Path): The root directory from which to start traversing. + git_ignore_rules (Callable): Optional. A function to filter files to ignore. - Methods: - print: Print the directory tree structure. + Returns: + str: A string representation of the directory tree. + + Example: + >>> tree(".") + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- __pycache__ + | +-- __init__.cpython-39.pyc + | +-- redis.cpython-39.pyc + | +-- singleton.cpython-39.pyc + +-- parse_docstring.py + + >>> from gitignore_parser import parse_gitignore + >>> tree(".", git_ignore_rules=parse_gitignore(full_path="../../.gitignore")) + utils + +-- serialize.py + +-- project_repo.py + +-- tree.py + +-- mmdc_playwright.py + +-- parse_docstring.py """ + root = Path(root).resolve() + dir_ = {root.name: _list_children(root=root, git_ignore_rules=git_ignore_rules)} + v = _print_tree(dir_) + return "\n".join(v) - root: str - tree: Dict[str, Dict] = Field(default_factory=dict) - def print(self, git_ignore_rules: Callable = None) -> str: - """ - Recursively traverses the directory structure and prints it out in a tree-like format. - - Args: - git_ignore_rules (Callable): Optional. A function to filter files to ignore. - - Returns: - str: A string representation of the directory tree. - - """ - root = Path(self.root).resolve() - self.tree[root.name] = self._list_children(root=root, git_ignore_rules=git_ignore_rules) - v = self._print_tree(self.tree) - return "\n".join(v) - - @staticmethod - def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: - tree = {} - for i in root.iterdir(): - if git_ignore_rules and git_ignore_rules(str(i)): - continue +def _list_children(root: Path, git_ignore_rules: Callable) -> Dict[str, Dict]: + dir_ = {} + for i in root.iterdir(): + if git_ignore_rules and git_ignore_rules(str(i)): + continue + try: if i.is_file(): - tree[i.name] = {} + dir_[i.name] = {} else: - tree[i.name] = Tree._list_children(root=i, git_ignore_rules=git_ignore_rules) - return tree + dir_[i.name] = _list_children(root=i, git_ignore_rules=git_ignore_rules) + except (FileNotFoundError, PermissionError, OSError): + dir_[i.name] = {} + return dir_ - @staticmethod - def _print_tree(tree: Dict[str:Dict], indent: int = 0) -> List[str]: - ret = [] - for name, children in tree.items(): - ret.append(name) - if not children: - continue - lines = Tree._print_tree(tree=children, indent=indent + 1) - for j, v in enumerate(lines): - if v[0] not in ["+", " ", "|"]: - ret = Tree._add_line(ret) - row = f"+-- {v}" - else: - row = f" {v}" - ret.append(row) - return ret - @staticmethod - def _add_line(rows: List[str]) -> List[str]: - for i in range(len(rows) - 1, -1, -1): - v = rows[i] - if v[0] != " ": - return rows - rows[i] = "|" + v[1:] - return rows +def _print_tree(dir_: Dict[str:Dict]) -> List[str]: + ret = [] + for name, children in dir_.items(): + ret.append(name) + if not children: + continue + lines = _print_tree(children) + for j, v in enumerate(lines): + if v[0] not in ["+", " ", "|"]: + ret = _add_line(ret) + row = f"+-- {v}" + else: + row = f" {v}" + ret.append(row) + return ret + + +def _add_line(rows: List[str]) -> List[str]: + for i in range(len(rows) - 1, -1, -1): + v = rows[i] + if v[0] != " ": + return rows + rows[i] = "|" + v[1:] + return rows diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py index 0d48f7ce3..34eae10cf 100644 --- a/tests/metagpt/utils/test_tree.py +++ b/tests/metagpt/utils/test_tree.py @@ -4,7 +4,7 @@ from typing import List import pytest from gitignore_parser import parse_gitignore -from metagpt.utils.tree import Tree +from metagpt.utils.tree import _print_tree, tree @pytest.mark.parametrize( @@ -16,8 +16,8 @@ from metagpt.utils.tree import Tree ) def test_tree(root: str, rules: str): gitignore_rules = parse_gitignore(full_path=rules) if rules else None - tree = Tree(root=root).print(git_ignore_rules=gitignore_rules) - assert tree + v = tree(root=root, git_ignore_rules=gitignore_rules) + assert v @pytest.mark.parametrize( @@ -46,7 +46,7 @@ def test_tree(root: str, rules: str): ], ) def test__print_tree(tree: dict, want: List[str]): - v = Tree._print_tree(tree) + v = _print_tree(tree) assert v == want From 6a8699cd4a8f0f2f6f3a8095c67bc72abe775b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 12 Mar 2024 11:14:35 +0800 Subject: [PATCH 017/127] refactor: Replace Tree class with tree() --- metagpt/utils/tree.py | 61 +------------------------------------------ 1 file changed, 1 insertion(+), 60 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index ad3373f5f..1c0060842 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -5,61 +5,18 @@ @Author : mashenquan @File : tree.py @Desc : Implement the same functionality as the `tree` command. -Example: - Usage: + Example: >>> print_tree(".") utils +-- serialize.py +-- project_repo.py +-- tree.py +-- mmdc_playwright.py - +-- dependency_file.py - +-- index.html - +-- make_sk_kernel.py - +-- token_counter.py - +-- embedding.py - +-- repair_llm_raw_output.py - +-- mermaid.py - +-- parse_html.py - +-- visual_graph_repo.py - +-- special_tokens.py - +-- ahttp_client.py - +-- __init__.py - +-- mmdc_ink.py - +-- di_graph_repository.py - +-- yaml_model.py +-- cost_manager.py +-- __pycache__ | +-- __init__.cpython-39.pyc | +-- redis.cpython-39.pyc | +-- singleton.cpython-39.pyc - | +-- mmdc_ink.cpython-39.pyc - | +-- read_document.cpython-39.pyc - | +-- mermaid.cpython-39.pyc - | +-- parse_html.cpython-39.pyc - | +-- human_interaction.cpython-39.pyc - | +-- cost_manager.cpython-39.pyc - | +-- json_to_markdown.cpython-39.pyc - | +-- graph_repository.cpython-39.pyc - | +-- ahttp_client.cpython-39.pyc - | +-- visual_graph_repo.cpython-39.pyc - | +-- file.cpython-39.pyc - | +-- di_graph_repository.cpython-39.pyc - | +-- pycst.cpython-39.pyc - | +-- save_code.cpython-39.pyc - | +-- dependency_file.cpython-39.pyc - | +-- text.cpython-39.pyc - | +-- token_counter.cpython-39.pyc - | +-- project_repo.cpython-39.pyc - | +-- yaml_model.cpython-39.pyc - | +-- serialize.cpython-39.pyc - | +-- git_repository.cpython-39.pyc - | +-- custom_decoder.cpython-39.pyc - | +-- parse_docstring.cpython-39.pyc - | +-- common.cpython-39.pyc - | +-- exceptions.cpython-39.pyc - | +-- repair_llm_raw_output.cpython-39.pyc - | +-- s3.cpython-39.pyc | +-- embedding.cpython-39.pyc | +-- make_sk_kernel.cpython-39.pyc | +-- file_repository.cpython-39.pyc @@ -67,22 +24,6 @@ Example: +-- save_code.py +-- common.py +-- redis.py - +-- text.py - +-- graph_repository.py - +-- singleton.py - +-- recovery_util.py - +-- file_repository.py - +-- pycst.py - +-- exceptions.py - +-- human_interaction.py - +-- highlight.py - +-- mmdc_pyppeteer.py - +-- s3.py - +-- json_to_markdown.py - +-- custom_decoder.py - +-- git_repository.py - +-- read_document.py - +-- parse_docstring.py """ from __future__ import annotations From 684730e94f7c5fe7f5bac8a31ed8fac1937b6d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 13 Mar 2024 15:06:40 +0800 Subject: [PATCH 018/127] feat: +`tree` command --- metagpt/utils/tree.py | 36 ++++++++++++++++++++++++++++---- tests/metagpt/utils/test_tree.py | 16 +++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index 1c0060842..c0386d822 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -27,17 +27,22 @@ """ from __future__ import annotations +import subprocess from pathlib import Path from typing import Callable, Dict, List +from gitignore_parser import parse_gitignore -def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: + +def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = True) -> str: """ Recursively traverses the directory structure and prints it out in a tree-like format. Args: root (str or Path): The root directory from which to start traversing. - git_ignore_rules (Callable): Optional. A function to filter files to ignore. + gitignore (str or Path): The filename of gitignore file. + run_command (bool): Whether to execute `tree` command. Execute the `tree` command and return the result if True, + otherwise execute python code instead. Returns: str: A string representation of the directory tree. @@ -55,8 +60,7 @@ def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: | +-- singleton.cpython-39.pyc +-- parse_docstring.py - >>> from gitignore_parser import parse_gitignore - >>> tree(".", git_ignore_rules=parse_gitignore(full_path="../../.gitignore")) + >>> tree(".", gitignore="../../.gitignore") utils +-- serialize.py +-- project_repo.py @@ -64,8 +68,21 @@ def tree(root: str | Path, git_ignore_rules: Callable = None) -> str: +-- mmdc_playwright.py +-- parse_docstring.py + >>> tree(".", gitignore="../../.gitignore", run_command=True) + utils + ├── serialize.py + ├── project_repo.py + ├── tree.py + ├── mmdc_playwright.py + └── parse_docstring.py + + """ root = Path(root).resolve() + if run_command: + return _execute_tree(root, gitignore) + + git_ignore_rules = parse_gitignore(gitignore) if gitignore else None dir_ = {root.name: _list_children(root=root, git_ignore_rules=git_ignore_rules)} v = _print_tree(dir_) return "\n".join(v) @@ -110,3 +127,14 @@ def _add_line(rows: List[str]) -> List[str]: return rows rows[i] = "|" + v[1:] return rows + + +def _execute_tree(root: Path, gitignore: str | Path) -> str: + args = ["--gitignore", str(gitignore)] if gitignore else [] + try: + result = subprocess.run(["tree"] + args + [str(root)], capture_output=True, text=True, check=True) + if result.returncode != 0: + raise ValueError(f"tree exits with code {result.returncode}") + return result.stdout + except subprocess.CalledProcessError as e: + raise e diff --git a/tests/metagpt/utils/test_tree.py b/tests/metagpt/utils/test_tree.py index 34eae10cf..03a2a5606 100644 --- a/tests/metagpt/utils/test_tree.py +++ b/tests/metagpt/utils/test_tree.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import List import pytest -from gitignore_parser import parse_gitignore from metagpt.utils.tree import _print_tree, tree @@ -15,8 +14,19 @@ from metagpt.utils.tree import _print_tree, tree ], ) def test_tree(root: str, rules: str): - gitignore_rules = parse_gitignore(full_path=rules) if rules else None - v = tree(root=root, git_ignore_rules=gitignore_rules) + v = tree(root=root, gitignore=rules) + assert v + + +@pytest.mark.parametrize( + ("root", "rules"), + [ + (str(Path(__file__).parent / "../.."), None), + (str(Path(__file__).parent / "../.."), str(Path(__file__).parent / "../../../.gitignore")), + ], +) +def test_tree_command(root: str, rules: str): + v = tree(root=root, gitignore=rules, run_command=True) assert v From 6f4d30825f84d90c35dab04e639fc1b483e1a823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 13 Mar 2024 15:10:13 +0800 Subject: [PATCH 019/127] feat: +`tree` command --- metagpt/utils/tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index c0386d822..fbf085e48 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -34,7 +34,7 @@ from typing import Callable, Dict, List from gitignore_parser import parse_gitignore -def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = True) -> str: +def tree(root: str | Path, gitignore: str | Path = None, run_command: bool = False) -> str: """ Recursively traverses the directory structure and prints it out in a tree-like format. From 32a86bdd4e1b4fda8010e261b435f899e3bf067f Mon Sep 17 00:00:00 2001 From: yzlin Date: Tue, 19 Mar 2024 20:26:36 +0800 Subject: [PATCH 020/127] update readme --- README.md | 60 +++++++++++++++++-------------------------------------- 1 file changed, 18 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 81b6a876f..027b3b243 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ # MetaGPT: The Multi-Agent Framework

## News -🚀 Mar. 14, 2024: Our Data Interpreter paper is on [arxiv](https://arxiv.org/abs/2402.18679). Check the [example](https://docs.deepwisdom.ai/main/en/DataInterpreter/) and [code](https://github.com/geekan/MetaGPT/tree/main/examples/di)! +🚀 Mar. 14, 2024: Our **Data Interpreter** paper is on [arxiv](https://arxiv.org/abs/2402.18679). Check the [example](https://docs.deepwisdom.ai/main/en/DataInterpreter/) and [code](https://github.com/geekan/MetaGPT/tree/main/examples/di)! 🚀 Feb. 08, 2024: [v0.7.0](https://github.com/geekan/MetaGPT/releases/tag/v0.7.0) released, supporting assigning different LLMs to different Roles. We also introduced [Data Interpreter](https://github.com/geekan/MetaGPT/blob/main/examples/di/README.md), a powerful agent capable of solving a wide range of real-world problems. @@ -55,9 +55,9 @@ ## Software Company as Multi-Agent System

Software Company Multi-Agent Schematic (Gradually Implementing)

-## Install +## Get Started -### Pip installation +### Installation > Ensure that Python 3.9+ is installed on your system. You can check this by using: `python --version`. > You can use conda like this: `conda create -n metagpt python=3.9 && conda activate metagpt` @@ -68,6 +68,9 @@ # https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html metagpt --init-config # it will create ~/.metagpt/config2.yaml, just modify it to your needs ``` +For detailed installation guidance, please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version) + or [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker) + ### Configuration You can configure `~/.metagpt/config2.yaml` according to the [example](https://github.com/geekan/MetaGPT/blob/main/config/config2.example.yaml) and [doc](https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html): @@ -82,13 +85,13 @@ ### Configuration ### Usage -After installation, you can use it as CLI +After installation, you can use MetaGPT at CLI ```bash metagpt "Create a 2048 game" # this will create a repo in ./workspace ``` -or you can use it as library +or use it as library ```python from metagpt.software_company import generate_repo, ProjectRepo @@ -96,47 +99,19 @@ ### Usage print(repo) # it will print the repo structure with files ``` -detail installation please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version) - or [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker) +You can also use its [Data Interpreter](https://github.com/geekan/MetaGPT/tree/main/examples/di) -### Docker installation -
⏬ Step 1: Download metagpt image and prepare config2.yaml :: click to expand :: -
+```python +import asyncio +from metagpt.roles.di.data_interpreter import DataInterpreter -```bash -docker pull metagpt/metagpt:latest -mkdir -p /opt/metagpt/{config,workspace} -docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml -vim /opt/metagpt/config/config2.yaml # Change the config +async def main(): + di = DataInterpreter() + await di.run("Run data analysis on sklearn Iris dataset, include a plot") + +asyncio.run(main()) # or await main() in a jupyter notebook setting ``` -
-
- -
⏬ Step 2: Run metagpt container :: click to expand :: -
- -```bash -docker run --name metagpt -d \ - --privileged \ - -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \ - -v /opt/metagpt/workspace:/app/metagpt/workspace \ - metagpt/metagpt:latest -``` - -
-
- -
⏬ Step 3: Use metagpt :: click to expand :: -
- -```bash -docker exec -it metagpt /bin/bash -$ metagpt "Create a 2048 game" # this will create a repo in ./workspace -``` - -
-
### QuickStart & Demo Video - Try it on [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT) @@ -156,6 +131,7 @@ ## Tutorial - 🧑‍💻 Contribution - [Develop Roadmap](docs/ROADMAP.md) - 🔖 Use Cases + - [Data Interpreter](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/interpreter/intro.html) - [Debate](https://docs.deepwisdom.ai/main/en/guide/use_cases/multi_agent/debate.html) - [Researcher](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/researcher.html) - [Recepit Assistant](https://docs.deepwisdom.ai/main/en/guide/use_cases/agent/receipt_assistant.html) From f91334deafc38f33b05deb43f1a0dc6102b7c8d1 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Tue, 19 Mar 2024 23:31:05 +0800 Subject: [PATCH 021/127] fix text ut error --- tests/metagpt/utils/test_text.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/metagpt/utils/test_text.py b/tests/metagpt/utils/test_text.py index c9a9753be..319a4ebe0 100644 --- a/tests/metagpt/utils/test_text.py +++ b/tests/metagpt/utils/test_text.py @@ -22,7 +22,7 @@ def _paragraphs(n): @pytest.mark.parametrize( "msgs, model_name, system_text, reserved, expected", [ - (_msgs(), "gpt-3.5-turbo", "System", 1500, 1), + (_msgs(), "gpt-3.5-turbo-0613", "System", 1500, 1), (_msgs(), "gpt-3.5-turbo-16k", "System", 3000, 6), (_msgs(), "gpt-3.5-turbo-16k", "Hello," * 1000, 3000, 5), (_msgs(), "gpt-4", "System", 2000, 3), @@ -32,22 +32,23 @@ def _paragraphs(n): ], ) def test_reduce_message_length(msgs, model_name, system_text, reserved, expected): - assert len(reduce_message_length(msgs, model_name, system_text, reserved)) / (len("Hello,")) / 1000 == expected + length = len(reduce_message_length(msgs, model_name, system_text, reserved)) / (len("Hello,")) / 1000 + assert length == expected @pytest.mark.parametrize( "text, prompt_template, model_name, system_text, reserved, expected", [ - (" ".join("Hello World." for _ in range(1000)), "Prompt: {}", "gpt-3.5-turbo", "System", 1500, 2), + (" ".join("Hello World." for _ in range(1000)), "Prompt: {}", "gpt-3.5-turbo-0613", "System", 1500, 2), (" ".join("Hello World." for _ in range(1000)), "Prompt: {}", "gpt-3.5-turbo-16k", "System", 3000, 1), (" ".join("Hello World." for _ in range(4000)), "Prompt: {}", "gpt-4", "System", 2000, 2), (" ".join("Hello World." for _ in range(8000)), "Prompt: {}", "gpt-4-32k", "System", 4000, 1), - (" ".join("Hello World" for _ in range(8000)), "Prompt: {}", "gpt-3.5-turbo", "System", 1000, 8), + (" ".join("Hello World" for _ in range(8000)), "Prompt: {}", "gpt-3.5-turbo-0613", "System", 1000, 8), ], ) def test_generate_prompt_chunk(text, prompt_template, model_name, system_text, reserved, expected): - ret = list(generate_prompt_chunk(text, prompt_template, model_name, system_text, reserved)) - assert len(ret) == expected + chunk = len(list(generate_prompt_chunk(text, prompt_template, model_name, system_text, reserved))) + assert chunk == expected @pytest.mark.parametrize( From 21c5d1955bf552612d0184e684ceb39235972067 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 20 Mar 2024 14:27:11 +0800 Subject: [PATCH 022/127] Update process for "Fix bug" --- metagpt/roles/engineer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 329b21553..9d8f6884f 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -240,8 +240,8 @@ class Engineer(Role): async def _think(self) -> Action | None: if not self.src_workspace: self.src_workspace = self.git_repo.workdir / self.git_repo.workdir.name - write_plan_and_change_filters = any_to_str_set([WriteTasks]) - write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode, FixBug]) + write_plan_and_change_filters = any_to_str_set([WriteTasks, FixBug]) + write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode]) summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) if not self.rc.news: return None From f6a11d508904e6a56a9e35895abfeb439f5c4110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 20 Mar 2024 17:34:30 +0800 Subject: [PATCH 023/127] fixbug: #1016 --- metagpt/actions/di/write_analysis_code.py | 4 +-- metagpt/provider/base_llm.py | 22 +++++++++++++++++ metagpt/provider/google_gemini_api.py | 30 +++++++++++++++++++++++ metagpt/provider/openai_api.py | 9 ++----- metagpt/utils/common.py | 23 ----------------- tests/mock/mock_llm.py | 3 +-- 6 files changed, 57 insertions(+), 34 deletions(-) diff --git a/metagpt/actions/di/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py index 185926e31..711e56d39 100644 --- a/metagpt/actions/di/write_analysis_code.py +++ b/metagpt/actions/di/write_analysis_code.py @@ -18,7 +18,7 @@ from metagpt.prompts.di.write_analysis_code import ( STRUCTUAL_PROMPT, ) from metagpt.schema import Message, Plan -from metagpt.utils.common import CodeParser, process_message, remove_comments +from metagpt.utils.common import CodeParser, remove_comments class WriteAnalysisCode(Action): @@ -50,7 +50,7 @@ class WriteAnalysisCode(Action): ) working_memory = working_memory or [] - context = process_message([Message(content=structual_prompt, role="user")] + working_memory) + context = self.llm.format_msg([Message(content=structual_prompt, role="user")] + working_memory) # LLM call if use_reflection: diff --git a/metagpt/provider/base_llm.py b/metagpt/provider/base_llm.py index 71308930a..601980d5e 100644 --- a/metagpt/provider/base_llm.py +++ b/metagpt/provider/base_llm.py @@ -73,6 +73,28 @@ class BaseLLM(ABC): def _system_msg(self, msg: str) -> dict[str, str]: return {"role": "system", "content": msg} + def format_msg(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: + """convert messages to list[dict].""" + from metagpt.schema import Message + + if not isinstance(messages, list): + messages = [messages] + + processed_messages = [] + for msg in messages: + if isinstance(msg, str): + processed_messages.append({"role": "user", "content": msg}) + elif isinstance(msg, dict): + assert set(msg.keys()) == set(["role", "content"]) + processed_messages.append(msg) + elif isinstance(msg, Message): + processed_messages.append(msg.to_dict()) + else: + raise ValueError( + f"Only support message type are: str, Message, dict, but got {type(messages).__name__}!" + ) + return processed_messages + def _system_msgs(self, msgs: list[str]) -> list[dict[str, str]]: return [self._system_msg(msg) for msg in msgs] diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 09e554205..7370747a5 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -18,6 +18,7 @@ from metagpt.configs.llm_config import LLMConfig, LLMType from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider +from metagpt.schema import Message class GeminiGenerativeModel(GenerativeModel): @@ -61,6 +62,35 @@ class GeminiLLM(BaseLLM): def _assistant_msg(self, msg: str) -> dict[str, str]: return {"role": "model", "parts": [msg]} + def _system_msg(self, msg: str) -> dict[str, str]: + return {"role": "user", "parts": [msg]} + + def format_msg(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: + """convert messages to list[dict].""" + from metagpt.schema import Message + + if not isinstance(messages, list): + messages = [messages] + + # REF: https://ai.google.dev/tutorials/python_quickstart + # As a dictionary, the message requires `role` and `parts` keys. + # The role in a conversation can either be the `user`, which provides the prompts, + # or `model`, which provides the responses. + processed_messages = [] + for msg in messages: + if isinstance(msg, str): + processed_messages.append({"role": "user", "parts": [msg]}) + elif isinstance(msg, dict): + assert set(msg.keys()) == set(["role", "parts"]) + processed_messages.append(msg) + elif isinstance(msg, Message): + processed_messages.append({"role": "user" if msg.role == "user" else "model", "parts": [msg.content]}) + else: + raise ValueError( + f"Only support message type are: str, Message, dict, but got {type(messages).__name__}!" + ) + return processed_messages + def _const_kwargs(self, messages: list[dict], stream: bool = False) -> dict: kwargs = {"contents": messages, "generation_config": GenerationConfig(temperature=0.3), "stream": stream} return kwargs diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index b4f99e69f..2fb64dc85 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -29,12 +29,7 @@ from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_llm import BaseLLM from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA from metagpt.provider.llm_provider_registry import register_provider -from metagpt.utils.common import ( - CodeParser, - decode_image, - log_and_reraise, - process_message, -) +from metagpt.utils.common import CodeParser, decode_image, log_and_reraise from metagpt.utils.cost_manager import CostManager from metagpt.utils.exceptions import handle_exception from metagpt.utils.token_counter import ( @@ -150,7 +145,7 @@ class OpenAILLM(BaseLLM): async def _achat_completion_function( self, messages: list[dict], timeout: int = 3, **chat_configs ) -> ChatCompletion: - messages = process_message(messages) + messages = self.format_msg(messages) kwargs = self._cons_kwargs(messages=messages, timeout=timeout, **chat_configs) rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs) self._update_costs(rsp.usage) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index e9cef69a4..7493712c2 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -802,29 +802,6 @@ def decode_image(img_url_or_b64: str) -> Image: return img -def process_message(messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: - """convert messages to list[dict].""" - from metagpt.schema import Message - - # 全部转成list - if not isinstance(messages, list): - messages = [messages] - - # 转成list[dict] - processed_messages = [] - for msg in messages: - if isinstance(msg, str): - processed_messages.append({"role": "user", "content": msg}) - elif isinstance(msg, dict): - assert set(msg.keys()) == set(["role", "content"]) - processed_messages.append(msg) - elif isinstance(msg, Message): - processed_messages.append(msg.to_dict()) - else: - raise ValueError(f"Only support message type are: str, Message, dict, but got {type(messages).__name__}!") - return processed_messages - - def log_and_reraise(retry_state: RetryCallState): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") logger.warning( diff --git a/tests/mock/mock_llm.py b/tests/mock/mock_llm.py index b4cdfa0cf..c4262e080 100644 --- a/tests/mock/mock_llm.py +++ b/tests/mock/mock_llm.py @@ -8,7 +8,6 @@ from metagpt.provider.azure_openai_api import AzureOpenAILLM from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA from metagpt.provider.openai_api import OpenAILLM from metagpt.schema import Message -from metagpt.utils.common import process_message OriginalLLM = OpenAILLM if config.llm.api_type == LLMType.OPENAI else AzureOpenAILLM @@ -105,7 +104,7 @@ class MockLLM(OriginalLLM): return rsp async def aask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: - msg_key = json.dumps(process_message(messages), ensure_ascii=False) + msg_key = json.dumps(self.format_msg(messages), ensure_ascii=False) rsp = await self._mock_rsp(msg_key, self.original_aask_code, messages, **kwargs) return rsp From af3a409ac4b7b1632512384ff86d46349a746145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 20 Mar 2024 21:24:41 +0800 Subject: [PATCH 024/127] fixbug: llm.timeout not working --- metagpt/actions/action_node.py | 6 +++--- metagpt/configs/llm_config.py | 2 +- metagpt/provider/anthropic_api.py | 8 ++++---- metagpt/provider/base_llm.py | 26 +++++++++++++++----------- metagpt/provider/dashscope_api.py | 8 ++++---- metagpt/provider/general_api_base.py | 2 +- metagpt/provider/google_gemini_api.py | 8 ++++---- metagpt/provider/human_provider.py | 14 +++++++------- metagpt/provider/ollama_api.py | 13 ++++++------- metagpt/provider/openai_api.py | 26 +++++++++++++------------- metagpt/provider/qianfan_api.py | 8 ++++---- metagpt/provider/spark_api.py | 8 ++++---- metagpt/provider/zhipuai_api.py | 10 +++++----- requirements.txt | 2 +- 14 files changed, 72 insertions(+), 69 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 09da4a988..63925a052 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -416,7 +416,7 @@ class ActionNode: images: Optional[Union[str, list[str]]] = None, system_msgs: Optional[list[str]] = None, schema="markdown", # compatible to original format - timeout=3, + timeout=0, ) -> (str, BaseModel): """Use ActionOutput to wrap the output of aask""" content = await self.llm.aask(prompt, system_msgs, images=images, timeout=timeout) @@ -448,7 +448,7 @@ class ActionNode: def set_context(self, context): self.set_recursive("context", context) - async def simple_fill(self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=3, exclude=None): + async def simple_fill(self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=0, exclude=None): prompt = self.compile(context=self.context, schema=schema, mode=mode, exclude=exclude) if schema != "raw": @@ -473,7 +473,7 @@ class ActionNode: mode="auto", strgy="simple", images: Optional[Union[str, list[str]]] = None, - timeout=3, + timeout=0, exclude=[], ): """Fill the node(s) with mode. diff --git a/metagpt/configs/llm_config.py b/metagpt/configs/llm_config.py index fa9bc0b1b..92b8e1512 100644 --- a/metagpt/configs/llm_config.py +++ b/metagpt/configs/llm_config.py @@ -74,7 +74,7 @@ class LLMConfig(YamlModel): stream: bool = False logprobs: Optional[bool] = None # https://cookbook.openai.com/examples/using_logprobs top_logprobs: Optional[int] = None - timeout: int = 60 + timeout: int = 600 # For Network proxy: Optional[str] = None diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 872f9b2c7..3125ffc22 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -41,15 +41,15 @@ class AnthropicLLM(BaseLLM): def get_choice_text(self, resp: Message) -> str: return resp.content[0].text - async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> Message: + async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> Message: resp: Message = await self.aclient.messages.create(**self._const_kwargs(messages)) self._update_costs(resp.usage, self.model) return resp - async def acompletion(self, messages: list[dict], timeout: int = 3) -> Message: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout: int = 0) -> Message: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: stream = await self.aclient.messages.create(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = Usage(input_tokens=0, output_tokens=0) diff --git a/metagpt/provider/base_llm.py b/metagpt/provider/base_llm.py index 71308930a..fa5119c67 100644 --- a/metagpt/provider/base_llm.py +++ b/metagpt/provider/base_llm.py @@ -23,6 +23,7 @@ from tenacity import ( ) from metagpt.configs.llm_config import LLMConfig +from metagpt.const import LLM_API_TIMEOUT from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import log_and_reraise @@ -108,7 +109,7 @@ class BaseLLM(ABC): system_msgs: Optional[list[str]] = None, format_msgs: Optional[list[dict[str, str]]] = None, images: Optional[Union[str, list[str]]] = None, - timeout=3, + timeout=0, stream=True, ) -> str: if system_msgs: @@ -124,31 +125,31 @@ class BaseLLM(ABC): else: message.extend(msg) logger.debug(message) - rsp = await self.acompletion_text(message, stream=stream, timeout=timeout) + rsp = await self.acompletion_text(message, stream=stream, timeout=self.get_timeout(timeout)) return rsp def _extract_assistant_rsp(self, context): return "\n".join([i["content"] for i in context if i["role"] == "assistant"]) - async def aask_batch(self, msgs: list, timeout=3) -> str: + async def aask_batch(self, msgs: list, timeout=0) -> str: """Sequential questioning""" context = [] for msg in msgs: umsg = self._user_msg(msg) context.append(umsg) - rsp_text = await self.acompletion_text(context, timeout=timeout) + rsp_text = await self.acompletion_text(context, timeout=self.get_timeout(timeout)) context.append(self._assistant_msg(rsp_text)) return self._extract_assistant_rsp(context) - async def aask_code(self, messages: Union[str, Message, list[dict]], timeout=3, **kwargs) -> dict: + async def aask_code(self, messages: Union[str, Message, list[dict]], timeout=0, **kwargs) -> dict: raise NotImplementedError @abstractmethod - async def _achat_completion(self, messages: list[dict], timeout=3): + async def _achat_completion(self, messages: list[dict], timeout=0): """_achat_completion implemented by inherited class""" @abstractmethod - async def acompletion(self, messages: list[dict], timeout=3): + async def acompletion(self, messages: list[dict], timeout=0): """Asynchronous version of completion All GPTAPIs are required to provide the standard OpenAI completion interface [ @@ -159,7 +160,7 @@ class BaseLLM(ABC): """ @abstractmethod - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: """_achat_completion_stream implemented by inherited class""" @retry( @@ -169,11 +170,11 @@ class BaseLLM(ABC): retry=retry_if_exception_type(ConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream: bool = False, timeout: int = 3) -> str: + async def acompletion_text(self, messages: list[dict], stream: bool = False, timeout: int = 0) -> str: """Asynchronous version of completion. Return str. Support stream-print""" if stream: - return await self._achat_completion_stream(messages, timeout=timeout) - resp = await self._achat_completion(messages, timeout=timeout) + return await self._achat_completion_stream(messages, timeout=self.get_timeout(timeout)) + resp = await self._achat_completion(messages, timeout=self.get_timeout(timeout)) return self.get_choice_text(resp) def get_choice_text(self, rsp: dict) -> str: @@ -236,3 +237,6 @@ class BaseLLM(ABC): """Set model and return self. For example, `with_model("gpt-3.5-turbo")`.""" self.config.model = model return self + + def get_timeout(self, timeout: int) -> int: + return timeout or self.config.timeout or LLM_API_TIMEOUT diff --git a/metagpt/provider/dashscope_api.py b/metagpt/provider/dashscope_api.py index 21f3ef351..a3efd5116 100644 --- a/metagpt/provider/dashscope_api.py +++ b/metagpt/provider/dashscope_api.py @@ -202,16 +202,16 @@ class DashScopeLLM(BaseLLM): self._update_costs(dict(resp.usage)) return resp.output - async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> GenerationOutput: + async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> GenerationOutput: resp: GenerationResponse = await self.aclient.acall(**self._const_kwargs(messages, stream=False)) self._check_response(resp) self._update_costs(dict(resp.usage)) return resp.output - async def acompletion(self, messages: list[dict], timeout=3) -> GenerationOutput: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout=0) -> GenerationOutput: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: resp = await self.aclient.acall(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = {} diff --git a/metagpt/provider/general_api_base.py b/metagpt/provider/general_api_base.py index 1b9149396..8e5da8f16 100644 --- a/metagpt/provider/general_api_base.py +++ b/metagpt/provider/general_api_base.py @@ -573,7 +573,7 @@ class APIRequestor: total=request_timeout[1], ) else: - timeout = aiohttp.ClientTimeout(total=request_timeout if request_timeout else TIMEOUT_SECS) + timeout = aiohttp.ClientTimeout(total=request_timeout or TIMEOUT_SECS) if files: # TODO: Use `aiohttp.MultipartWriter` to create the multipart form data here. diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 09e554205..2a00d4b5a 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -88,16 +88,16 @@ class GeminiLLM(BaseLLM): self._update_costs(usage) return resp - async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> "AsyncGenerateContentResponse": + async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> "AsyncGenerateContentResponse": resp: AsyncGenerateContentResponse = await self.llm.generate_content_async(**self._const_kwargs(messages)) usage = await self.aget_usage(messages, resp.text) self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=3) -> dict: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout=0) -> dict: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: resp: AsyncGenerateContentResponse = await self.llm.generate_content_async( **self._const_kwargs(messages, stream=True) ) diff --git a/metagpt/provider/human_provider.py b/metagpt/provider/human_provider.py index e5f37c5b9..df63a8bc9 100644 --- a/metagpt/provider/human_provider.py +++ b/metagpt/provider/human_provider.py @@ -18,7 +18,7 @@ class HumanProvider(BaseLLM): def __init__(self, config: LLMConfig): pass - def ask(self, msg: str, timeout=3) -> str: + def ask(self, msg: str, timeout=0) -> str: logger.info("It's your turn, please type in your response. You may also refer to the context below") rsp = input(msg) if rsp in ["exit", "quit"]: @@ -31,20 +31,20 @@ class HumanProvider(BaseLLM): system_msgs: Optional[list[str]] = None, format_msgs: Optional[list[dict[str, str]]] = None, generator: bool = False, - timeout=3, + timeout=0, ) -> str: - return self.ask(msg, timeout=timeout) + return self.ask(msg, timeout=self.get_timeout(timeout)) - async def _achat_completion(self, messages: list[dict], timeout=3): + async def _achat_completion(self, messages: list[dict], timeout=0): pass - async def acompletion(self, messages: list[dict], timeout=3): + async def acompletion(self, messages: list[dict], timeout=0): """dummy implementation of abstract method in base""" return [] - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: pass - async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout=0) -> str: """dummy implementation of abstract method in base""" return "" diff --git a/metagpt/provider/ollama_api.py b/metagpt/provider/ollama_api.py index f65d7e411..723abb574 100644 --- a/metagpt/provider/ollama_api.py +++ b/metagpt/provider/ollama_api.py @@ -5,7 +5,6 @@ import json from metagpt.configs.llm_config import LLMConfig, LLMType -from metagpt.const import LLM_API_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.general_api_requestor import GeneralAPIRequestor @@ -50,28 +49,28 @@ class OllamaLLM(BaseLLM): chunk = chunk.decode(encoding) return json.loads(chunk) - async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> dict: + async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> dict: resp, _, _ = await self.client.arequest( method=self.http_method, url=self.suffix_url, params=self._const_kwargs(messages), - request_timeout=LLM_API_TIMEOUT, + request_timeout=self.get_timeout(timeout), ) resp = self._decode_and_load(resp) usage = self.get_usage(resp) self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=3) -> dict: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout=0) -> dict: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: stream_resp, _, _ = await self.client.arequest( method=self.http_method, url=self.suffix_url, stream=True, params=self._const_kwargs(messages, stream=True), - request_timeout=LLM_API_TIMEOUT, + request_timeout=self.get_timeout(timeout), ) collected_content = [] diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index b4f99e69f..5b4abae61 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -79,9 +79,9 @@ class OpenAILLM(BaseLLM): return params - async def _achat_completion_stream(self, messages: list[dict], timeout=3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout=0) -> str: response: AsyncStream[ChatCompletionChunk] = await self.aclient.chat.completions.create( - **self._cons_kwargs(messages, timeout=timeout), stream=True + **self._cons_kwargs(messages, timeout=self.get_timeout(timeout)), stream=True ) usage = None collected_messages = [] @@ -109,7 +109,7 @@ class OpenAILLM(BaseLLM): self._update_costs(usage) return full_reply_content - def _cons_kwargs(self, messages: list[dict], timeout=3, **extra_kwargs) -> dict: + def _cons_kwargs(self, messages: list[dict], timeout=0, **extra_kwargs) -> dict: kwargs = { "messages": messages, "max_tokens": self._get_max_tokens(messages), @@ -117,20 +117,20 @@ class OpenAILLM(BaseLLM): # "stop": None, # default it's None and gpt4-v can't have this one "temperature": self.config.temperature, "model": self.model, - "timeout": max(self.config.timeout, timeout), + "timeout": self.get_timeout(timeout), } if extra_kwargs: kwargs.update(extra_kwargs) return kwargs - async def _achat_completion(self, messages: list[dict], timeout=3) -> ChatCompletion: - kwargs = self._cons_kwargs(messages, timeout=timeout) + async def _achat_completion(self, messages: list[dict], timeout=0) -> ChatCompletion: + kwargs = self._cons_kwargs(messages, timeout=self.get_timeout(timeout)) rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs) self._update_costs(rsp.usage) return rsp - async def acompletion(self, messages: list[dict], timeout=3) -> ChatCompletion: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout=0) -> ChatCompletion: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) @retry( wait=wait_random_exponential(min=1, max=60), @@ -139,24 +139,24 @@ class OpenAILLM(BaseLLM): retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream=False, timeout=3) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout=0) -> str: """when streaming, print each token in place.""" if stream: return await self._achat_completion_stream(messages, timeout=timeout) - rsp = await self._achat_completion(messages, timeout=timeout) + rsp = await self._achat_completion(messages, timeout=self.get_timeout(timeout)) return self.get_choice_text(rsp) async def _achat_completion_function( - self, messages: list[dict], timeout: int = 3, **chat_configs + self, messages: list[dict], timeout: int = 0, **chat_configs ) -> ChatCompletion: messages = process_message(messages) - kwargs = self._cons_kwargs(messages=messages, timeout=timeout, **chat_configs) + kwargs = self._cons_kwargs(messages=messages, timeout=self.get_timeout(timeout), **chat_configs) rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs) self._update_costs(rsp.usage) return rsp - async def aask_code(self, messages: list[dict], timeout: int = 3, **kwargs) -> dict: + async def aask_code(self, messages: list[dict], timeout: int = 0, **kwargs) -> dict: """Use function of tools to ask a code. Note: Keep kwargs consistent with https://platform.openai.com/docs/api-reference/chat/create diff --git a/metagpt/provider/qianfan_api.py b/metagpt/provider/qianfan_api.py index 50916fa3e..7e0bf009e 100644 --- a/metagpt/provider/qianfan_api.py +++ b/metagpt/provider/qianfan_api.py @@ -107,15 +107,15 @@ class QianFanLLM(BaseLLM): self._update_costs(resp.body.get("usage", {})) return resp.body - async def _achat_completion(self, messages: list[dict], timeout: int = 3) -> JsonBody: + async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> JsonBody: resp = await self.aclient.ado(**self._const_kwargs(messages=messages, stream=False)) self._update_costs(resp.body.get("usage", {})) return resp.body - async def acompletion(self, messages: list[dict], timeout: int = 3) -> JsonBody: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout: int = 0) -> JsonBody: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: resp = await self.aclient.ado(**self._const_kwargs(messages=messages, stream=True)) collected_content = [] usage = {} diff --git a/metagpt/provider/spark_api.py b/metagpt/provider/spark_api.py index 882c6ce85..0f450ccf5 100644 --- a/metagpt/provider/spark_api.py +++ b/metagpt/provider/spark_api.py @@ -31,19 +31,19 @@ class SparkLLM(BaseLLM): def get_choice_text(self, rsp: dict) -> str: return rsp["payload"]["choices"]["text"][-1]["content"] - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: pass - async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 3) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 0) -> str: # 不支持 # logger.warning("当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。") w = GetMessageFromWeb(messages, self.config) return w.run() - async def _achat_completion(self, messages: list[dict], timeout=3): + async def _achat_completion(self, messages: list[dict], timeout=0): pass - async def acompletion(self, messages: list[dict], timeout=3): + async def acompletion(self, messages: list[dict], timeout=0): # 不支持异步 w = GetMessageFromWeb(messages, self.config) return w.run() diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 14ad1a36b..8c5284770 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -45,22 +45,22 @@ class ZhiPuAILLM(BaseLLM): kwargs = {"model": self.model, "messages": messages, "stream": stream, "temperature": 0.3} return kwargs - def completion(self, messages: list[dict], timeout=3) -> dict: + def completion(self, messages: list[dict], timeout=0) -> dict: resp: Completion = self.llm.chat.completions.create(**self._const_kwargs(messages)) usage = resp.usage.model_dump() self._update_costs(usage) return resp.model_dump() - async def _achat_completion(self, messages: list[dict], timeout=3) -> dict: + async def _achat_completion(self, messages: list[dict], timeout=0) -> dict: resp = await self.llm.acreate(**self._const_kwargs(messages)) usage = resp.get("usage", {}) self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=3) -> dict: - return await self._achat_completion(messages, timeout=timeout) + async def acompletion(self, messages: list[dict], timeout=0) -> dict: + return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout=3) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout=0) -> str: response = await self.llm.acreate_stream(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = {} diff --git a/requirements.txt b/requirements.txt index 83565278b..6b23e47b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ PyYAML==6.0.1 # sentence_transformers==2.2.2 setuptools==65.6.3 tenacity==8.2.3 -tiktoken==0.5.2 +tiktoken==0.6.0 tqdm==4.66.2 #unstructured[local-inference] # selenium>4 From 7c8f57e46c16aaa5a3acfeae6f3659f47b8952d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 20 Mar 2024 21:37:47 +0800 Subject: [PATCH 025/127] feat: + timeout --- config/config2.example.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config2.example.yaml b/config/config2.example.yaml index 3a5cc3585..46059e7e1 100644 --- a/config/config2.example.yaml +++ b/config/config2.example.yaml @@ -4,6 +4,7 @@ llm: api_key: "YOUR_API_KEY" model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview proxy: "YOUR_PROXY" # for LLM API requests + # timeout: 600 # Optional. pricing_plan: "" # Optional. If invalid, it will be automatically filled in with the value of the `model`. # Azure-exclusive pricing plan mappings： # - gpt-3.5-turbo 4k: "gpt-3.5-turbo-1106" From 067a39e7b923e4d9394ca8547f04cc1e24cd079d Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 21 Mar 2024 10:45:53 +0800 Subject: [PATCH 026/127] remove incorrect file header --- examples/di/arxiv_reader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/di/arxiv_reader.py b/examples/di/arxiv_reader.py index fab0e2e48..7f2c6e331 100644 --- a/examples/di/arxiv_reader.py +++ b/examples/di/arxiv_reader.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- """ @Time : 2024/01/15 -@Author : mannaandpoem -@File : imitate_webpage.py """ from metagpt.roles.di.data_interpreter import DataInterpreter From dd348d07071c9d59ec243058922d3d37c90eb6c3 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 21 Mar 2024 10:45:58 +0800 Subject: [PATCH 027/127] remove incorrect file header --- examples/di/arxiv_reader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/di/arxiv_reader.py b/examples/di/arxiv_reader.py index 7f2c6e331..6e1939b81 100644 --- a/examples/di/arxiv_reader.py +++ b/examples/di/arxiv_reader.py @@ -1,8 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" -@Time : 2024/01/15 -""" from metagpt.roles.di.data_interpreter import DataInterpreter From 191a86f93e0c448b40db201f5e4f697d29737e8c Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Thu, 21 Mar 2024 12:04:06 +0800 Subject: [PATCH 028/127] rag add es --- examples/rag_pipeline.py | 74 +++++++++++++++++++---- metagpt/rag/factories/index.py | 47 +++++++++----- metagpt/rag/factories/retriever.py | 17 +++++- metagpt/rag/retrievers/es_retriever.py | 17 ++++++ metagpt/rag/retrievers/faiss_retriever.py | 2 +- metagpt/rag/schema.py | 28 ++++++++- requirements.txt | 3 + 7 files changed, 157 insertions(+), 31 deletions(-) create mode 100644 metagpt/rag/retrievers/es_retriever.py diff --git a/examples/rag_pipeline.py b/examples/rag_pipeline.py index 5a313d7bb..ae6e7b7bc 100644 --- a/examples/rag_pipeline.py +++ b/examples/rag_pipeline.py @@ -1,6 +1,7 @@ """RAG pipeline""" import asyncio +from functools import wraps from pydantic import BaseModel @@ -11,6 +12,9 @@ from metagpt.rag.schema import ( BM25RetrieverConfig, ChromaIndexConfig, ChromaRetrieverConfig, + ElasticsearchIndexConfig, + ElasticsearchRetrieverConfig, + ElasticsearchStoreConfig, FAISSRetrieverConfig, LLMRankerConfig, ) @@ -24,6 +28,17 @@ TRAVEL_QUESTION = "What does Bob like?" LLM_TIP = "If you not sure, just answer I don't know." +def catch_exception(func): + @wraps(func) + async def wrapper(*args, **kwargs): + try: + return await func(*args, **kwargs) + except Exception as e: + logger.error(f"{func.__name__} exception: {e}") + + return wrapper + + class Player(BaseModel): """To demonstrate rag add objs.""" @@ -39,12 +54,22 @@ class Player(BaseModel): class RAGExample: """Show how to use RAG.""" - def __init__(self): - self.engine = SimpleEngine.from_docs( - input_files=[DOC_PATH], - retriever_configs=[FAISSRetrieverConfig(), BM25RetrieverConfig()], - ranker_configs=[LLMRankerConfig()], - ) + def __init__(self, engine: SimpleEngine = None): + self._engine = engine + + @property + def engine(self): + if not self._engine: + self._engine = SimpleEngine.from_docs( + input_files=[DOC_PATH], + retriever_configs=[FAISSRetrieverConfig(), BM25RetrieverConfig()], + ranker_configs=[LLMRankerConfig()], + ) + return self._engine + + @engine.setter + def engine(self, value: SimpleEngine): + self._engine = value async def run_pipeline(self, question=QUESTION, print_title=True): """This example run rag pipeline, use faiss&bm25 retriever and llm ranker, will print something like: @@ -97,6 +122,7 @@ class RAGExample: self.engine.add_docs([travel_filepath]) await self.run_pipeline(question=travel_question, print_title=False) + @catch_exception async def add_objects(self, print_title=True): """This example show how to add objects. @@ -154,20 +180,43 @@ class RAGExample: """ self._print_title("Init And Query ChromaDB") - # save index + # 1.save index output_dir = DATA_PATH / "rag" SimpleEngine.from_docs( input_files=[TRAVEL_DOC_PATH], retriever_configs=[ChromaRetrieverConfig(persist_path=output_dir)], ) - # load index - engine = SimpleEngine.from_index( - index_config=ChromaIndexConfig(persist_path=output_dir), + # 2.load index + engine = SimpleEngine.from_index(index_config=ChromaIndexConfig(persist_path=output_dir)) + + # 3.query + answer = await engine.aquery(TRAVEL_QUESTION) + self._print_query_result(answer) + + @catch_exception + async def init_and_query_es(self): + """This example show how to use es. how to save and load index. will print something like: + + Query Result: + Bob likes traveling. + + If `Unclosed client session`, it's llamaindex elasticsearch problem, maybe fixed later. + """ + self._print_title("Init And Query Elasticsearch") + + # 1.create es index and save docs + store_config = ElasticsearchStoreConfig(index_name="travel", es_url="http://127.0.0.1:9200") + engine = SimpleEngine.from_docs( + input_files=[TRAVEL_DOC_PATH], + retriever_configs=[ElasticsearchRetrieverConfig(store_config=store_config)], ) - # query - answer = engine.query(TRAVEL_QUESTION) + # 2.load index + engine = SimpleEngine.from_index(index_config=ElasticsearchIndexConfig(store_config=store_config)) + + # 3.query + answer = await engine.aquery(TRAVEL_QUESTION) self._print_query_result(answer) @staticmethod @@ -205,6 +254,7 @@ async def main(): await e.add_objects() await e.init_objects() await e.init_and_query_chromadb() + await e.init_and_query_es() if __name__ == "__main__": diff --git a/metagpt/rag/factories/index.py b/metagpt/rag/factories/index.py index 6aad695e7..5ab7992a0 100644 --- a/metagpt/rag/factories/index.py +++ b/metagpt/rag/factories/index.py @@ -4,6 +4,8 @@ import chromadb from llama_index.core import StorageContext, VectorStoreIndex, load_index_from_storage from llama_index.core.embeddings import BaseEmbedding from llama_index.core.indices.base import BaseIndex +from llama_index.core.vector_stores.types import BasePydanticVectorStore +from llama_index.vector_stores.elasticsearch import ElasticsearchStore from llama_index.vector_stores.faiss import FaissVectorStore from metagpt.rag.factories.base import ConfigBasedFactory @@ -11,6 +13,7 @@ from metagpt.rag.schema import ( BaseIndexConfig, BM25IndexConfig, ChromaIndexConfig, + ElasticsearchIndexConfig, FAISSIndexConfig, ) from metagpt.rag.vector_stores.chroma import ChromaVectorStore @@ -22,6 +25,7 @@ class RAGIndexFactory(ConfigBasedFactory): FAISSIndexConfig: self._create_faiss, ChromaIndexConfig: self._create_chroma, BM25IndexConfig: self._create_bm25, + ElasticsearchIndexConfig: self._create_es, } super().__init__(creators) @@ -30,31 +34,44 @@ class RAGIndexFactory(ConfigBasedFactory): return super().get_instance(config, **kwargs) def _create_faiss(self, config: FAISSIndexConfig, **kwargs) -> VectorStoreIndex: - embed_model = self._extract_embed_model(config, **kwargs) - vector_store = FaissVectorStore.from_persist_dir(str(config.persist_path)) storage_context = StorageContext.from_defaults(vector_store=vector_store, persist_dir=config.persist_path) - index = load_index_from_storage(storage_context=storage_context, embed_model=embed_model) - return index + + return self._index_from_storage(storage_context=storage_context, config=config, **kwargs) + + def _create_bm25(self, config: BM25IndexConfig, **kwargs) -> VectorStoreIndex: + storage_context = StorageContext.from_defaults(persist_dir=config.persist_path) + + return self._index_from_storage(storage_context=storage_context, config=config, **kwargs) def _create_chroma(self, config: ChromaIndexConfig, **kwargs) -> VectorStoreIndex: - embed_model = self._extract_embed_model(config, **kwargs) - db = chromadb.PersistentClient(str(config.persist_path)) chroma_collection = db.get_or_create_collection(config.collection_name) vector_store = ChromaVectorStore(chroma_collection=chroma_collection) - index = VectorStoreIndex.from_vector_store( - vector_store, - embed_model=embed_model, - ) - return index - def _create_bm25(self, config: BM25IndexConfig, **kwargs) -> VectorStoreIndex: + return self._index_from_vector_store(vector_store=vector_store, config=config, **kwargs) + + def _create_es(self, config: ElasticsearchIndexConfig, **kwargs) -> VectorStoreIndex: + vector_store = ElasticsearchStore(**config.store_config.model_dump()) + + return self._index_from_vector_store(vector_store=vector_store, config=config, **kwargs) + + def _index_from_storage( + self, storage_context: StorageContext, config: BaseIndexConfig, **kwargs + ) -> VectorStoreIndex: embed_model = self._extract_embed_model(config, **kwargs) - storage_context = StorageContext.from_defaults(persist_dir=config.persist_path) - index = load_index_from_storage(storage_context=storage_context, embed_model=embed_model) - return index + return load_index_from_storage(storage_context=storage_context, embed_model=embed_model) + + def _index_from_vector_store( + self, vector_store: BasePydanticVectorStore, config: BaseIndexConfig, **kwargs + ) -> VectorStoreIndex: + embed_model = self._extract_embed_model(config, **kwargs) + + return VectorStoreIndex.from_vector_store( + vector_store=vector_store, + embed_model=embed_model, + ) def _extract_embed_model(self, config, **kwargs) -> BaseEmbedding: return self._val_from_config_or_kwargs("embed_model", config, **kwargs) diff --git a/metagpt/rag/factories/retriever.py b/metagpt/rag/factories/retriever.py index ba48c753e..47ceadf00 100644 --- a/metagpt/rag/factories/retriever.py +++ b/metagpt/rag/factories/retriever.py @@ -6,18 +6,21 @@ import chromadb import faiss from llama_index.core import StorageContext, VectorStoreIndex from llama_index.core.vector_stores.types import BasePydanticVectorStore +from llama_index.vector_stores.elasticsearch import ElasticsearchStore from llama_index.vector_stores.faiss import FaissVectorStore from metagpt.rag.factories.base import ConfigBasedFactory from metagpt.rag.retrievers.base import RAGRetriever from metagpt.rag.retrievers.bm25_retriever import DynamicBM25Retriever from metagpt.rag.retrievers.chroma_retriever import ChromaRetriever +from metagpt.rag.retrievers.es_retriever import ElasticsearchRetriever from metagpt.rag.retrievers.faiss_retriever import FAISSRetriever from metagpt.rag.retrievers.hybrid_retriever import SimpleHybridRetriever from metagpt.rag.schema import ( BaseRetrieverConfig, BM25RetrieverConfig, ChromaRetrieverConfig, + ElasticsearchRetrieverConfig, FAISSRetrieverConfig, IndexRetrieverConfig, ) @@ -32,6 +35,7 @@ class RetrieverFactory(ConfigBasedFactory): FAISSRetrieverConfig: self._create_faiss_retriever, BM25RetrieverConfig: self._create_bm25_retriever, ChromaRetrieverConfig: self._create_chroma_retriever, + ElasticsearchRetrieverConfig: self._create_es_retriever, } super().__init__(creators) @@ -53,20 +57,29 @@ class RetrieverFactory(ConfigBasedFactory): def _create_faiss_retriever(self, config: FAISSRetrieverConfig, **kwargs) -> FAISSRetriever: vector_store = FaissVectorStore(faiss_index=faiss.IndexFlatL2(config.dimensions)) config.index = self._build_index_from_vector_store(config, vector_store, **kwargs) + return FAISSRetriever(**config.model_dump()) def _create_bm25_retriever(self, config: BM25RetrieverConfig, **kwargs) -> DynamicBM25Retriever: config.index = copy.deepcopy(self._extract_index(config, **kwargs)) - nodes = list(config.index.docstore.docs.values()) - return DynamicBM25Retriever(nodes=nodes, **config.model_dump()) + + return DynamicBM25Retriever(nodes=list(config.index.docstore.docs.values()), **config.model_dump()) def _create_chroma_retriever(self, config: ChromaRetrieverConfig, **kwargs) -> ChromaRetriever: db = chromadb.PersistentClient(path=str(config.persist_path)) chroma_collection = db.get_or_create_collection(config.collection_name) + vector_store = ChromaVectorStore(chroma_collection=chroma_collection) config.index = self._build_index_from_vector_store(config, vector_store, **kwargs) + return ChromaRetriever(**config.model_dump()) + def _create_es_retriever(self, config: ElasticsearchRetrieverConfig, **kwargs) -> ElasticsearchRetriever: + vector_store = ElasticsearchStore(**config.store_config.model_dump()) + config.index = self._build_index_from_vector_store(config, vector_store, **kwargs) + + return ElasticsearchRetriever(**config.model_dump()) + def _extract_index(self, config: BaseRetrieverConfig = None, **kwargs) -> VectorStoreIndex: return self._val_from_config_or_kwargs("index", config, **kwargs) diff --git a/metagpt/rag/retrievers/es_retriever.py b/metagpt/rag/retrievers/es_retriever.py new file mode 100644 index 000000000..a1a0a6138 --- /dev/null +++ b/metagpt/rag/retrievers/es_retriever.py @@ -0,0 +1,17 @@ +"""Elasticsearch retriever.""" + +from llama_index.core.retrievers import VectorIndexRetriever +from llama_index.core.schema import BaseNode + + +class ElasticsearchRetriever(VectorIndexRetriever): + """Elasticsearch retriever.""" + + def add_nodes(self, nodes: list[BaseNode], **kwargs) -> None: + """Support add nodes.""" + self._index.insert_nodes(nodes, **kwargs) + + def persist(self, persist_dir: str, **kwargs) -> None: + """Support persist. + + Elasticsearch automatically saves, so there is no need to implement.""" diff --git a/metagpt/rag/retrievers/faiss_retriever.py b/metagpt/rag/retrievers/faiss_retriever.py index 7e543cce2..80b409292 100644 --- a/metagpt/rag/retrievers/faiss_retriever.py +++ b/metagpt/rag/retrievers/faiss_retriever.py @@ -8,7 +8,7 @@ class FAISSRetriever(VectorIndexRetriever): """FAISS retriever.""" def add_nodes(self, nodes: list[BaseNode], **kwargs) -> None: - """Support add nodes""" + """Support add nodes.""" self._index.insert_nodes(nodes, **kwargs) def persist(self, persist_dir: str, **kwargs) -> None: diff --git a/metagpt/rag/schema.py b/metagpt/rag/schema.py index cae1c2979..e98a6fc89 100644 --- a/metagpt/rag/schema.py +++ b/metagpt/rag/schema.py @@ -6,6 +6,7 @@ from typing import Any, Union from llama_index.core.embeddings import BaseEmbedding from llama_index.core.indices.base import BaseIndex from llama_index.core.schema import TextNode +from llama_index.core.vector_stores.types import VectorStoreQueryMode from pydantic import BaseModel, ConfigDict, Field, PrivateAttr from metagpt.rag.interface import RAGObject @@ -46,6 +47,24 @@ class ChromaRetrieverConfig(IndexRetrieverConfig): collection_name: str = Field(default="metagpt", description="The name of the collection.") +class ElasticsearchStoreConfig(BaseModel): + index_name: str = Field(default="metagpt", description="Name of the Elasticsearch index.") + es_url: str = Field(default=None, description="Elasticsearch URL.") + es_cloud_id: str = Field(default=None, description="Elasticsearch cloud ID.") + es_api_key: str = Field(default=None, description="Elasticsearch API key.") + es_user: str = Field(default=None, description="Elasticsearch username.") + es_password: str = Field(default=None, description="Elasticsearch password.") + batch_size: int = Field(default=200, description="Batch size for bulk indexing.") + distance_strategy: str = Field(default="COSINE", description="Distance strategy to use for similarity search.") + + +class ElasticsearchRetrieverConfig(IndexRetrieverConfig): + """Config for Elasticsearch-based retrievers.""" + + store_config: ElasticsearchStoreConfig = Field(..., description="ElasticsearchStore config.") + vector_store_query_mode: VectorStoreQueryMode = VectorStoreQueryMode.DEFAULT + + class BaseRankerConfig(BaseModel): """Common config for rankers. @@ -53,7 +72,6 @@ class BaseRankerConfig(BaseModel): """ model_config = ConfigDict(arbitrary_types_allowed=True) - top_n: int = Field(default=5, description="The number of top results to return.") @@ -72,6 +90,7 @@ class BaseIndexConfig(BaseModel): If add new subconfig, it is necessary to add the corresponding instance implementation in rag.factories.index. """ + model_config = ConfigDict(arbitrary_types_allowed=True) persist_path: Union[str, Path] = Field(description="The directory of saved data.") @@ -97,6 +116,13 @@ class BM25IndexConfig(BaseIndexConfig): _no_embedding: bool = PrivateAttr(default=True) +class ElasticsearchIndexConfig(VectorIndexConfig): + """Config for es-based index.""" + + store_config: ElasticsearchStoreConfig = Field(..., description="ElasticsearchStore config.") + persist_path: Union[str, Path] = "" + + class ObjectNodeMetadata(BaseModel): """Metadata of ObjectNode.""" diff --git a/requirements.txt b/requirements.txt index 326fa8bb9..3e545d146 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ llama-index-llms-azure-openai==0.1.4 llama-index-readers-file==0.1.4 llama-index-retrievers-bm25==0.1.3 llama-index-vector-stores-faiss==0.1.1 +llama-index-vector-stores-elasticsearch==0.1.5 chromadb==0.4.23 loguru==0.6.0 meilisearch==0.21.0 @@ -76,3 +77,5 @@ Pillow imap_tools==1.5.0 # Used by metagpt/tools/libs/email_login.py qianfan==0.3.2 dashscope==1.14.1 +rank-bm25==0.2.2 # for tool recommendation +jieba==0.42.1 # for tool recommendation From 9350e214b4280049387d50bf8af2ae6bf5205029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 12:59:44 +0800 Subject: [PATCH 029/127] feat: + repo to markdown --- .gitignore | 2 +- metagpt/utils/common.py | 19 +++++ metagpt/utils/repo_to_markdown.py | 80 ++++++++++++++++++++ metagpt/utils/tree.py | 2 +- tests/metagpt/utils/test_repo_to_markdown.py | 25 ++++++ 5 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 metagpt/utils/repo_to_markdown.py create mode 100644 tests/metagpt/utils/test_repo_to_markdown.py diff --git a/.gitignore b/.gitignore index 922116d12..aa5edd74a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ ### Python template # Byte-compiled / optimized / DLL files -__pycache__/ +__pycache__ *.py[cod] *$py.class diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index e9cef69a4..cc40e3762 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -18,6 +18,7 @@ import csv import importlib import inspect import json +import mimetypes import os import platform import re @@ -834,3 +835,21 @@ See FAQ 5.8 """ ) raise retry_state.outcome.exception() + + +def get_markdown_codeblock_type(filename: str) -> str: + """Return the markdown code-block type corresponding to the file extension.""" + mime_type, _ = mimetypes.guess_type(filename) + mappings = { + "text/x-shellscript": "bash", + "text/x-c++src": "cpp", + "text/css": "css", + "text/html": "html", + "text/x-java": "java", + "application/javascript": "javascript", + "application/json": "json", + "text/x-python": "python", + "text/x-ruby": "ruby", + "application/sql": "sql", + } + return mappings.get(mime_type, "text") diff --git a/metagpt/utils/repo_to_markdown.py b/metagpt/utils/repo_to_markdown.py new file mode 100644 index 000000000..76dfe1b82 --- /dev/null +++ b/metagpt/utils/repo_to_markdown.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +This file provides functionality to convert a local repository into a markdown representation. +""" +from __future__ import annotations + +import mimetypes +from pathlib import Path + +from gitignore_parser import parse_gitignore + +from metagpt.logs import logger +from metagpt.utils.common import aread, awrite, get_markdown_codeblock_type, list_files +from metagpt.utils.tree import tree + + +async def repo_to_markdown(repo_path: str | Path, output: str | Path = None, gitignore: str | Path = None) -> str: + """ + Convert a local repository into a markdown representation. + + This function takes a path to a local repository and generates a markdown representation of the repository structure, + including directory trees and file listings. + + Args: + repo_path (str | Path): The path to the local repository. + output (str | Path, optional): The path to save the generated markdown file. Defaults to None. + gitignore (str | Path, optional): The path to the .gitignore file. Defaults to None. + + Returns: + str: The markdown representation of the repository. + """ + repo_path = Path(repo_path) + gitignore = Path(gitignore or Path(__file__).parent / "../../.gitignore").resolve() + + markdown = await _write_dir_tree(repo_path=repo_path, gitignore=gitignore) + + gitignore_rules = parse_gitignore(full_path=str(gitignore)) + markdown += await _write_files(repo_path=repo_path, gitignore_rules=gitignore_rules) + + if output: + await awrite(filename=str(output), data=markdown, encoding="utf-8") + return markdown + + +async def _write_dir_tree(repo_path: Path, gitignore: Path) -> str: + try: + content = tree(repo_path, gitignore, run_command=True) + except Exception as e: + logger.info(f"{e}, using safe mode.") + content = tree(repo_path, gitignore, run_command=False) + + doc = f"## Directory Tree\n```text\n{content}\n```\n---\n\n" + return doc + + +async def _write_files(repo_path, gitignore_rules) -> str: + filenames = list_files(repo_path) + markdown = "" + for filename in filenames: + if gitignore_rules(str(filename)): + continue + markdown += await _write_file(filename=filename, repo_path=repo_path) + return markdown + + +async def _write_file(filename: Path, repo_path: Path) -> str: + relative_path = filename.relative_to(repo_path) + markdown = f"## {relative_path}\n" + + mime_type, _ = mimetypes.guess_type(filename.name) + if "text/" not in mime_type: + logger.info(f"Ignore content: {filename}") + markdown += "\n---\n\n" + return markdown + content = await aread(filename, encoding="utf-8") + content = content.replace("```", "\\`\\`\\`").replace("---", "\\-\\-\\-") + code_block_type = get_markdown_codeblock_type(filename.name) + markdown += f"```{code_block_type}\n{content}\n```\n---\n\n" + return markdown diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index fbf085e48..bd7922290 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -130,7 +130,7 @@ def _add_line(rows: List[str]) -> List[str]: def _execute_tree(root: Path, gitignore: str | Path) -> str: - args = ["--gitignore", str(gitignore)] if gitignore else [] + args = ["--gitfile", str(gitignore)] if gitignore else [] try: result = subprocess.run(["tree"] + args + [str(root)], capture_output=True, text=True, check=True) if result.returncode != 0: diff --git a/tests/metagpt/utils/test_repo_to_markdown.py b/tests/metagpt/utils/test_repo_to_markdown.py new file mode 100644 index 000000000..914c50dd7 --- /dev/null +++ b/tests/metagpt/utils/test_repo_to_markdown.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import uuid +from pathlib import Path + +import pytest + +from metagpt.utils.repo_to_markdown import repo_to_markdown + + +@pytest.mark.parametrize( + ["repo_path", "output"], + [(Path(__file__).parent.parent, Path(__file__).parent.parent.parent / f"workspace/unittest/{uuid.uuid4().hex}.md")], +) +@pytest.mark.asyncio +async def test_repo_to_markdown(repo_path: Path, output: Path): + markdown = await repo_to_markdown(repo_path=repo_path, output=output) + assert output.exists() + assert markdown + + output.unlink(missing_ok=True) + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From b42cf5cbd6d1cc7b0ffb22e893efb754022da8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 13:21:24 +0800 Subject: [PATCH 030/127] refactor: timeout 0 --- config/config2.example.yaml | 2 +- metagpt/actions/action_node.py | 9 ++++++--- metagpt/configs/llm_config.py | 6 ++++++ metagpt/const.py | 5 ++++- metagpt/provider/anthropic_api.py | 7 ++++--- metagpt/provider/base_llm.py | 18 ++++++++++-------- metagpt/provider/dashscope_api.py | 7 ++++--- metagpt/provider/google_gemini_api.py | 9 ++++++--- metagpt/provider/human_provider.py | 13 +++++++------ metagpt/provider/ollama_api.py | 7 ++++--- metagpt/provider/openai_api.py | 15 ++++++++------- metagpt/provider/qianfan_api.py | 7 ++++--- metagpt/provider/spark_api.py | 9 +++++---- metagpt/provider/zhipuai_api.py | 9 +++++---- 14 files changed, 74 insertions(+), 49 deletions(-) diff --git a/config/config2.example.yaml b/config/config2.example.yaml index 46059e7e1..c5454ec32 100644 --- a/config/config2.example.yaml +++ b/config/config2.example.yaml @@ -4,7 +4,7 @@ llm: api_key: "YOUR_API_KEY" model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview proxy: "YOUR_PROXY" # for LLM API requests - # timeout: 600 # Optional. + # timeout: 600 # Optional. If set to 0, default value is 300. pricing_plan: "" # Optional. If invalid, it will be automatically filled in with the value of the `model`. # Azure-exclusive pricing plan mappings： # - gpt-3.5-turbo 4k: "gpt-3.5-turbo-1106" diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 63925a052..3f822568e 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -17,6 +17,7 @@ from pydantic import BaseModel, Field, create_model, model_validator from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action_outcls_registry import register_action_outcls +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.llm import BaseLLM from metagpt.logs import logger from metagpt.provider.postprocess.llm_output_postprocess import llm_output_postprocess @@ -416,7 +417,7 @@ class ActionNode: images: Optional[Union[str, list[str]]] = None, system_msgs: Optional[list[str]] = None, schema="markdown", # compatible to original format - timeout=0, + timeout=USE_CONFIG_TIMEOUT, ) -> (str, BaseModel): """Use ActionOutput to wrap the output of aask""" content = await self.llm.aask(prompt, system_msgs, images=images, timeout=timeout) @@ -448,7 +449,9 @@ class ActionNode: def set_context(self, context): self.set_recursive("context", context) - async def simple_fill(self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=0, exclude=None): + async def simple_fill( + self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=USE_CONFIG_TIMEOUT, exclude=None + ): prompt = self.compile(context=self.context, schema=schema, mode=mode, exclude=exclude) if schema != "raw": @@ -473,7 +476,7 @@ class ActionNode: mode="auto", strgy="simple", images: Optional[Union[str, list[str]]] = None, - timeout=0, + timeout=USE_CONFIG_TIMEOUT, exclude=[], ): """Fill the node(s) with mode. diff --git a/metagpt/configs/llm_config.py b/metagpt/configs/llm_config.py index 92b8e1512..af8f56372 100644 --- a/metagpt/configs/llm_config.py +++ b/metagpt/configs/llm_config.py @@ -10,6 +10,7 @@ from typing import Optional from pydantic import field_validator +from metagpt.const import LLM_API_TIMEOUT from metagpt.utils.yaml_model import YamlModel @@ -88,3 +89,8 @@ class LLMConfig(YamlModel): if v in ["", None, "YOUR_API_KEY"]: raise ValueError("Please set your API key in config2.yaml") return v + + @field_validator("timeout") + @classmethod + def check_timeout(cls, v): + return v or LLM_API_TIMEOUT diff --git a/metagpt/const.py b/metagpt/const.py index 6dbbfe0c1..e4cebfd96 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -123,7 +123,6 @@ BASE64_FORMAT = "base64" # REDIS REDIS_KEY = "REDIS_KEY" -LLM_API_TIMEOUT = 300 # Message id IGNORED_MESSAGE_ID = "0" @@ -132,3 +131,7 @@ IGNORED_MESSAGE_ID = "0" GENERALIZATION = "Generalize" COMPOSITION = "Composite" AGGREGATION = "Aggregate" + +# Timeout +USE_CONFIG_TIMEOUT = 0 # Using llm.timeout configuration. +LLM_API_TIMEOUT = 300 diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 3125ffc22..1aeacbe83 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -5,6 +5,7 @@ from anthropic import AsyncAnthropic from anthropic.types import Message, Usage from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -41,15 +42,15 @@ class AnthropicLLM(BaseLLM): def get_choice_text(self, resp: Message) -> str: return resp.content[0].text - async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> Message: + async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> Message: resp: Message = await self.aclient.messages.create(**self._const_kwargs(messages)) self._update_costs(resp.usage, self.model) return resp - async def acompletion(self, messages: list[dict], timeout: int = 0) -> Message: + async def acompletion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> Message: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: stream = await self.aclient.messages.create(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = Usage(input_tokens=0, output_tokens=0) diff --git a/metagpt/provider/base_llm.py b/metagpt/provider/base_llm.py index fa5119c67..e085d0187 100644 --- a/metagpt/provider/base_llm.py +++ b/metagpt/provider/base_llm.py @@ -23,7 +23,7 @@ from tenacity import ( ) from metagpt.configs.llm_config import LLMConfig -from metagpt.const import LLM_API_TIMEOUT +from metagpt.const import LLM_API_TIMEOUT, USE_CONFIG_TIMEOUT from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import log_and_reraise @@ -109,7 +109,7 @@ class BaseLLM(ABC): system_msgs: Optional[list[str]] = None, format_msgs: Optional[list[dict[str, str]]] = None, images: Optional[Union[str, list[str]]] = None, - timeout=0, + timeout=USE_CONFIG_TIMEOUT, stream=True, ) -> str: if system_msgs: @@ -131,7 +131,7 @@ class BaseLLM(ABC): def _extract_assistant_rsp(self, context): return "\n".join([i["content"] for i in context if i["role"] == "assistant"]) - async def aask_batch(self, msgs: list, timeout=0) -> str: + async def aask_batch(self, msgs: list, timeout=USE_CONFIG_TIMEOUT) -> str: """Sequential questioning""" context = [] for msg in msgs: @@ -141,15 +141,15 @@ class BaseLLM(ABC): context.append(self._assistant_msg(rsp_text)) return self._extract_assistant_rsp(context) - async def aask_code(self, messages: Union[str, Message, list[dict]], timeout=0, **kwargs) -> dict: + async def aask_code(self, messages: Union[str, Message, list[dict]], timeout=USE_CONFIG_TIMEOUT, **kwargs) -> dict: raise NotImplementedError @abstractmethod - async def _achat_completion(self, messages: list[dict], timeout=0): + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): """_achat_completion implemented by inherited class""" @abstractmethod - async def acompletion(self, messages: list[dict], timeout=0): + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): """Asynchronous version of completion All GPTAPIs are required to provide the standard OpenAI completion interface [ @@ -160,7 +160,7 @@ class BaseLLM(ABC): """ @abstractmethod - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: """_achat_completion_stream implemented by inherited class""" @retry( @@ -170,7 +170,9 @@ class BaseLLM(ABC): retry=retry_if_exception_type(ConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream: bool = False, timeout: int = 0) -> str: + async def acompletion_text( + self, messages: list[dict], stream: bool = False, timeout: int = USE_CONFIG_TIMEOUT + ) -> str: """Asynchronous version of completion. Return str. Support stream-print""" if stream: return await self._achat_completion_stream(messages, timeout=self.get_timeout(timeout)) diff --git a/metagpt/provider/dashscope_api.py b/metagpt/provider/dashscope_api.py index a3efd5116..82224e893 100644 --- a/metagpt/provider/dashscope_api.py +++ b/metagpt/provider/dashscope_api.py @@ -25,6 +25,7 @@ from dashscope.common.error import ( UnsupportedApiProtocol, ) +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM, LLMConfig from metagpt.provider.llm_provider_registry import LLMType, register_provider @@ -202,16 +203,16 @@ class DashScopeLLM(BaseLLM): self._update_costs(dict(resp.usage)) return resp.output - async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> GenerationOutput: + async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> GenerationOutput: resp: GenerationResponse = await self.aclient.acall(**self._const_kwargs(messages, stream=False)) self._check_response(resp) self._update_costs(dict(resp.usage)) return resp.output - async def acompletion(self, messages: list[dict], timeout=0) -> GenerationOutput: + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> GenerationOutput: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: resp = await self.aclient.acall(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = {} diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 2a00d4b5a..e041f4c87 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -15,6 +15,7 @@ from google.generativeai.types.generation_types import ( ) from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -88,16 +89,18 @@ class GeminiLLM(BaseLLM): self._update_costs(usage) return resp - async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> "AsyncGenerateContentResponse": + async def _achat_completion( + self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT + ) -> "AsyncGenerateContentResponse": resp: AsyncGenerateContentResponse = await self.llm.generate_content_async(**self._const_kwargs(messages)) usage = await self.aget_usage(messages, resp.text) self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=0) -> dict: + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: resp: AsyncGenerateContentResponse = await self.llm.generate_content_async( **self._const_kwargs(messages, stream=True) ) diff --git a/metagpt/provider/human_provider.py b/metagpt/provider/human_provider.py index df63a8bc9..f205ecd1f 100644 --- a/metagpt/provider/human_provider.py +++ b/metagpt/provider/human_provider.py @@ -6,6 +6,7 @@ Author: garylin2099 from typing import Optional from metagpt.configs.llm_config import LLMConfig +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import logger from metagpt.provider.base_llm import BaseLLM @@ -18,7 +19,7 @@ class HumanProvider(BaseLLM): def __init__(self, config: LLMConfig): pass - def ask(self, msg: str, timeout=0) -> str: + def ask(self, msg: str, timeout=USE_CONFIG_TIMEOUT) -> str: logger.info("It's your turn, please type in your response. You may also refer to the context below") rsp = input(msg) if rsp in ["exit", "quit"]: @@ -31,20 +32,20 @@ class HumanProvider(BaseLLM): system_msgs: Optional[list[str]] = None, format_msgs: Optional[list[dict[str, str]]] = None, generator: bool = False, - timeout=0, + timeout=USE_CONFIG_TIMEOUT, ) -> str: return self.ask(msg, timeout=self.get_timeout(timeout)) - async def _achat_completion(self, messages: list[dict], timeout=0): + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): pass - async def acompletion(self, messages: list[dict], timeout=0): + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): """dummy implementation of abstract method in base""" return [] - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: pass - async def acompletion_text(self, messages: list[dict], stream=False, timeout=0) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout=USE_CONFIG_TIMEOUT) -> str: """dummy implementation of abstract method in base""" return "" diff --git a/metagpt/provider/ollama_api.py b/metagpt/provider/ollama_api.py index 723abb574..2913eb1dd 100644 --- a/metagpt/provider/ollama_api.py +++ b/metagpt/provider/ollama_api.py @@ -5,6 +5,7 @@ import json from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.general_api_requestor import GeneralAPIRequestor @@ -49,7 +50,7 @@ class OllamaLLM(BaseLLM): chunk = chunk.decode(encoding) return json.loads(chunk) - async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> dict: + async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> dict: resp, _, _ = await self.client.arequest( method=self.http_method, url=self.suffix_url, @@ -61,10 +62,10 @@ class OllamaLLM(BaseLLM): self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=0) -> dict: + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: stream_resp, _, _ = await self.client.arequest( method=self.http_method, url=self.suffix_url, diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 5b4abae61..10b7749d6 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -25,6 +25,7 @@ from tenacity import ( ) from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_llm import BaseLLM from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA @@ -79,7 +80,7 @@ class OpenAILLM(BaseLLM): return params - async def _achat_completion_stream(self, messages: list[dict], timeout=0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> str: response: AsyncStream[ChatCompletionChunk] = await self.aclient.chat.completions.create( **self._cons_kwargs(messages, timeout=self.get_timeout(timeout)), stream=True ) @@ -109,7 +110,7 @@ class OpenAILLM(BaseLLM): self._update_costs(usage) return full_reply_content - def _cons_kwargs(self, messages: list[dict], timeout=0, **extra_kwargs) -> dict: + def _cons_kwargs(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT, **extra_kwargs) -> dict: kwargs = { "messages": messages, "max_tokens": self._get_max_tokens(messages), @@ -123,13 +124,13 @@ class OpenAILLM(BaseLLM): kwargs.update(extra_kwargs) return kwargs - async def _achat_completion(self, messages: list[dict], timeout=0) -> ChatCompletion: + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> ChatCompletion: kwargs = self._cons_kwargs(messages, timeout=self.get_timeout(timeout)) rsp: ChatCompletion = await self.aclient.chat.completions.create(**kwargs) self._update_costs(rsp.usage) return rsp - async def acompletion(self, messages: list[dict], timeout=0) -> ChatCompletion: + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> ChatCompletion: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) @retry( @@ -139,7 +140,7 @@ class OpenAILLM(BaseLLM): retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream=False, timeout=0) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout=USE_CONFIG_TIMEOUT) -> str: """when streaming, print each token in place.""" if stream: return await self._achat_completion_stream(messages, timeout=timeout) @@ -148,7 +149,7 @@ class OpenAILLM(BaseLLM): return self.get_choice_text(rsp) async def _achat_completion_function( - self, messages: list[dict], timeout: int = 0, **chat_configs + self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT, **chat_configs ) -> ChatCompletion: messages = process_message(messages) kwargs = self._cons_kwargs(messages=messages, timeout=self.get_timeout(timeout), **chat_configs) @@ -156,7 +157,7 @@ class OpenAILLM(BaseLLM): self._update_costs(rsp.usage) return rsp - async def aask_code(self, messages: list[dict], timeout: int = 0, **kwargs) -> dict: + async def aask_code(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT, **kwargs) -> dict: """Use function of tools to ask a code. Note: Keep kwargs consistent with https://platform.openai.com/docs/api-reference/chat/create diff --git a/metagpt/provider/qianfan_api.py b/metagpt/provider/qianfan_api.py index 7e0bf009e..3d78c8bfc 100644 --- a/metagpt/provider/qianfan_api.py +++ b/metagpt/provider/qianfan_api.py @@ -9,6 +9,7 @@ from qianfan import ChatCompletion from qianfan.resources.typing import JsonBody from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -107,15 +108,15 @@ class QianFanLLM(BaseLLM): self._update_costs(resp.body.get("usage", {})) return resp.body - async def _achat_completion(self, messages: list[dict], timeout: int = 0) -> JsonBody: + async def _achat_completion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> JsonBody: resp = await self.aclient.ado(**self._const_kwargs(messages=messages, stream=False)) self._update_costs(resp.body.get("usage", {})) return resp.body - async def acompletion(self, messages: list[dict], timeout: int = 0) -> JsonBody: + async def acompletion(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> JsonBody: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: resp = await self.aclient.ado(**self._const_kwargs(messages=messages, stream=True)) collected_content = [] usage = {} diff --git a/metagpt/provider/spark_api.py b/metagpt/provider/spark_api.py index 0f450ccf5..594267259 100644 --- a/metagpt/provider/spark_api.py +++ b/metagpt/provider/spark_api.py @@ -17,6 +17,7 @@ from wsgiref.handlers import format_date_time import websocket # 使用websocket_client from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import logger from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -31,19 +32,19 @@ class SparkLLM(BaseLLM): def get_choice_text(self, rsp: dict) -> str: return rsp["payload"]["choices"]["text"][-1]["content"] - async def _achat_completion_stream(self, messages: list[dict], timeout: int = 0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: pass - async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = 0) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, timeout: int = USE_CONFIG_TIMEOUT) -> str: # 不支持 # logger.warning("当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。") w = GetMessageFromWeb(messages, self.config) return w.run() - async def _achat_completion(self, messages: list[dict], timeout=0): + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): pass - async def acompletion(self, messages: list[dict], timeout=0): + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): # 不支持异步 w = GetMessageFromWeb(messages, self.config) return w.run() diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 8c5284770..2db441991 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -8,6 +8,7 @@ from typing import Optional from zhipuai.types.chat.chat_completion import Completion from metagpt.configs.llm_config import LLMConfig, LLMType +from metagpt.const import USE_CONFIG_TIMEOUT from metagpt.logs import log_llm_stream from metagpt.provider.base_llm import BaseLLM from metagpt.provider.llm_provider_registry import register_provider @@ -45,22 +46,22 @@ class ZhiPuAILLM(BaseLLM): kwargs = {"model": self.model, "messages": messages, "stream": stream, "temperature": 0.3} return kwargs - def completion(self, messages: list[dict], timeout=0) -> dict: + def completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict: resp: Completion = self.llm.chat.completions.create(**self._const_kwargs(messages)) usage = resp.usage.model_dump() self._update_costs(usage) return resp.model_dump() - async def _achat_completion(self, messages: list[dict], timeout=0) -> dict: + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict: resp = await self.llm.acreate(**self._const_kwargs(messages)) usage = resp.get("usage", {}) self._update_costs(usage) return resp - async def acompletion(self, messages: list[dict], timeout=0) -> dict: + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> dict: return await self._achat_completion(messages, timeout=self.get_timeout(timeout)) - async def _achat_completion_stream(self, messages: list[dict], timeout=0) -> str: + async def _achat_completion_stream(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT) -> str: response = await self.llm.acreate_stream(**self._const_kwargs(messages, stream=True)) collected_content = [] usage = {} From fcb2ea48638d1890faf277b87acc2e63e59521c5 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 21 Mar 2024 13:57:32 +0800 Subject: [PATCH 031/127] add params to search engine --- examples/search_with_specific_engine.py | 2 +- metagpt/configs/search_config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index 1eee762d5..48bcc67b8 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -13,7 +13,7 @@ async def main(): question = "What are the most interesting human facts?" search = Config.default().search - kwargs = {"api_key": search.api_key, "cse_id": search.cse_id, "proxy": None} + kwargs = {"api_key": search.api_key, "cse_id": search.cse_id, "proxy": None, "params": search.params} await Searcher(search_engine=SearchEngine(engine=search.api_type, **kwargs)).run(question) diff --git a/metagpt/configs/search_config.py b/metagpt/configs/search_config.py index af928b02a..38af4d91f 100644 --- a/metagpt/configs/search_config.py +++ b/metagpt/configs/search_config.py @@ -18,3 +18,4 @@ class SearchConfig(YamlModel): api_key: str = "" cse_id: str = "" # for google search_func: Optional[Callable] = None + params: dict = {} From 8486e520485359410a7560b23ee39895fdae6de2 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 21 Mar 2024 14:52:17 +0800 Subject: [PATCH 032/127] add params to search engine --- metagpt/actions/action_node.py | 2 +- metagpt/configs/search_config.py | 11 ++++++++++- tests/metagpt/tools/test_search_engine.py | 4 ---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 09da4a988..59559ad04 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -330,7 +330,7 @@ class ActionNode: def compile_to(self, i: Dict, schema, kv_sep) -> str: if schema == "json": - return json.dumps(i, indent=4) + return json.dumps(i, indent=4, ensure_ascii=False) elif schema == "markdown": return dict_to_markdown(i, kv_sep=kv_sep) else: diff --git a/metagpt/configs/search_config.py b/metagpt/configs/search_config.py index 38af4d91f..e28b14c99 100644 --- a/metagpt/configs/search_config.py +++ b/metagpt/configs/search_config.py @@ -7,6 +7,8 @@ """ from typing import Callable, Optional +from pydantic import Field + from metagpt.tools import SearchEngineType from metagpt.utils.yaml_model import YamlModel @@ -18,4 +20,11 @@ class SearchConfig(YamlModel): api_key: str = "" cse_id: str = "" # for google search_func: Optional[Callable] = None - params: dict = {} + params: dict = Field( + default_factory=lambda: { + "engine": "google", + "google_domain": "google.com", + "gl": "us", + "hl": "en", + } + ) diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index a1f03ef7b..964ead02f 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -11,7 +11,6 @@ from typing import Callable import pytest -from metagpt.config2 import config from metagpt.configs.search_config import SearchConfig from metagpt.logs import logger from metagpt.tools import SearchEngineType @@ -53,14 +52,11 @@ async def test_search_engine( search_engine_config = {"engine": search_engine_type, "run_func": run_func} if search_engine_type is SearchEngineType.SERPAPI_GOOGLE: - assert config.search search_engine_config["api_key"] = "mock-serpapi-key" elif search_engine_type is SearchEngineType.DIRECT_GOOGLE: - assert config.search search_engine_config["api_key"] = "mock-google-key" search_engine_config["cse_id"] = "mock-google-cse" elif search_engine_type is SearchEngineType.SERPER_GOOGLE: - assert config.search search_engine_config["api_key"] = "mock-serper-key" async def test(search_engine): From 8cda8c6f1b0f35e90ecca9467a3ee346b441860b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 14:54:14 +0800 Subject: [PATCH 033/127] fixbug: reset ocr_receipt.py --- examples/di/ocr_receipt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/di/ocr_receipt.py b/examples/di/ocr_receipt.py index 8b48be4f1..6b969ee99 100644 --- a/examples/di/ocr_receipt.py +++ b/examples/di/ocr_receipt.py @@ -6,7 +6,9 @@ async def main(): image_path = "image.jpg" language = "English" requirement = f"""This is a {language} receipt image. - Your goal is to perform OCR on images using PaddleOCR, then extract the total amount from ocr text results, and finally save as table. Image path: {image_path}. + Your goal is to perform OCR on images using PaddleOCR, output text content from the OCR results and discard + coordinates and confidence levels, then recognize the total amount from ocr text content, and finally save as table. + Image path: {image_path}. NOTE: The environments for Paddle and PaddleOCR are all ready and has been fully installed.""" di = DataInterpreter() From 587c13489320c62c8958f57bea54842e5a993614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 14:55:00 +0800 Subject: [PATCH 034/127] fixbug: reset ocr_receipt.py --- examples/di/ocr_receipt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/di/ocr_receipt.py b/examples/di/ocr_receipt.py index 6b969ee99..af54d519b 100644 --- a/examples/di/ocr_receipt.py +++ b/examples/di/ocr_receipt.py @@ -8,7 +8,7 @@ async def main(): requirement = f"""This is a {language} receipt image. Your goal is to perform OCR on images using PaddleOCR, output text content from the OCR results and discard coordinates and confidence levels, then recognize the total amount from ocr text content, and finally save as table. - Image path: {image_path}. + Image path: {image_path}. NOTE: The environments for Paddle and PaddleOCR are all ready and has been fully installed.""" di = DataInterpreter() From f051104bce125539f4ea24598b4b35a9e2c0ae69 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 21 Mar 2024 14:55:42 +0800 Subject: [PATCH 035/127] simplify code --- examples/search_with_specific_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index 48bcc67b8..276431ed8 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -13,7 +13,7 @@ async def main(): question = "What are the most interesting human facts?" search = Config.default().search - kwargs = {"api_key": search.api_key, "cse_id": search.cse_id, "proxy": None, "params": search.params} + kwargs = search.model_dump() await Searcher(search_engine=SearchEngine(engine=search.api_type, **kwargs)).run(question) From 9cfcfb1ea8bae086ed4fc5fe9d8365038c95e89e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 21 Mar 2024 15:14:14 +0800 Subject: [PATCH 036/127] feat: use --gitfile --- .gitignore | 2 +- metagpt/utils/tree.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6bc67fa61..1542bbb98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ ### Python template # Byte-compiled / optimized / DLL files -__pycache__/ +__pycache__ *.py[cod] *$py.class diff --git a/metagpt/utils/tree.py b/metagpt/utils/tree.py index fbf085e48..bd7922290 100644 --- a/metagpt/utils/tree.py +++ b/metagpt/utils/tree.py @@ -130,7 +130,7 @@ def _add_line(rows: List[str]) -> List[str]: def _execute_tree(root: Path, gitignore: str | Path) -> str: - args = ["--gitignore", str(gitignore)] if gitignore else [] + args = ["--gitfile", str(gitignore)] if gitignore else [] try: result = subprocess.run(["tree"] + args + [str(root)], capture_output=True, text=True, check=True) if result.returncode != 0: From e53188f8981d7748343e902821b544a59170fd6b Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Thu, 21 Mar 2024 16:39:53 +0800 Subject: [PATCH 037/127] fix potential pydantic ValidationError --- metagpt/rag/engines/simple.py | 4 +++- metagpt/rag/factories/llm.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/metagpt/rag/engines/simple.py b/metagpt/rag/engines/simple.py index 02f9ca7b1..5c5810308 100644 --- a/metagpt/rag/engines/simple.py +++ b/metagpt/rag/engines/simple.py @@ -130,10 +130,12 @@ class SimpleEngine(RetrieverQueryEngine): retriever_configs: Configuration for retrievers. If more than one config, will use SimpleHybridRetriever. ranker_configs: Configuration for rankers. """ + objs = objs or [] + retriever_configs = retriever_configs or [] + if not objs and any(isinstance(config, BM25RetrieverConfig) for config in retriever_configs): raise ValueError("In BM25RetrieverConfig, Objs must not be empty.") - objs = objs or [] nodes = [ObjectNode(text=obj.rag_key(), metadata=ObjectNode.get_obj_metadata(obj)) for obj in objs] index = VectorStoreIndex( nodes=nodes, diff --git a/metagpt/rag/factories/llm.py b/metagpt/rag/factories/llm.py index 1cdbab14d..17c499b76 100644 --- a/metagpt/rag/factories/llm.py +++ b/metagpt/rag/factories/llm.py @@ -33,7 +33,9 @@ class RAGLLM(CustomLLM): @property def metadata(self) -> LLMMetadata: """Get LLM metadata.""" - return LLMMetadata(context_window=self.context_window, num_output=self.num_output, model_name=self.model_name) + return LLMMetadata( + context_window=self.context_window, num_output=self.num_output, model_name=self.model_name or "unknown" + ) @llm_completion_callback() def complete(self, prompt: str, **kwargs: Any) -> CompletionResponse: From 6e30b42cc0ee343ce7f9a706632b4fac1c71744a Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Thu, 21 Mar 2024 16:50:59 +0800 Subject: [PATCH 038/127] add FLAREEngine and ColbertRerank --- metagpt/rag/engines/__init__.py | 3 ++- metagpt/rag/engines/flare.py | 0 metagpt/rag/factories/ranker.py | 10 ++++++---- metagpt/rag/schema.py | 6 ++++++ requirements.txt | 1 + 5 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 metagpt/rag/engines/flare.py diff --git a/metagpt/rag/engines/__init__.py b/metagpt/rag/engines/__init__.py index 373181384..93699db88 100644 --- a/metagpt/rag/engines/__init__.py +++ b/metagpt/rag/engines/__init__.py @@ -1,5 +1,6 @@ """Engines init""" from metagpt.rag.engines.simple import SimpleEngine +from metagpt.rag.engines.flare import FLAREEngine -__all__ = ["SimpleEngine"] +__all__ = ["SimpleEngine", "FLAREEngine"] diff --git a/metagpt/rag/engines/flare.py b/metagpt/rag/engines/flare.py new file mode 100644 index 000000000..e69de29bb diff --git a/metagpt/rag/factories/ranker.py b/metagpt/rag/factories/ranker.py index f05599e15..15dc55bf9 100644 --- a/metagpt/rag/factories/ranker.py +++ b/metagpt/rag/factories/ranker.py @@ -3,18 +3,17 @@ from llama_index.core.llms import LLM from llama_index.core.postprocessor import LLMRerank from llama_index.core.postprocessor.types import BaseNodePostprocessor +from llama_index.postprocessor.colbert_rerank import ColbertRerank from metagpt.rag.factories.base import ConfigBasedFactory -from metagpt.rag.schema import BaseRankerConfig, LLMRankerConfig +from metagpt.rag.schema import BaseRankerConfig, ColbertRerankConfig, LLMRankerConfig class RankerFactory(ConfigBasedFactory): """Modify creators for dynamically instance implementation.""" def __init__(self): - creators = { - LLMRankerConfig: self._create_llm_ranker, - } + creators = {LLMRankerConfig: self._create_llm_ranker, ColbertRerankConfig: self._create_colbert_ranker} super().__init__(creators) def get_rankers(self, configs: list[BaseRankerConfig] = None, **kwargs) -> list[BaseNodePostprocessor]: @@ -28,6 +27,9 @@ class RankerFactory(ConfigBasedFactory): config.llm = self._extract_llm(config, **kwargs) return LLMRerank(**config.model_dump()) + def _create_colbert_ranker(self, config: ColbertRerankConfig, **kwargs) -> LLMRerank: + return ColbertRerank(**config.model_dump()) + def _extract_llm(self, config: BaseRankerConfig = None, **kwargs) -> LLM: return self._val_from_config_or_kwargs("llm", config, **kwargs) diff --git a/metagpt/rag/schema.py b/metagpt/rag/schema.py index e98a6fc89..cacce3178 100644 --- a/metagpt/rag/schema.py +++ b/metagpt/rag/schema.py @@ -84,6 +84,12 @@ class LLMRankerConfig(BaseRankerConfig): ) +class ColbertRerankConfig(BaseRankerConfig): + model: str = Field(default="colbert-ir/colbertv2.0", description="Colbert model name.") + device: str = Field(default="cpu", description="Device to use for sentence transformer.") + keep_retrieval_score: bool = Field(default=False, description="Whether to keep the retrieval score in metadata.") + + class BaseIndexConfig(BaseModel): """Common config for index. diff --git a/requirements.txt b/requirements.txt index 3e545d146..9bcd2a45b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ llama-index-readers-file==0.1.4 llama-index-retrievers-bm25==0.1.3 llama-index-vector-stores-faiss==0.1.1 llama-index-vector-stores-elasticsearch==0.1.5 +llama-index-postprocessor-colbert-rerank==0.1.1 chromadb==0.4.23 loguru==0.6.0 meilisearch==0.21.0 From 73953c025d16ec99994f2262fa8cae9b6aa0f58c Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Thu, 21 Mar 2024 17:11:18 +0800 Subject: [PATCH 039/127] add FLAREEngine and ColbertRerank --- metagpt/rag/engines/flare.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/metagpt/rag/engines/flare.py b/metagpt/rag/engines/flare.py index e69de29bb..3fd1bf84b 100644 --- a/metagpt/rag/engines/flare.py +++ b/metagpt/rag/engines/flare.py @@ -0,0 +1,9 @@ +"""FLARE Engine. + +Use llamaindex's FLAREInstructQueryEngine, which accepts other engines as parameters. +For example, Create a simple engine, and then pass it to FLAREEngine. +""" + +from llama_index.core.query_engine import ( # noqa: F401 + FLAREInstructQueryEngine as FLAREEngine, +) From 7c1c4b2a35659520e4f8e779acbeba54dd1cab91 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Thu, 21 Mar 2024 17:14:22 +0800 Subject: [PATCH 040/127] update comment --- metagpt/rag/engines/flare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/rag/engines/flare.py b/metagpt/rag/engines/flare.py index 3fd1bf84b..dc05bd3dd 100644 --- a/metagpt/rag/engines/flare.py +++ b/metagpt/rag/engines/flare.py @@ -1,6 +1,6 @@ """FLARE Engine. -Use llamaindex's FLAREInstructQueryEngine, which accepts other engines as parameters. +Use llamaindex's FLAREInstructQueryEngine as FLAREEngine, which accepts other engines as parameters. For example, Create a simple engine, and then pass it to FLAREEngine. """ From 12551ab60cc9ee0e59cf31a1a29c24e13d9e7abd Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 21 Mar 2024 21:57:59 +0800 Subject: [PATCH 041/127] fix state value extract for https://github.com/geekan/MetaGPT/issues/1067 --- metagpt/utils/repair_llm_raw_output.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index b8756e8c6..17e095c5f 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -340,7 +340,9 @@ def extract_state_value_from_output(content: str) -> str: content (str): llm's output from `Role._think` """ content = content.strip() # deal the output cases like " 0", "0\n" and so on. - pattern = r"([0-9])" # TODO find the number using a more proper method not just extract from content using pattern + pattern = ( + r"(? 0 else "-1" From e88aa63bf0f49e3e90a33bf818a91f45ba950752 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Fri, 22 Mar 2024 10:28:11 +0800 Subject: [PATCH 042/127] make the rag module optional --- requirements.txt | 8 -------- setup.py | 11 ++++++++++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 83565278b..a447eef13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,14 +10,6 @@ typer==0.9.0 # godot==0.1.1 # google_api_python_client==2.93.0 # Used by search_engine.py lancedb==0.4.0 -llama-index-core==0.10.15 -llama-index-embeddings-azure-openai==0.1.6 -llama-index-embeddings-openai==0.1.5 -llama-index-llms-azure-openai==0.1.4 -llama-index-readers-file==0.1.4 -llama-index-retrievers-bm25==0.1.3 -llama-index-vector-stores-faiss==0.1.1 -chromadb==0.4.23 loguru==0.6.0 meilisearch==0.21.0 numpy==1.24.3 diff --git a/setup.py b/setup.py index df9bedc9b..f834b4c44 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,16 @@ extras_require = { "search-google": ["google-api-python-client==2.94.0"], "search-ddg": ["duckduckgo-search~=4.1.1"], "ocr": ["paddlepaddle==2.4.2", "paddleocr>=2.0.1", "tabulate==0.9.0"], + "rag": [ + "llama-index-core==0.10.15", + "llama-index-embeddings-azure-openai==0.1.6", + "llama-index-embeddings-openai==0.1.5", + "llama-index-llms-azure-openai==0.1.4", + "llama-index-readers-file==0.1.4", + "llama-index-retrievers-bm25==0.1.3", + "llama-index-vector-stores-faiss==0.1.1", + "chromadb==0.4.23", + ], } extras_require["test"] = [ @@ -42,7 +52,6 @@ extras_require["test"] = [ "connexion[uvicorn]~=3.0.5", "azure-cognitiveservices-speech~=1.31.0", "aioboto3~=11.3.0", - "chromadb==0.4.23", "gradio==3.0.0", "grpcio-status==1.48.2", "pylint==3.0.3", From bac81ee64b82feae76ef07cf8068a26d89974ca3 Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 22 Mar 2024 10:58:22 +0800 Subject: [PATCH 043/127] add custom tool example --- examples/di/custom_tool.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 examples/di/custom_tool.py diff --git a/examples/di/custom_tool.py b/examples/di/custom_tool.py new file mode 100644 index 000000000..3f955529c --- /dev/null +++ b/examples/di/custom_tool.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/3/22 10:54 +@Author : alexanderwu +@File : custom_tool.py +""" + +from metagpt.roles.di.data_interpreter import DataInterpreter +from metagpt.tools.tool_registry import register_tool + + +@register_tool() +def magic_function(arg1: str, arg2: int) -> dict: + """ + The magic function that does something. + + Args: + arg1 (str): ... + arg2 (int): ... + + Returns: + dict: ... + """ + return {"arg1": arg1 * 3, "arg2": arg2 * 5} + + +async def main(): + di = DataInterpreter(tools=["magic_function"]) + await di.run("Just call the magic function with arg1 'A' and arg2 'B'. Tell me the result.") + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) From aea06852bad093aa0bb1f68dacba7d5384d9c429 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Fri, 22 Mar 2024 11:15:31 +0800 Subject: [PATCH 044/127] make the rag module optional --- metagpt/environment/__init__.py | 3 +-- metagpt/roles/__init__.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/metagpt/environment/__init__.py b/metagpt/environment/__init__.py index 692672fa7..28981f2f8 100644 --- a/metagpt/environment/__init__.py +++ b/metagpt/environment/__init__.py @@ -4,10 +4,9 @@ from metagpt.environment.base_env import Environment from metagpt.environment.android_env.android_env import AndroidEnv -from metagpt.environment.mincraft_env.mincraft_env import MincraftExtEnv from metagpt.environment.werewolf_env.werewolf_env import WerewolfEnv from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv from metagpt.environment.software_env.software_env import SoftwareEnv -__all__ = ["AndroidEnv", "MincraftExtEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"] +__all__ = ["AndroidEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"] diff --git a/metagpt/roles/__init__.py b/metagpt/roles/__init__.py index f033a5dfa..08a0406b3 100644 --- a/metagpt/roles/__init__.py +++ b/metagpt/roles/__init__.py @@ -14,7 +14,6 @@ from metagpt.roles.engineer import Engineer from metagpt.roles.qa_engineer import QaEngineer from metagpt.roles.searcher import Searcher from metagpt.roles.sales import Sales -from metagpt.roles.customer_service import CustomerService __all__ = [ @@ -26,5 +25,4 @@ __all__ = [ "QaEngineer", "Searcher", "Sales", - "CustomerService", ] From 0c27be629b1f9b38a414b055b77ded284ba33a01 Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 22 Mar 2024 11:19:20 +0800 Subject: [PATCH 045/127] add custom tool example --- examples/di/custom_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/di/custom_tool.py b/examples/di/custom_tool.py index 3f955529c..cbe7380c7 100644 --- a/examples/di/custom_tool.py +++ b/examples/di/custom_tool.py @@ -27,7 +27,7 @@ def magic_function(arg1: str, arg2: int) -> dict: async def main(): di = DataInterpreter(tools=["magic_function"]) - await di.run("Just call the magic function with arg1 'A' and arg2 'B'. Tell me the result.") + await di.run("Just call the magic function with arg1 'A' and arg2 2. Tell me the result.") if __name__ == "__main__": From bf44a699585a3bddfd8665f565c8f1c6b853f109 Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 22 Mar 2024 11:25:22 +0800 Subject: [PATCH 046/127] fix minecraft typo --- metagpt/environment/README.md | 2 +- metagpt/environment/__init__.py | 4 ++-- metagpt/environment/base_env.py | 2 +- .../{mincraft_env => minecraft_env}/__init__.py | 0 .../{mincraft_env => minecraft_env}/const.py | 4 ++-- .../mincraft_env.py => minecraft_env/minecraft_env.py} | 10 +++++----- .../minecraft_ext_env.py} | 10 +++++----- .../mineflayer/.gitignore | 0 .../mineflayer/.prettierignore | 0 .../mineflayer/.prettierrc.json | 0 .../mineflayer/index.js | 0 .../mineflayer/lib/observation/base.js | 0 .../mineflayer/lib/observation/chests.js | 0 .../mineflayer/lib/observation/inventory.js | 0 .../mineflayer/lib/observation/onChat.js | 0 .../mineflayer/lib/observation/onError.js | 0 .../mineflayer/lib/observation/onSave.js | 0 .../mineflayer/lib/observation/status.js | 0 .../mineflayer/lib/observation/voxels.js | 0 .../mineflayer/lib/skillLoader.js | 0 .../mineflayer/lib/utils.js | 0 .../mineflayer/mineflayer-collectblock/.gitignore | 0 .../mineflayer/mineflayer-collectblock/LICENSE | 0 .../mineflayer/mineflayer-collectblock/README.md | 0 .../mineflayer/mineflayer-collectblock/_config.yml | 0 .../mineflayer/mineflayer-collectblock/docs/api.md | 0 .../mineflayer-collectblock/examples/collector.js | 0 .../mineflayer-collectblock/examples/oreMiner.js | 0 .../mineflayer-collectblock/examples/storageBot.js | 0 .../mineflayer/mineflayer-collectblock/package.json | 0 .../mineflayer-collectblock/src/BlockVeins.ts | 0 .../mineflayer-collectblock/src/CollectBlock.ts | 0 .../mineflayer-collectblock/src/Inventory.ts | 0 .../mineflayer/mineflayer-collectblock/src/Targets.ts | 0 .../mineflayer-collectblock/src/TaskQueue.ts | 0 .../mineflayer-collectblock/src/TemporarySubscriber.ts | 0 .../mineflayer/mineflayer-collectblock/src/Util.ts | 0 .../mineflayer/mineflayer-collectblock/src/index.ts | 0 .../mineflayer/mineflayer-collectblock/tsconfig.json | 0 .../mineflayer/package.json | 0 .../{mincraft_env => minecraft_env}/process_monitor.py | 0 metagpt/utils/common.py | 2 +- .../environment/mincraft_env/test_mincraft_ext_env.py | 10 +++++----- 43 files changed, 22 insertions(+), 22 deletions(-) rename metagpt/environment/{mincraft_env => minecraft_env}/__init__.py (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/const.py (92%) rename metagpt/environment/{mincraft_env/mincraft_env.py => minecraft_env/minecraft_env.py} (98%) rename metagpt/environment/{mincraft_env/mincraft_ext_env.py => minecraft_env/minecraft_ext_env.py} (95%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/.gitignore (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/.prettierignore (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/.prettierrc.json (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/index.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/lib/observation/base.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/lib/observation/chests.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/lib/observation/inventory.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/lib/observation/onChat.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/lib/observation/onError.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/lib/observation/onSave.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/lib/observation/status.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/lib/observation/voxels.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/lib/skillLoader.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/lib/utils.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/.gitignore (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/LICENSE (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/README.md (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/_config.yml (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/docs/api.md (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/examples/collector.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/examples/oreMiner.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/examples/storageBot.js (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/package.json (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/src/BlockVeins.ts (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/src/CollectBlock.ts (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/src/Inventory.ts (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/src/Targets.ts (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/src/TaskQueue.ts (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/src/TemporarySubscriber.ts (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/src/Util.ts (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/src/index.ts (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/mineflayer-collectblock/tsconfig.json (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/mineflayer/package.json (100%) rename metagpt/environment/{mincraft_env => minecraft_env}/process_monitor.py (100%) diff --git a/metagpt/environment/README.md b/metagpt/environment/README.md index 9476ac75a..ff56b73f1 100644 --- a/metagpt/environment/README.md +++ b/metagpt/environment/README.md @@ -34,5 +34,5 @@ # do a `tap` action on the screen ## TODO - add android app operation assistant under `examples/android_assistant` - migrate roles/actions of werewolf game from old version into current version -- migrate roles/actions of mincraft game from old version into current version +- migrate roles/actions of minecraft game from old version into current version - migrate roles/actions of stanford_town game from old version into current version diff --git a/metagpt/environment/__init__.py b/metagpt/environment/__init__.py index 692672fa7..a844827d8 100644 --- a/metagpt/environment/__init__.py +++ b/metagpt/environment/__init__.py @@ -4,10 +4,10 @@ from metagpt.environment.base_env import Environment from metagpt.environment.android_env.android_env import AndroidEnv -from metagpt.environment.mincraft_env.mincraft_env import MincraftExtEnv +from metagpt.environment.minecraft_env.minecraft_env import MinecraftExtEnv from metagpt.environment.werewolf_env.werewolf_env import WerewolfEnv from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv from metagpt.environment.software_env.software_env import SoftwareEnv -__all__ = ["AndroidEnv", "MincraftExtEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"] +__all__ = ["AndroidEnv", "MinecraftExtEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"] diff --git a/metagpt/environment/base_env.py b/metagpt/environment/base_env.py index 14023e3b7..942bf2409 100644 --- a/metagpt/environment/base_env.py +++ b/metagpt/environment/base_env.py @@ -26,7 +26,7 @@ class EnvType(Enum): ANDROID = "Android" GYM = "Gym" WEREWOLF = "Werewolf" - MINCRAFT = "Mincraft" + MINECRAFT = "Minecraft" STANFORDTOWN = "StanfordTown" diff --git a/metagpt/environment/mincraft_env/__init__.py b/metagpt/environment/minecraft_env/__init__.py similarity index 100% rename from metagpt/environment/mincraft_env/__init__.py rename to metagpt/environment/minecraft_env/__init__.py diff --git a/metagpt/environment/mincraft_env/const.py b/metagpt/environment/minecraft_env/const.py similarity index 92% rename from metagpt/environment/mincraft_env/const.py rename to metagpt/environment/minecraft_env/const.py index a7222f9cd..8ac15decc 100644 --- a/metagpt/environment/mincraft_env/const.py +++ b/metagpt/environment/minecraft_env/const.py @@ -4,8 +4,8 @@ from metagpt.const import METAGPT_ROOT -# For Mincraft Game Agent -MC_CKPT_DIR = METAGPT_ROOT / "data/mincraft/ckpt" +# For Minecraft Game Agent +MC_CKPT_DIR = METAGPT_ROOT / "data/minecraft/ckpt" MC_LOG_DIR = METAGPT_ROOT / "logs" MC_DEFAULT_WARMUP = { "context": 15, diff --git a/metagpt/environment/mincraft_env/mincraft_env.py b/metagpt/environment/minecraft_env/minecraft_env.py similarity index 98% rename from metagpt/environment/mincraft_env/mincraft_env.py rename to metagpt/environment/minecraft_env/minecraft_env.py index fdc477164..26d4d03a8 100644 --- a/metagpt/environment/mincraft_env/mincraft_env.py +++ b/metagpt/environment/minecraft_env/minecraft_env.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : MG Mincraft Env +# @Desc : MG Minecraft Env # refs to `voyager voyager.py` import json @@ -12,15 +12,15 @@ from pydantic import ConfigDict, Field from metagpt.config2 import config as CONFIG from metagpt.environment.base_env import Environment -from metagpt.environment.mincraft_env.const import MC_CKPT_DIR -from metagpt.environment.mincraft_env.mincraft_ext_env import MincraftExtEnv +from metagpt.environment.minecraft_env.const import MC_CKPT_DIR +from metagpt.environment.minecraft_env.minecraft_ext_env import MinecraftExtEnv from metagpt.logs import logger from metagpt.rag.vector_stores.chroma import ChromaVectorStore from metagpt.utils.common import load_mc_skills_code, read_json_file, write_json_file -class MincraftEnv(Environment, MincraftExtEnv): - """MincraftEnv, including shared memory of cache and information between roles""" +class MinecraftEnv(Environment, MinecraftExtEnv): + """MinecraftEnv, including shared memory of cache and information between roles""" model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/metagpt/environment/mincraft_env/mincraft_ext_env.py b/metagpt/environment/minecraft_env/minecraft_ext_env.py similarity index 95% rename from metagpt/environment/mincraft_env/mincraft_ext_env.py rename to metagpt/environment/minecraft_env/minecraft_ext_env.py index b86250d8c..3b793079f 100644 --- a/metagpt/environment/mincraft_env/mincraft_ext_env.py +++ b/metagpt/environment/minecraft_env/minecraft_ext_env.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : The Mincraft external environment to integrate with Mincraft game +# @Desc : The Minecraft external environment to integrate with Minecraft game # refs to `voyager bridge.py` import json @@ -11,18 +11,18 @@ import requests from pydantic import ConfigDict, Field, model_validator from metagpt.environment.base_env import ExtEnv, mark_as_writeable -from metagpt.environment.mincraft_env.const import ( +from metagpt.environment.minecraft_env.const import ( MC_CKPT_DIR, MC_CORE_INVENTORY_ITEMS, MC_CURRICULUM_OB, MC_DEFAULT_WARMUP, METAGPT_ROOT, ) -from metagpt.environment.mincraft_env.process_monitor import SubprocessMonitor +from metagpt.environment.minecraft_env.process_monitor import SubprocessMonitor from metagpt.logs import logger -class MincraftExtEnv(ExtEnv): +class MinecraftExtEnv(ExtEnv): model_config = ConfigDict(arbitrary_types_allowed=True) mc_port: Optional[int] = Field(default=None) @@ -48,7 +48,7 @@ class MincraftExtEnv(ExtEnv): self.mineflayer = SubprocessMonitor( commands=[ "node", - METAGPT_ROOT.joinpath("metagpt", "environment", "mincraft_env", "mineflayer", "index.js"), + METAGPT_ROOT.joinpath("metagpt", "environment", "minecraft_env", "mineflayer", "index.js"), str(self.server_port), ], name="mineflayer", diff --git a/metagpt/environment/mincraft_env/mineflayer/.gitignore b/metagpt/environment/minecraft_env/mineflayer/.gitignore similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/.gitignore rename to metagpt/environment/minecraft_env/mineflayer/.gitignore diff --git a/metagpt/environment/mincraft_env/mineflayer/.prettierignore b/metagpt/environment/minecraft_env/mineflayer/.prettierignore similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/.prettierignore rename to metagpt/environment/minecraft_env/mineflayer/.prettierignore diff --git a/metagpt/environment/mincraft_env/mineflayer/.prettierrc.json b/metagpt/environment/minecraft_env/mineflayer/.prettierrc.json similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/.prettierrc.json rename to metagpt/environment/minecraft_env/mineflayer/.prettierrc.json diff --git a/metagpt/environment/mincraft_env/mineflayer/index.js b/metagpt/environment/minecraft_env/mineflayer/index.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/index.js rename to metagpt/environment/minecraft_env/mineflayer/index.js diff --git a/metagpt/environment/mincraft_env/mineflayer/lib/observation/base.js b/metagpt/environment/minecraft_env/mineflayer/lib/observation/base.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/lib/observation/base.js rename to metagpt/environment/minecraft_env/mineflayer/lib/observation/base.js diff --git a/metagpt/environment/mincraft_env/mineflayer/lib/observation/chests.js b/metagpt/environment/minecraft_env/mineflayer/lib/observation/chests.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/lib/observation/chests.js rename to metagpt/environment/minecraft_env/mineflayer/lib/observation/chests.js diff --git a/metagpt/environment/mincraft_env/mineflayer/lib/observation/inventory.js b/metagpt/environment/minecraft_env/mineflayer/lib/observation/inventory.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/lib/observation/inventory.js rename to metagpt/environment/minecraft_env/mineflayer/lib/observation/inventory.js diff --git a/metagpt/environment/mincraft_env/mineflayer/lib/observation/onChat.js b/metagpt/environment/minecraft_env/mineflayer/lib/observation/onChat.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/lib/observation/onChat.js rename to metagpt/environment/minecraft_env/mineflayer/lib/observation/onChat.js diff --git a/metagpt/environment/mincraft_env/mineflayer/lib/observation/onError.js b/metagpt/environment/minecraft_env/mineflayer/lib/observation/onError.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/lib/observation/onError.js rename to metagpt/environment/minecraft_env/mineflayer/lib/observation/onError.js diff --git a/metagpt/environment/mincraft_env/mineflayer/lib/observation/onSave.js b/metagpt/environment/minecraft_env/mineflayer/lib/observation/onSave.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/lib/observation/onSave.js rename to metagpt/environment/minecraft_env/mineflayer/lib/observation/onSave.js diff --git a/metagpt/environment/mincraft_env/mineflayer/lib/observation/status.js b/metagpt/environment/minecraft_env/mineflayer/lib/observation/status.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/lib/observation/status.js rename to metagpt/environment/minecraft_env/mineflayer/lib/observation/status.js diff --git a/metagpt/environment/mincraft_env/mineflayer/lib/observation/voxels.js b/metagpt/environment/minecraft_env/mineflayer/lib/observation/voxels.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/lib/observation/voxels.js rename to metagpt/environment/minecraft_env/mineflayer/lib/observation/voxels.js diff --git a/metagpt/environment/mincraft_env/mineflayer/lib/skillLoader.js b/metagpt/environment/minecraft_env/mineflayer/lib/skillLoader.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/lib/skillLoader.js rename to metagpt/environment/minecraft_env/mineflayer/lib/skillLoader.js diff --git a/metagpt/environment/mincraft_env/mineflayer/lib/utils.js b/metagpt/environment/minecraft_env/mineflayer/lib/utils.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/lib/utils.js rename to metagpt/environment/minecraft_env/mineflayer/lib/utils.js diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/.gitignore b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/.gitignore similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/.gitignore rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/.gitignore diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/LICENSE b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/LICENSE similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/LICENSE rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/LICENSE diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/README.md b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/README.md similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/README.md rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/README.md diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/_config.yml b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/_config.yml similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/_config.yml rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/_config.yml diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/docs/api.md b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/docs/api.md similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/docs/api.md rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/docs/api.md diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/examples/collector.js b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/collector.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/examples/collector.js rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/collector.js diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/examples/oreMiner.js b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/oreMiner.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/examples/oreMiner.js rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/oreMiner.js diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/examples/storageBot.js b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/storageBot.js similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/examples/storageBot.js rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/storageBot.js diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/package.json b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/package.json similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/package.json rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/package.json diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/BlockVeins.ts b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/BlockVeins.ts similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/BlockVeins.ts rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/BlockVeins.ts diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/CollectBlock.ts b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/CollectBlock.ts similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/CollectBlock.ts rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/CollectBlock.ts diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/Inventory.ts b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Inventory.ts similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/Inventory.ts rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Inventory.ts diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/Targets.ts b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Targets.ts similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/Targets.ts rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Targets.ts diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/TaskQueue.ts b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/TaskQueue.ts similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/TaskQueue.ts rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/TaskQueue.ts diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/TemporarySubscriber.ts b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/TemporarySubscriber.ts similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/TemporarySubscriber.ts rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/TemporarySubscriber.ts diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/Util.ts b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Util.ts similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/Util.ts rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Util.ts diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/index.ts b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/index.ts similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/index.ts rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/index.ts diff --git a/metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/tsconfig.json b/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/tsconfig.json similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/tsconfig.json rename to metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/tsconfig.json diff --git a/metagpt/environment/mincraft_env/mineflayer/package.json b/metagpt/environment/minecraft_env/mineflayer/package.json similarity index 100% rename from metagpt/environment/mincraft_env/mineflayer/package.json rename to metagpt/environment/minecraft_env/mineflayer/package.json diff --git a/metagpt/environment/mincraft_env/process_monitor.py b/metagpt/environment/minecraft_env/process_monitor.py similarity index 100% rename from metagpt/environment/mincraft_env/process_monitor.py rename to metagpt/environment/minecraft_env/process_monitor.py diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 7493712c2..ee0f74989 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -765,7 +765,7 @@ def is_coroutine_func(func: Callable) -> bool: def load_mc_skills_code(skill_names: list[str] = None, skills_dir: Path = None) -> list[str]: - """load mincraft skill from js files""" + """load minecraft skill from js files""" if not skills_dir: skills_dir = Path(__file__).parent.absolute() if skill_names is None: diff --git a/tests/metagpt/environment/mincraft_env/test_mincraft_ext_env.py b/tests/metagpt/environment/mincraft_env/test_mincraft_ext_env.py index ad3376141..ed08b719f 100644 --- a/tests/metagpt/environment/mincraft_env/test_mincraft_ext_env.py +++ b/tests/metagpt/environment/mincraft_env/test_mincraft_ext_env.py @@ -1,14 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the unittest of MincraftExtEnv +# @Desc : the unittest of MinecraftExtEnv -from metagpt.environment.mincraft_env.const import MC_CKPT_DIR -from metagpt.environment.mincraft_env.mincraft_ext_env import MincraftExtEnv +from metagpt.environment.minecraft_env.const import MC_CKPT_DIR +from metagpt.environment.minecraft_env.minecraft_ext_env import MinecraftExtEnv -def test_mincraft_ext_env(): - ext_env = MincraftExtEnv() +def test_minecraft_ext_env(): + ext_env = MinecraftExtEnv() assert ext_env.server, f"{ext_env.server_host}:{ext_env.server_port}" assert MC_CKPT_DIR.joinpath("skill/code").exists() assert ext_env.warm_up.get("optional_inventory_items") == 7 From 82fa7f487c444802ce3cc050add7b360158faaa7 Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 22 Mar 2024 11:55:14 +0800 Subject: [PATCH 047/127] fix minecraft typo --- .../environment/{mincraft_env => minecraft_env}/__init__.py | 0 .../test_minecraft_ext_env.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/metagpt/environment/{mincraft_env => minecraft_env}/__init__.py (100%) rename tests/metagpt/environment/{mincraft_env/test_mincraft_ext_env.py => minecraft_env/test_minecraft_ext_env.py} (100%) diff --git a/tests/metagpt/environment/mincraft_env/__init__.py b/tests/metagpt/environment/minecraft_env/__init__.py similarity index 100% rename from tests/metagpt/environment/mincraft_env/__init__.py rename to tests/metagpt/environment/minecraft_env/__init__.py diff --git a/tests/metagpt/environment/mincraft_env/test_mincraft_ext_env.py b/tests/metagpt/environment/minecraft_env/test_minecraft_ext_env.py similarity index 100% rename from tests/metagpt/environment/mincraft_env/test_mincraft_ext_env.py rename to tests/metagpt/environment/minecraft_env/test_minecraft_ext_env.py From 34a3c1ad0753316188655f8b90fa1996a2f95523 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Fri, 22 Mar 2024 15:58:59 +0800 Subject: [PATCH 048/127] upgrade llama-index-vector-stores-elasticsearch --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9bcd2a45b..6e84f4612 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ llama-index-llms-azure-openai==0.1.4 llama-index-readers-file==0.1.4 llama-index-retrievers-bm25==0.1.3 llama-index-vector-stores-faiss==0.1.1 -llama-index-vector-stores-elasticsearch==0.1.5 +llama-index-vector-stores-elasticsearch==0.1.6 llama-index-postprocessor-colbert-rerank==0.1.1 chromadb==0.4.23 loguru==0.6.0 From 092ef26425279f76318366e880246c85739940fb Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Fri, 22 Mar 2024 17:30:35 +0800 Subject: [PATCH 049/127] support elasticsearch text only --- metagpt/rag/factories/base.py | 2 +- metagpt/rag/factories/index.py | 2 ++ metagpt/rag/factories/retriever.py | 2 ++ metagpt/rag/schema.py | 15 ++++++++++++++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/metagpt/rag/factories/base.py b/metagpt/rag/factories/base.py index 8f8155914..fbdfbf1a8 100644 --- a/metagpt/rag/factories/base.py +++ b/metagpt/rag/factories/base.py @@ -41,7 +41,7 @@ class ConfigBasedFactory(GenericFactory): if creator: return creator(key, **kwargs) - raise ValueError(f"Unknown config: {key}") + raise ValueError(f"Unknown config: `{type(key)}`, {key}") @staticmethod def _val_from_config_or_kwargs(key: str, config: object = None, **kwargs) -> Any: diff --git a/metagpt/rag/factories/index.py b/metagpt/rag/factories/index.py index 5ab7992a0..f200fc94f 100644 --- a/metagpt/rag/factories/index.py +++ b/metagpt/rag/factories/index.py @@ -14,6 +14,7 @@ from metagpt.rag.schema import ( BM25IndexConfig, ChromaIndexConfig, ElasticsearchIndexConfig, + ElasticsearchKeywordIndexConfig, FAISSIndexConfig, ) from metagpt.rag.vector_stores.chroma import ChromaVectorStore @@ -26,6 +27,7 @@ class RAGIndexFactory(ConfigBasedFactory): ChromaIndexConfig: self._create_chroma, BM25IndexConfig: self._create_bm25, ElasticsearchIndexConfig: self._create_es, + ElasticsearchKeywordIndexConfig: self._create_es, } super().__init__(creators) diff --git a/metagpt/rag/factories/retriever.py b/metagpt/rag/factories/retriever.py index 47ceadf00..a107d9573 100644 --- a/metagpt/rag/factories/retriever.py +++ b/metagpt/rag/factories/retriever.py @@ -20,6 +20,7 @@ from metagpt.rag.schema import ( BaseRetrieverConfig, BM25RetrieverConfig, ChromaRetrieverConfig, + ElasticsearchKeywordRetrieverConfig, ElasticsearchRetrieverConfig, FAISSRetrieverConfig, IndexRetrieverConfig, @@ -36,6 +37,7 @@ class RetrieverFactory(ConfigBasedFactory): BM25RetrieverConfig: self._create_bm25_retriever, ChromaRetrieverConfig: self._create_chroma_retriever, ElasticsearchRetrieverConfig: self._create_es_retriever, + ElasticsearchKeywordRetrieverConfig: self._create_es_retriever, } super().__init__(creators) diff --git a/metagpt/rag/schema.py b/metagpt/rag/schema.py index cacce3178..cb5f1aac0 100644 --- a/metagpt/rag/schema.py +++ b/metagpt/rag/schema.py @@ -59,12 +59,19 @@ class ElasticsearchStoreConfig(BaseModel): class ElasticsearchRetrieverConfig(IndexRetrieverConfig): - """Config for Elasticsearch-based retrievers.""" + """Config for Elasticsearch-based retrievers. Support both vector and text.""" store_config: ElasticsearchStoreConfig = Field(..., description="ElasticsearchStore config.") vector_store_query_mode: VectorStoreQueryMode = VectorStoreQueryMode.DEFAULT +class ElasticsearchKeywordRetrieverConfig(ElasticsearchRetrieverConfig): + """Config for Elasticsearch-based retrievers. Support text only.""" + + _no_embedding: bool = PrivateAttr(default=True) + vector_store_query_mode: VectorStoreQueryMode = VectorStoreQueryMode.TEXT_SEARCH + + class BaseRankerConfig(BaseModel): """Common config for rankers. @@ -129,6 +136,12 @@ class ElasticsearchIndexConfig(VectorIndexConfig): persist_path: Union[str, Path] = "" +class ElasticsearchKeywordIndexConfig(ElasticsearchIndexConfig): + """Config for es-based index. no embedding.""" + + _no_embedding: bool = PrivateAttr(default=True) + + class ObjectNodeMetadata(BaseModel): """Metadata of ObjectNode.""" From aaae00441b21945009f2594003a57a3b5e8bdee2 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Fri, 22 Mar 2024 18:05:32 +0800 Subject: [PATCH 050/127] use Literal to restrict vector_store_query_mode of ElasticsearchKeywordRetrieverConfig --- metagpt/rag/schema.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/metagpt/rag/schema.py b/metagpt/rag/schema.py index cb5f1aac0..0711f5c83 100644 --- a/metagpt/rag/schema.py +++ b/metagpt/rag/schema.py @@ -1,7 +1,7 @@ """RAG schemas.""" from pathlib import Path -from typing import Any, Union +from typing import Any, Literal, Union from llama_index.core.embeddings import BaseEmbedding from llama_index.core.indices.base import BaseIndex @@ -62,14 +62,18 @@ class ElasticsearchRetrieverConfig(IndexRetrieverConfig): """Config for Elasticsearch-based retrievers. Support both vector and text.""" store_config: ElasticsearchStoreConfig = Field(..., description="ElasticsearchStore config.") - vector_store_query_mode: VectorStoreQueryMode = VectorStoreQueryMode.DEFAULT + vector_store_query_mode: VectorStoreQueryMode = Field( + default=VectorStoreQueryMode.DEFAULT, description="default is vector query." + ) class ElasticsearchKeywordRetrieverConfig(ElasticsearchRetrieverConfig): """Config for Elasticsearch-based retrievers. Support text only.""" _no_embedding: bool = PrivateAttr(default=True) - vector_store_query_mode: VectorStoreQueryMode = VectorStoreQueryMode.TEXT_SEARCH + vector_store_query_mode: Literal[VectorStoreQueryMode.TEXT_SEARCH] = Field( + default=VectorStoreQueryMode.TEXT_SEARCH, description="text query only." + ) class BaseRankerConfig(BaseModel): From 89b263a2fb47ccd37a5d287e74f91e70899ce430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 23 Mar 2024 13:24:23 +0800 Subject: [PATCH 051/127] fixbug: #1076 --- metagpt/provider/human_provider.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/human_provider.py b/metagpt/provider/human_provider.py index f205ecd1f..824acd345 100644 --- a/metagpt/provider/human_provider.py +++ b/metagpt/provider/human_provider.py @@ -6,7 +6,7 @@ Author: garylin2099 from typing import Optional from metagpt.configs.llm_config import LLMConfig -from metagpt.const import USE_CONFIG_TIMEOUT +from metagpt.const import LLM_API_TIMEOUT, USE_CONFIG_TIMEOUT from metagpt.logs import logger from metagpt.provider.base_llm import BaseLLM @@ -49,3 +49,6 @@ class HumanProvider(BaseLLM): async def acompletion_text(self, messages: list[dict], stream=False, timeout=USE_CONFIG_TIMEOUT) -> str: """dummy implementation of abstract method in base""" return "" + + def get_timeout(self, timeout: int) -> int: + return timeout or LLM_API_TIMEOUT From d53cfd39f7711824d14fec63e8d9b5f29097464b Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Wed, 21 Feb 2024 15:40:23 +0800 Subject: [PATCH 052/127] replace *.deepwisdom.ai for scraping by local server --- metagpt/tools/web_browser_engine_selenium.py | 2 +- tests/conftest.py | 9 +++--- tests/metagpt/tools/libs/test_web_scraping.py | 5 +-- .../metagpt/tools/test_web_browser_engine.py | 11 ++++--- .../test_web_browser_engine_playwright.py | 26 +++++++++++---- .../tools/test_web_browser_engine_selenium.py | 32 +++++++++---------- 6 files changed, 50 insertions(+), 35 deletions(-) diff --git a/metagpt/tools/web_browser_engine_selenium.py b/metagpt/tools/web_browser_engine_selenium.py index 3b1682291..786715461 100644 --- a/metagpt/tools/web_browser_engine_selenium.py +++ b/metagpt/tools/web_browser_engine_selenium.py @@ -103,7 +103,7 @@ class WDMHttpProxyClient(WDMHttpClient): def get(self, url, **kwargs): if "proxies" not in kwargs and self.proxy: - kwargs["proxies"] = {"all_proxy": self.proxy} + kwargs["proxies"] = {"all": self.proxy} return super().get(url, **kwargs) diff --git a/tests/conftest.py b/tests/conftest.py index efd782417..8603c752a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -113,12 +113,13 @@ def proxy(): while not reader.at_eof(): writer.write(await reader.read(2048)) writer.close() + await writer.wait_closed() async def handle_client(reader, writer): data = await reader.readuntil(b"\r\n\r\n") - print(f"Proxy: {data}") # checking with capfd fixture infos = pattern.match(data) host, port = infos.group("host"), infos.group("port") + print(f"Proxy: {host}") # checking with capfd fixture port = int(port) if port else 80 remote_reader, remote_writer = await asyncio.open_connection(host, port) if data.startswith(b"CONNECT"): @@ -257,10 +258,10 @@ def http_server(): server = aiohttp.web.Server(handler) runner = aiohttp.web.ServerRunner(server) await runner.setup() - site = aiohttp.web.TCPSite(runner, "localhost", 0) + site = aiohttp.web.TCPSite(runner, "127.0.0.1", 0) await site.start() - host, port = site._server.sockets[0].getsockname() - return site, f"http://{host}:{port}" + _, port, *_ = site._server.sockets[0].getsockname() + return site, f"http://127.0.0.1:{port}" return start diff --git a/tests/metagpt/tools/libs/test_web_scraping.py b/tests/metagpt/tools/libs/test_web_scraping.py index c11960e68..3d8877b8d 100644 --- a/tests/metagpt/tools/libs/test_web_scraping.py +++ b/tests/metagpt/tools/libs/test_web_scraping.py @@ -4,8 +4,8 @@ from metagpt.tools.libs.web_scraping import scrape_web_playwright @pytest.mark.asyncio -async def test_scrape_web_playwright(): - test_url = "https://www.deepwisdom.ai" +async def test_scrape_web_playwright(http_server): + server, test_url = await http_server() result = await scrape_web_playwright(test_url) @@ -21,3 +21,4 @@ async def test_scrape_web_playwright(): assert not result["inner_text"].endswith(" ") assert not result["html"].startswith(" ") assert not result["html"].endswith(" ") + await server.stop() diff --git a/tests/metagpt/tools/test_web_browser_engine.py b/tests/metagpt/tools/test_web_browser_engine.py index ceebd67fc..7a344e0ad 100644 --- a/tests/metagpt/tools/test_web_browser_engine.py +++ b/tests/metagpt/tools/test_web_browser_engine.py @@ -9,14 +9,16 @@ from metagpt.utils.parse_html import WebPage @pytest.mark.asyncio @pytest.mark.parametrize( - "browser_type, url, urls", + "browser_type", [ - (WebBrowserEngineType.PLAYWRIGHT, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), - (WebBrowserEngineType.SELENIUM, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + WebBrowserEngineType.PLAYWRIGHT, + WebBrowserEngineType.SELENIUM, ], ids=["playwright", "selenium"], ) -async def test_scrape_web_page(browser_type, url, urls): +async def test_scrape_web_page(browser_type, http_server): + server, url = await http_server() + urls = [url, url, url] browser = web_browser_engine.WebBrowserEngine(engine=browser_type) result = await browser.run(url) assert isinstance(result, WebPage) @@ -27,6 +29,7 @@ async def test_scrape_web_page(browser_type, url, urls): assert isinstance(results, list) assert len(results) == len(urls) + 1 assert all(("MetaGPT" in i.inner_text) for i in results) + await server.stop() if __name__ == "__main__": diff --git a/tests/metagpt/tools/test_web_browser_engine_playwright.py b/tests/metagpt/tools/test_web_browser_engine_playwright.py index f35848cf4..12ea96d7b 100644 --- a/tests/metagpt/tools/test_web_browser_engine_playwright.py +++ b/tests/metagpt/tools/test_web_browser_engine_playwright.py @@ -9,18 +9,28 @@ from metagpt.utils.parse_html import WebPage @pytest.mark.asyncio @pytest.mark.parametrize( - "browser_type, use_proxy, kwagrs, url, urls", + "browser_type, use_proxy, kwagrs,", [ - ("chromium", {"proxy": True}, {}, "https://www.deepwisdom.ai", ("https://www.deepwisdom.ai",)), - ("firefox", {}, {"ignore_https_errors": True}, "https://www.deepwisdom.ai", ("https://www.deepwisdom.ai",)), - ("webkit", {}, {"ignore_https_errors": True}, "https://www.deepwisdom.ai", ("https://www.deepwisdom.ai",)), + ("chromium", {"proxy": True}, {}), + ( + "firefox", + {}, + {"ignore_https_errors": True}, + ), + ( + "webkit", + {}, + {"ignore_https_errors": True}, + ), ], ids=["chromium-normal", "firefox-normal", "webkit-normal"], ) -async def test_scrape_web_page(browser_type, use_proxy, kwagrs, url, urls, proxy, capfd): +async def test_scrape_web_page(browser_type, use_proxy, kwagrs, proxy, capfd, http_server): + server, url = await http_server() + urls = [url, url, url] proxy_url = None if use_proxy: - server, proxy_url = await proxy() + proxy_server, proxy_url = await proxy() browser = web_browser_engine_playwright.PlaywrightWrapper(browser_type=browser_type, proxy=proxy_url, **kwagrs) result = await browser.run(url) assert isinstance(result, WebPage) @@ -32,8 +42,10 @@ async def test_scrape_web_page(browser_type, use_proxy, kwagrs, url, urls, proxy assert len(results) == len(urls) + 1 assert all(("MetaGPT" in i.inner_text) for i in results) if use_proxy: - server.close() + proxy_server.close() + await proxy_server.wait_closed() assert "Proxy:" in capfd.readouterr().out + await server.stop() if __name__ == "__main__": diff --git a/tests/metagpt/tools/test_web_browser_engine_selenium.py b/tests/metagpt/tools/test_web_browser_engine_selenium.py index a88a5d0f4..a214748bd 100644 --- a/tests/metagpt/tools/test_web_browser_engine_selenium.py +++ b/tests/metagpt/tools/test_web_browser_engine_selenium.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- + import browsers import pytest @@ -10,51 +11,48 @@ from metagpt.utils.parse_html import WebPage @pytest.mark.asyncio @pytest.mark.parametrize( - "browser_type, use_proxy, url, urls", + "browser_type, use_proxy,", [ pytest.param( "chrome", - True, - "https://deepwisdom.ai", - ("https://deepwisdom.ai",), + False, marks=pytest.mark.skipif(not browsers.get("chrome"), reason="chrome browser not found"), ), pytest.param( "firefox", False, - "https://deepwisdom.ai", - ("https://deepwisdom.ai",), marks=pytest.mark.skipif(not browsers.get("firefox"), reason="firefox browser not found"), ), pytest.param( "edge", False, - "https://deepwisdom.ai", - ("https://deepwisdom.ai",), marks=pytest.mark.skipif(not browsers.get("msedge"), reason="edge browser not found"), ), ], ids=["chrome-normal", "firefox-normal", "edge-normal"], ) -async def test_scrape_web_page(browser_type, use_proxy, url, urls, proxy, capfd): +async def test_scrape_web_page(browser_type, use_proxy, proxy, capfd, http_server): # Prerequisites # firefox, chrome, Microsoft Edge + server, url = await http_server() + urls = [url, url, url] proxy_url = None if use_proxy: - server, proxy_url = await proxy() + proxy_server, proxy_url = await proxy() browser = web_browser_engine_selenium.SeleniumWrapper(browser_type=browser_type, proxy=proxy_url) result = await browser.run(url) assert isinstance(result, WebPage) assert "MetaGPT" in result.inner_text - if urls: - results = await browser.run(url, *urls) - assert isinstance(results, list) - assert len(results) == len(urls) + 1 - assert all(("MetaGPT" in i.inner_text) for i in results) + results = await browser.run(url, *urls) + assert isinstance(results, list) + assert len(results) == len(urls) + 1 + assert all(("MetaGPT" in i.inner_text) for i in results) if use_proxy: - server.close() - assert "Proxy:" in capfd.readouterr().out + proxy_server.close() + await proxy_server.wait_closed() + assert "Proxy: localhost" in capfd.readouterr().out + await server.stop() if __name__ == "__main__": From a988d9c4f3c8fad9802a092cb088659e3fe1fbcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 23 Mar 2024 17:57:10 +0800 Subject: [PATCH 053/127] fixbug: proxy --- metagpt/provider/google_gemini_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index bdbf7acd6..d604313a5 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -60,8 +60,8 @@ class GeminiLLM(BaseLLM): def __init_gemini(self, config: LLMConfig): if config.proxy: logger.info(f"Use proxy: {config.proxy}") - os.environ["HTTP_PROXY"] = config.proxy - os.environ["HTTP_PROXYS"] = config.proxy + os.environ["http_proxy"] = config.proxy + os.environ["https_proxy"] = config.proxy genai.configure(api_key=config.api_key) def _user_msg(self, msg: str, images: Optional[Union[str, list[str]]] = None) -> dict[str, str]: From 83e5eb133a8d48a84c67b46d237d5c95e5caca2f Mon Sep 17 00:00:00 2001 From: luxiangtao Date: Sun, 24 Mar 2024 15:01:45 +0800 Subject: [PATCH 054/127] updata di example and Change the location of the .ipynb locations --- examples/di/data_visualization.py | 15 +++++++++++---- metagpt/utils/recovery_util.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/examples/di/data_visualization.py b/examples/di/data_visualization.py index 1a21ab7cb..62e7e7c2f 100644 --- a/examples/di/data_visualization.py +++ b/examples/di/data_visualization.py @@ -1,14 +1,21 @@ import asyncio - +import os +import sys +import metagpt.utils +from metagpt.logs import logger +from metagpt.utils.recovery_util import save_history from metagpt.roles.di.data_interpreter import DataInterpreter - async def main(requirement: str = ""): + di = DataInterpreter() - await di.run(requirement) + rsp = await di.run(requirement) + logger.info(rsp) + save_history(role=di) if __name__ == "__main__": - requirement = "Run data analysis on sklearn Iris dataset, include a plot" + requirement = "Run data analysis on sklearn Iris dataset, include a plot" asyncio.run(main(requirement)) + diff --git a/metagpt/utils/recovery_util.py b/metagpt/utils/recovery_util.py index d0b197e69..2089ae018 100644 --- a/metagpt/utils/recovery_util.py +++ b/metagpt/utils/recovery_util.py @@ -54,5 +54,5 @@ def save_history(role: Role, save_dir: str = ""): with open(save_path / "plan.json", "w", encoding="utf-8") as plan_file: json.dump(plan, plan_file, indent=4, ensure_ascii=False) - save_code_file(name=Path(record_time) / "history_nb", code_context=role.execute_code.nb, file_format="ipynb") + save_code_file(name=Path(record_time), code_context=role.execute_code.nb, file_format="ipynb") return save_path From 37fcabfc3ca312a68a62349744ce5993b4316df6 Mon Sep 17 00:00:00 2001 From: luxiangtao Date: Sun, 24 Mar 2024 15:04:42 +0800 Subject: [PATCH 055/127] remove di example's uselsee codes --- examples/di/data_visualization.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/di/data_visualization.py b/examples/di/data_visualization.py index 62e7e7c2f..4c88ffe00 100644 --- a/examples/di/data_visualization.py +++ b/examples/di/data_visualization.py @@ -1,7 +1,4 @@ import asyncio -import os -import sys -import metagpt.utils from metagpt.logs import logger from metagpt.utils.recovery_util import save_history from metagpt.roles.di.data_interpreter import DataInterpreter From 82ca12f4da251ae2b63d9723eea26c98f5b87b4d Mon Sep 17 00:00:00 2001 From: luxiangtao Date: Mon, 25 Mar 2024 11:24:02 +0800 Subject: [PATCH 056/127] pre-commit --- examples/di/data_visualization.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/di/data_visualization.py b/examples/di/data_visualization.py index 4c88ffe00..184e04f26 100644 --- a/examples/di/data_visualization.py +++ b/examples/di/data_visualization.py @@ -1,10 +1,11 @@ import asyncio + from metagpt.logs import logger -from metagpt.utils.recovery_util import save_history from metagpt.roles.di.data_interpreter import DataInterpreter +from metagpt.utils.recovery_util import save_history + async def main(requirement: str = ""): - di = DataInterpreter() rsp = await di.run(requirement) logger.info(rsp) @@ -12,7 +13,5 @@ async def main(requirement: str = ""): if __name__ == "__main__": - requirement = "Run data analysis on sklearn Iris dataset, include a plot" asyncio.run(main(requirement)) - From d0dc2674cab060da71f6e2289cc0a7ad3d1388e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Mon, 25 Mar 2024 16:30:53 +0800 Subject: [PATCH 057/127] fix #868 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6b23e47b6..aca31ef00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ numpy==1.24.3 openai==1.6.1 openpyxl beautifulsoup4==4.12.3 -pandas==2.0.3 +pandas==2.1.1 pydantic==2.5.3 #pygame==2.1.3 #pymilvus==2.2.8 From b403f2b2098dbd7e9ea86b5171d5820315483a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 25 Mar 2024 22:56:18 +0800 Subject: [PATCH 058/127] fixbug: #1100 --- metagpt/provider/google_gemini_api.py | 19 ++++++++++++++++--- requirements.txt | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 4ff49befe..e4b3a3f17 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -1,9 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # @Desc : Google Gemini LLM from https://ai.google.dev/tutorials/python_quickstart - +import json import os -from typing import Optional, Union +from dataclasses import asdict +from typing import List, Optional, Union import google.generativeai as genai from google.ai import generativelanguage as glm @@ -11,6 +12,7 @@ from google.generativeai.generative_models import GenerativeModel from google.generativeai.types import content_types from google.generativeai.types.generation_types import ( AsyncGenerateContentResponse, + BlockedPromptException, GenerateContentResponse, GenerationConfig, ) @@ -141,7 +143,11 @@ class GeminiLLM(BaseLLM): ) collected_content = [] async for chunk in resp: - content = chunk.text + try: + content = chunk.text + except Exception as e: + logger.warning(f"messages: {messages}\nerrors: {e}\n{BlockedPromptException(str(chunk))}") + raise BlockedPromptException(str(chunk)) log_llm_stream(content) collected_content.append(content) log_llm_stream("\n") @@ -150,3 +156,10 @@ class GeminiLLM(BaseLLM): usage = await self.aget_usage(messages, full_content) self._update_costs(usage) return full_content + + def list_models(self) -> List: + models = [] + for model in genai.list_models(page_size=100): + models.append(asdict(model)) + logger.info(json.dumps(models)) + return models diff --git a/requirements.txt b/requirements.txt index a0ce1d1ac..da8aa26b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -60,7 +60,7 @@ gitignore-parser==0.1.9 # connexion[uvicorn]~=3.0.5 # Used by metagpt/tools/openapi_v3_hello.py websockets~=11.0 networkx~=3.2.1 -google-generativeai==0.3.2 +google-generativeai==0.4.1 playwright>=1.26 # used at metagpt/tools/libs/web_scraping.py anytree ipywidgets==8.1.1 From 4bd80c5e7dbf5cd4d52444951a2c6bb44e46a5a1 Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 26 Mar 2024 10:22:00 +0800 Subject: [PATCH 059/127] fix provider failed ut --- metagpt/provider/human_provider.py | 2 +- tests/metagpt/provider/test_base_llm.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/human_provider.py b/metagpt/provider/human_provider.py index 824acd345..87dbd105f 100644 --- a/metagpt/provider/human_provider.py +++ b/metagpt/provider/human_provider.py @@ -17,7 +17,7 @@ class HumanProvider(BaseLLM): """ def __init__(self, config: LLMConfig): - pass + self.config = config def ask(self, msg: str, timeout=USE_CONFIG_TIMEOUT) -> str: logger.info("It's your turn, please type in your response. You may also refer to the context below") diff --git a/tests/metagpt/provider/test_base_llm.py b/tests/metagpt/provider/test_base_llm.py index bff8dbde4..40a9fda92 100644 --- a/tests/metagpt/provider/test_base_llm.py +++ b/tests/metagpt/provider/test_base_llm.py @@ -11,6 +11,7 @@ import pytest from metagpt.configs.llm_config import LLMConfig from metagpt.provider.base_llm import BaseLLM from metagpt.schema import Message +from tests.metagpt.provider.mock_llm_config import mock_llm_config from tests.metagpt.provider.req_resp_const import ( default_resp_cont, get_part_chat_completion, @@ -22,7 +23,7 @@ name = "GPT" class MockBaseLLM(BaseLLM): def __init__(self, config: LLMConfig = None): - pass + self.config = config or mock_llm_config def completion(self, messages: list[dict], timeout=3): return get_part_chat_completion(name) From b2fd9608299a17cb805449a8491a7862d2950add Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 26 Mar 2024 10:22:32 +0800 Subject: [PATCH 060/127] skip rebuild seq view --- tests/metagpt/actions/test_rebuild_sequence_view.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/metagpt/actions/test_rebuild_sequence_view.py b/tests/metagpt/actions/test_rebuild_sequence_view.py index 0e10e3776..9be3e8a99 100644 --- a/tests/metagpt/actions/test_rebuild_sequence_view.py +++ b/tests/metagpt/actions/test_rebuild_sequence_view.py @@ -18,6 +18,7 @@ from metagpt.utils.git_repository import ChangeType from metagpt.utils.graph_repository import SPO +@pytest.mark.skip @pytest.mark.asyncio async def test_rebuild(context, mocker): # Mock From a22d7d89830970c68bacb7cdf9a1cf33c4e29a18 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Tue, 26 Mar 2024 16:36:45 +0800 Subject: [PATCH 061/127] add object ranker --- metagpt/rag/factories/ranker.py | 17 +++++- metagpt/rag/rankers/object_ranker.py | 54 +++++++++++++++++ metagpt/rag/schema.py | 5 ++ .../metagpt/rag/rankers/test_object_ranker.py | 60 +++++++++++++++++++ 4 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 metagpt/rag/rankers/object_ranker.py create mode 100644 tests/metagpt/rag/rankers/test_object_ranker.py diff --git a/metagpt/rag/factories/ranker.py b/metagpt/rag/factories/ranker.py index 15dc55bf9..07cb1b929 100644 --- a/metagpt/rag/factories/ranker.py +++ b/metagpt/rag/factories/ranker.py @@ -6,14 +6,24 @@ from llama_index.core.postprocessor.types import BaseNodePostprocessor from llama_index.postprocessor.colbert_rerank import ColbertRerank from metagpt.rag.factories.base import ConfigBasedFactory -from metagpt.rag.schema import BaseRankerConfig, ColbertRerankConfig, LLMRankerConfig +from metagpt.rag.rankers.object_ranker import ObjectSortPostprocessor +from metagpt.rag.schema import ( + BaseRankerConfig, + ColbertRerankConfig, + LLMRankerConfig, + ObjectRankerConfig, +) class RankerFactory(ConfigBasedFactory): """Modify creators for dynamically instance implementation.""" def __init__(self): - creators = {LLMRankerConfig: self._create_llm_ranker, ColbertRerankConfig: self._create_colbert_ranker} + creators = { + LLMRankerConfig: self._create_llm_ranker, + ColbertRerankConfig: self._create_colbert_ranker, + ObjectRankerConfig: self._create_object_ranker, + } super().__init__(creators) def get_rankers(self, configs: list[BaseRankerConfig] = None, **kwargs) -> list[BaseNodePostprocessor]: @@ -30,6 +40,9 @@ class RankerFactory(ConfigBasedFactory): def _create_colbert_ranker(self, config: ColbertRerankConfig, **kwargs) -> LLMRerank: return ColbertRerank(**config.model_dump()) + def _create_object_ranker(self, config: ObjectRankerConfig, **kwargs) -> LLMRerank: + return ObjectSortPostprocessor(**config.model_dump()) + def _extract_llm(self, config: BaseRankerConfig = None, **kwargs) -> LLM: return self._val_from_config_or_kwargs("llm", config, **kwargs) diff --git a/metagpt/rag/rankers/object_ranker.py b/metagpt/rag/rankers/object_ranker.py new file mode 100644 index 000000000..fe45f9395 --- /dev/null +++ b/metagpt/rag/rankers/object_ranker.py @@ -0,0 +1,54 @@ +"""Object ranker.""" + +import heapq +import json +from typing import Literal, Optional + +from llama_index.core.postprocessor.types import BaseNodePostprocessor +from llama_index.core.schema import NodeWithScore, QueryBundle +from pydantic import Field + +from metagpt.rag.schema import ObjectNode + + +class ObjectSortPostprocessor(BaseNodePostprocessor): + """Sorted by object's field, desc or asc. + + Assumes nodes is list of ObjectNode with score. + """ + + field_name: str = Field(..., description="field name of the object, field's value must can be compared.") + order: Literal["desc", "asc"] = Field(default="desc", description="the direction of order.") + top_n: int = 5 + + @classmethod + def class_name(cls) -> str: + return "ObjectSortPostprocessor" + + def _postprocess_nodes( + self, + nodes: list[NodeWithScore], + query_bundle: Optional[QueryBundle] = None, + ) -> list[NodeWithScore]: + """Postprocess nodes.""" + if query_bundle is None: + raise ValueError("Missing query bundle in extra info.") + + if not nodes: + return [] + + self._check_metadata(nodes[0].node) + sort_key = lambda node: json.loads(node.node.metadata["obj_json"])[self.field_name] + return self._get_sort_func()(self.top_n, nodes, key=sort_key) + + def _get_sort_func(self): + return heapq.nlargest if self.order == "desc" else heapq.nsmallest + + def _check_metadata(self, node: ObjectNode): + try: + obj_dict = json.loads(node.metadata.get("obj_json")) + except Exception as e: + raise ValueError(f"Invalid object json in metadata: {node.metadata}, error: {e}") + + if self.field_name not in obj_dict: + raise ValueError(f"Field '{self.field_name}' not found in object: {obj_dict}") diff --git a/metagpt/rag/schema.py b/metagpt/rag/schema.py index 0711f5c83..183f6e0c7 100644 --- a/metagpt/rag/schema.py +++ b/metagpt/rag/schema.py @@ -101,6 +101,11 @@ class ColbertRerankConfig(BaseRankerConfig): keep_retrieval_score: bool = Field(default=False, description="Whether to keep the retrieval score in metadata.") +class ObjectRankerConfig(BaseRankerConfig): + field_name: str = Field(..., description="field name of the object, field's value must can be compared.") + order: Literal["desc", "asc"] = Field(default="desc", description="the direction of order.") + + class BaseIndexConfig(BaseModel): """Common config for index. diff --git a/tests/metagpt/rag/rankers/test_object_ranker.py b/tests/metagpt/rag/rankers/test_object_ranker.py new file mode 100644 index 000000000..7ea6b7488 --- /dev/null +++ b/tests/metagpt/rag/rankers/test_object_ranker.py @@ -0,0 +1,60 @@ +import json + +import pytest +from llama_index.core.schema import NodeWithScore, QueryBundle +from pydantic import BaseModel + +from metagpt.rag.rankers.object_ranker import ObjectSortPostprocessor +from metagpt.rag.schema import ObjectNode + + +class Record(BaseModel): + score: int + + +class TestObjectSortPostprocessor: + @pytest.fixture + def nodes_with_scores(self): + nodes = [ + NodeWithScore(node=ObjectNode(metadata={"obj_json": Record(score=10).model_dump_json()}), score=10), + NodeWithScore(node=ObjectNode(metadata={"obj_json": Record(score=20).model_dump_json()}), score=20), + NodeWithScore(node=ObjectNode(metadata={"obj_json": Record(score=5).model_dump_json()}), score=5), + ] + return nodes + + @pytest.fixture + def query_bundle(self, mocker): + return mocker.MagicMock(spec=QueryBundle) + + def test_sort_descending(self, nodes_with_scores, query_bundle): + postprocessor = ObjectSortPostprocessor(field_name="score", order="desc") + sorted_nodes = postprocessor._postprocess_nodes(nodes_with_scores, query_bundle) + assert [node.score for node in sorted_nodes] == [20, 10, 5] + + def test_sort_ascending(self, nodes_with_scores, query_bundle): + postprocessor = ObjectSortPostprocessor(field_name="score", order="asc") + sorted_nodes = postprocessor._postprocess_nodes(nodes_with_scores, query_bundle) + assert [node.score for node in sorted_nodes] == [5, 10, 20] + + def test_top_n_limit(self, nodes_with_scores, query_bundle): + postprocessor = ObjectSortPostprocessor(field_name="score", order="desc", top_n=2) + sorted_nodes = postprocessor._postprocess_nodes(nodes_with_scores, query_bundle) + assert len(sorted_nodes) == 2 + assert [node.score for node in sorted_nodes] == [20, 10] + + def test_invalid_json_metadata(self, query_bundle): + nodes = [NodeWithScore(node=ObjectNode(metadata={"obj_json": "invalid_json"}), score=10)] + postprocessor = ObjectSortPostprocessor(field_name="score", order="desc") + with pytest.raises(ValueError): + postprocessor._postprocess_nodes(nodes, query_bundle) + + def test_missing_query_bundle(self, nodes_with_scores): + postprocessor = ObjectSortPostprocessor(field_name="score", order="desc") + with pytest.raises(ValueError): + postprocessor._postprocess_nodes(nodes_with_scores, query_bundle=None) + + def test_field_not_found_in_object(self): + nodes = [NodeWithScore(node=ObjectNode(metadata={"obj_json": json.dumps({"not_score": 10})}), score=10)] + postprocessor = ObjectSortPostprocessor(field_name="score", order="desc") + with pytest.raises(ValueError): + postprocessor._postprocess_nodes(nodes) From 1eb141a45f794af987f171442820703e447e3e53 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Tue, 26 Mar 2024 16:40:10 +0800 Subject: [PATCH 062/127] add object ranker --- metagpt/rag/rankers/object_ranker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/rag/rankers/object_ranker.py b/metagpt/rag/rankers/object_ranker.py index fe45f9395..b8456803f 100644 --- a/metagpt/rag/rankers/object_ranker.py +++ b/metagpt/rag/rankers/object_ranker.py @@ -38,12 +38,10 @@ class ObjectSortPostprocessor(BaseNodePostprocessor): return [] self._check_metadata(nodes[0].node) + sort_key = lambda node: json.loads(node.node.metadata["obj_json"])[self.field_name] return self._get_sort_func()(self.top_n, nodes, key=sort_key) - def _get_sort_func(self): - return heapq.nlargest if self.order == "desc" else heapq.nsmallest - def _check_metadata(self, node: ObjectNode): try: obj_dict = json.loads(node.metadata.get("obj_json")) @@ -52,3 +50,6 @@ class ObjectSortPostprocessor(BaseNodePostprocessor): if self.field_name not in obj_dict: raise ValueError(f"Field '{self.field_name}' not found in object: {obj_dict}") + + def _get_sort_func(self): + return heapq.nlargest if self.order == "desc" else heapq.nsmallest From 5e6f2757108d0ab8f1971db5d0c064c2cd10bd59 Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 26 Mar 2024 20:22:45 +0800 Subject: [PATCH 063/127] add base environment action_space/observation space and update stanford_town_env --- .../st_game/actions/gen_action_details.py | 6 +- examples/st_game/actions/gen_iter_chat_utt.py | 6 +- examples/st_game/roles/st_role.py | 82 +++++++------- examples/st_game/storage/.gitignore | 1 + .../tests/actions/test_gen_action_details.py | 2 +- metagpt/environment/base_env.py | 60 +++++++--- metagpt/environment/base_env_space.py | 33 ++++++ .../stanford_town_env/env_space.py | 105 ++++++++++++++++++ .../stanford_town_ext_env.py | 84 +++++++++++++- requirements.txt | 3 +- .../test_stanford_town_ext_env.py | 26 ++++- tests/metagpt/environment/test_base_env.py | 8 +- 12 files changed, 341 insertions(+), 75 deletions(-) create mode 100644 metagpt/environment/base_env_space.py create mode 100644 metagpt/environment/stanford_town_env/env_space.py diff --git a/examples/st_game/actions/gen_action_details.py b/examples/st_game/actions/gen_action_details.py index 92a53087a..6af2cb338 100644 --- a/examples/st_game/actions/gen_action_details.py +++ b/examples/st_game/actions/gen_action_details.py @@ -4,7 +4,7 @@ import random -from metagpt.environment.api.env_api import EnvAPIAbstract +from metagpt.environment.stanford_town_env.env_space import EnvObsParams, EnvObsType from metagpt.logs import logger from .st_action import STAction @@ -367,8 +367,8 @@ class GenActionDetails(STAction): return fs async def run(self, role: "STRole", act_desp: str, act_dura): - access_tile = await role.rc.env.observe( - EnvAPIAbstract(api_name="access_tile", kwargs={"tile": role.scratch.curr_tile}) + access_tile = role.rc.env.observe( + obs_params=EnvObsParams(obs_type=EnvObsType.GET_TITLE, coord=role.scratch.curr_tile) ) act_world = access_tile["world"] act_sector = await GenActionSector().run(role, access_tile, act_desp) diff --git a/examples/st_game/actions/gen_iter_chat_utt.py b/examples/st_game/actions/gen_iter_chat_utt.py index 2b0d46f4e..eb5f569c7 100644 --- a/examples/st_game/actions/gen_iter_chat_utt.py +++ b/examples/st_game/actions/gen_iter_chat_utt.py @@ -4,7 +4,7 @@ from examples.st_game.actions.st_action import STAction from examples.st_game.utils.utils import extract_first_json_dict -from metagpt.environment.api.env_api import EnvAPIAbstract +from metagpt.environment.stanford_town_env.env_space import EnvObsParams, EnvObsType from metagpt.logs import logger @@ -113,8 +113,8 @@ class GenIterChatUTT(STAction): ] return prompt_input - access_tile = await init_role.rc.env.observe( - EnvAPIAbstract(api_name="access_tile", kwargs={"tile": init_role.scratch.curr_tile}) + access_tile = init_role.rc.env.observe( + obs_params=EnvObsParams(obs_type=EnvObsType.GET_TITLE, coord=init_role.scratch.curr_tile) ) prompt_input = create_prompt_input(access_tile, init_role, target_role, retrieved, curr_context, curr_chat) prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "iterative_convo_v1.txt") diff --git a/examples/st_game/roles/st_role.py b/examples/st_game/roles/st_role.py index d5dd994f9..48de34f15 100644 --- a/examples/st_game/roles/st_role.py +++ b/examples/st_game/roles/st_role.py @@ -36,7 +36,12 @@ from examples.st_game.utils.mg_ga_transform import ( ) from examples.st_game.utils.utils import get_embedding, path_finder from metagpt.actions.add_requirement import UserRequirement -from metagpt.environment.api.env_api import EnvAPIAbstract +from metagpt.environment.stanford_town_env.env_space import ( + EnvAction, + EnvActionType, + EnvObsParams, + EnvObsType, +) from metagpt.logs import logger from metagpt.roles.role import Role, RoleContext from metagpt.schema import Message @@ -115,10 +120,12 @@ class STRole(Role): pt_x = role_env["x"] pt_y = role_env["y"] self.rc.scratch.curr_tile = (pt_x, pt_y) - await self.rc.env.step( - EnvAPIAbstract( - api_name="add_tiles_event", - kwargs={"pt_y": pt_y, "pt_x": pt_x, "event": self.scratch.get_curr_event_and_desc()}, + + self.rc.env.step( + EnvAction( + action_type=EnvActionType.ADD_TILE_EVENT, + coord=(pt_x, pt_y), + event=self.scratch.get_curr_event_and_desc(), ) ) @@ -231,24 +238,24 @@ class STRole(Role): # PERCEIVE SPACE # We get the nearby tiles given our current tile and the persona's vision # radius. - nearby_tiles = await self.rc.env.observe( - EnvAPIAbstract( - api_name="get_nearby_tiles", - kwargs={"tile": self.rc.scratch.curr_tile, "vision_r": self.rc.scratch.vision_r}, + nearby_tiles = self.rc.env.observe( + EnvObsParams( + obs_type=EnvObsType.TILE_NBR, coord=self.rc.scratch.curr_tile, vision_radius=self.rc.scratch.vision_r ) ) # We then store the perceived space. Note that the s_mem of the persona is # in the form of a tree constructed using dictionaries. for tile in nearby_tiles: - tile_info = await self.rc.env.observe(EnvAPIAbstract(api_name="access_tile", kwargs={"tile": tile})) + tile_info = self.rc.env.observe(EnvObsParams(obs_type=EnvObsType.GET_TITLE, coord=tile)) self.rc.spatial_memory.add_tile_info(tile_info) # PERCEIVE EVENTS. # We will perceive events that take place in the same arena as the # persona's current arena. - curr_arena_path = await self.rc.env.observe( - EnvAPIAbstract(api_name="get_tile_path", kwargs={"tile": self.rc.scratch.curr_tile, "level": "arena"}) + + curr_arena_path = self.rc.env.observe( + EnvObsParams(obs_type=EnvObsType.TILE_PATH, coord=self.rc.scratch.curr_tile, level="arena") ) # We do not perceive the same event twice (this can happen if an object is @@ -260,10 +267,10 @@ class STRole(Role): # First, we put all events that are occuring in the nearby tiles into the # percept_events_list for tile in nearby_tiles: - tile_details = await self.rc.env.observe(EnvAPIAbstract(api_name="access_tile", kwargs={"tile": tile})) + tile_details = self.rc.env.observe(EnvObsParams(obs_type=EnvObsType.GET_TITLE, coord=tile)) if tile_details["events"]: - tmp_arena_path = await self.rc.env.observe( - EnvAPIAbstract(api_name="get_tile_path", kwargs={"tile": tile, "level": "arena"}) + tmp_arena_path = self.rc.env.observe( + EnvObsParams(obs_type=EnvObsType.TILE_PATH, coord=tile, level="arena") ) if tmp_arena_path == curr_arena_path: @@ -418,14 +425,14 @@ class STRole(Role): if "" in plan: # Executing persona-persona interaction. target_p_tile = roles[plan.split("")[-1].strip()].scratch.curr_tile - collision_maze = await self.rc.env.observe(EnvAPIAbstract(api_name="get_collision_maze")) + collision_maze = self.rc.env.observe()["collision_maze"] potential_path = path_finder( collision_maze, self.rc.scratch.curr_tile, target_p_tile, collision_block_id ) if len(potential_path) <= 2: target_tiles = [potential_path[0]] else: - collision_maze = await self.rc.env.observe(EnvAPIAbstract(api_name="get_collision_maze")) + collision_maze = self.rc.env.observe()["collision_maze"] potential_1 = path_finder( collision_maze, self.rc.scratch.curr_tile, @@ -455,7 +462,7 @@ class STRole(Role): # Executing a random location action. plan = ":".join(plan.split(":")[:-1]) - address_tiles = await self.rc.env.observe(EnvAPIAbstract(api_name="get_address_tiles")) + address_tiles = self.rc.env.observe()["address_tiles"] target_tiles = address_tiles[plan] target_tiles = random.sample(list(target_tiles), 1) @@ -465,7 +472,7 @@ class STRole(Role): # Retrieve the target addresses. Again, plan is an action address in its # string form. takes this and returns candidate # coordinates. - address_tiles = await self.rc.env.observe(EnvAPIAbstract(api_name="get_address_tiles")) + address_tiles = self.rc.env.observe()["address_tiles"] if plan not in address_tiles: address_tiles["Johnson Park:park:park garden"] # ERRORRRRRRR else: @@ -485,7 +492,7 @@ class STRole(Role): persona_name_set = set(roles.keys()) new_target_tiles = [] for i in target_tiles: - access_tile = await self.rc.env.observe(EnvAPIAbstract(api_name="access_tile", kwargs={"tile": i})) + access_tile = self.rc.env.observe(EnvObsParams(obs_type=EnvObsType.GET_TITLE, coord=i)) curr_event_set = access_tile["events"] pass_curr_tile = False for j in curr_event_set: @@ -507,7 +514,7 @@ class STRole(Role): # an input, and returns a list of coordinate tuples that becomes the # path. # e.g., [(0, 1), (1, 1), (1, 2), (1, 3), (1, 4)...] - collision_maze = await self.rc.env.observe(EnvAPIAbstract(api_name="get_collision_maze")) + collision_maze = self.rc.env.observe()["collision_maze"] curr_path = path_finder(collision_maze, curr_tile, i, collision_block_id) if not closest_target_tile: closest_target_tile = i @@ -539,23 +546,20 @@ class STRole(Role): ret = True if role_env: for key, val in self.game_obj_cleanup.items(): - await self.rc.env.step( - EnvAPIAbstract(api_name="turn_event_from_tile_idle", kwargs={"curr_event": key, "tile": val}) - ) + self.rc.env.step(EnvAction(action_type=EnvActionType.TURN_TILE_EVENT_IDLE, coord=val, event=key)) # reset game_obj_cleanup self.game_obj_cleanup = dict() curr_tile = self.role_tile new_tile = (role_env["x"], role_env["y"]) - await self.rc.env.step( - EnvAPIAbstract( - api_name="remove_subject_events_from_tile", kwargs={"subject": self.name, "tile": curr_tile} - ) + self.rc.env.step( + EnvAction(action_type=EnvActionType.RM_TITLE_SUB_EVENT, coord=curr_tile, subject=self.name) ) - await self.rc.env.step( - EnvAPIAbstract( - api_name="add_event_from_tile", - kwargs={"curr_event": self.scratch.get_curr_event_and_desc(), "tile": new_tile}, + self.rc.env.step( + EnvAction( + action_type=EnvActionType.ADD_TILE_EVENT, + coord=new_tile, + event=self.scratch.get_curr_event_and_desc(), ) ) @@ -563,16 +567,16 @@ class STRole(Role): # the persona gets there, we activate the object action. if not self.scratch.planned_path: self.game_obj_cleanup[self.scratch.get_curr_event_and_desc()] = new_tile - await self.rc.env.step( - EnvAPIAbstract( - api_name="add_event_from_tile", - kwargs={"curr_event": self.scratch.get_curr_event_and_desc(), "tile": new_tile}, + self.rc.env.step( + EnvAction( + action_type=EnvActionType.ADD_TILE_EVENT, + coord=new_tile, + event=self.scratch.get_curr_event_and_desc(), ) ) + blank = (self.scratch.get_curr_obj_event_and_desc()[0], None, None, None) - await self.rc.env.step( - EnvAPIAbstract(api_name="remove_event_from_tile", kwargs={"curr_event": blank, "tile": new_tile}) - ) + self.rc.env.step(EnvAction(action_type=EnvActionType.RM_TILE_EVENT, coord=new_tile, event=blank)) # update role's new tile self.rc.scratch.curr_tile = new_tile diff --git a/examples/st_game/storage/.gitignore b/examples/st_game/storage/.gitignore index 6c37f8efd..72b125e04 100644 --- a/examples/st_game/storage/.gitignore +++ b/examples/st_game/storage/.gitignore @@ -1,2 +1,3 @@ # path to store simulation data test_* +July* \ No newline at end of file diff --git a/examples/st_game/tests/actions/test_gen_action_details.py b/examples/st_game/tests/actions/test_gen_action_details.py index 3edf9b116..49e24481d 100644 --- a/examples/st_game/tests/actions/test_gen_action_details.py +++ b/examples/st_game/tests/actions/test_gen_action_details.py @@ -31,7 +31,7 @@ async def test_gen_action_details(): act_desp = "sleeping" act_dura = "120" - access_tile = await role.rc.env.observe( + access_tile = await role.rc.env.read_from_api( EnvAPIAbstract(api_name="access_tile", kwargs={"tile": role.scratch.curr_tile}) ) act_world = access_tile["world"] diff --git a/metagpt/environment/base_env.py b/metagpt/environment/base_env.py index 942bf2409..c6bfcbc12 100644 --- a/metagpt/environment/base_env.py +++ b/metagpt/environment/base_env.py @@ -3,9 +3,12 @@ # @Desc : base env of executing environment import asyncio +from abc import abstractmethod from enum import Enum from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Set, Union +from gymnasium import spaces +from gymnasium.core import ActType, ObsType from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, model_validator from metagpt.context import Context @@ -14,6 +17,7 @@ from metagpt.environment.api.env_api import ( ReadAPIRegistry, WriteAPIRegistry, ) +from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import get_function_schema, is_coroutine_func, is_send_to @@ -49,6 +53,11 @@ def mark_as_writeable(func): class ExtEnv(BaseModel): """External Env to integrate actual game environment""" + model_config = ConfigDict(arbitrary_types_allowed=True) + + action_space: spaces.Space[ActType] = Field(default_factory=spaces.Space, exclude=True) + observation_space: spaces.Space[ObsType] = Field(default_factory=spaces.Space, exclude=True) + def _check_api_exist(self, rw_api: Optional[str] = None): if not rw_api: raise ValueError(f"{rw_api} not exists") @@ -61,39 +70,56 @@ class ExtEnv(BaseModel): else: return env_write_api_registry.get_apis() - async def observe(self, env_action: Union[str, EnvAPIAbstract]): + async def read_from_api(self, env_action: Union[str, EnvAPIAbstract]): """get observation from particular api of ExtEnv""" if isinstance(env_action, str): - read_api = env_read_api_registry.get(api_name=env_action)["func"] - self._check_api_exist(read_api) - if is_coroutine_func(read_api): - res = await read_api(self) + env_read_api = env_read_api_registry.get(api_name=env_action)["func"] + self._check_api_exist(env_read_api) + if is_coroutine_func(env_read_api): + res = await env_read_api(self) else: - res = read_api(self) + res = env_read_api(self) elif isinstance(env_action, EnvAPIAbstract): - read_api = env_read_api_registry.get(api_name=env_action.api_name)["func"] - self._check_api_exist(read_api) - if is_coroutine_func(read_api): - res = await read_api(self, *env_action.args, **env_action.kwargs) + env_read_api = env_read_api_registry.get(api_name=env_action.api_name)["func"] + self._check_api_exist(env_read_api) + if is_coroutine_func(env_read_api): + res = await env_read_api(self, *env_action.args, **env_action.kwargs) else: - res = read_api(self, *env_action.args, **env_action.kwargs) + res = env_read_api(self, *env_action.args, **env_action.kwargs) return res - async def step(self, env_action: Union[str, Message, EnvAPIAbstract, list[EnvAPIAbstract]]): + async def write_thru_api(self, env_action: Union[str, Message, EnvAPIAbstract, list[EnvAPIAbstract]]): """execute through particular api of ExtEnv""" res = None if isinstance(env_action, Message): self.publish_message(env_action) elif isinstance(env_action, EnvAPIAbstract): - write_api = env_write_api_registry.get(env_action.api_name)["func"] - self._check_api_exist(write_api) - if is_coroutine_func(write_api): - res = await write_api(self, *env_action.args, **env_action.kwargs) + env_write_api = env_write_api_registry.get(env_action.api_name)["func"] + self._check_api_exist(env_write_api) + if is_coroutine_func(env_write_api): + res = await env_write_api(self, *env_action.args, **env_action.kwargs) else: - res = write_api(self, *env_action.args, **env_action.kwargs) + res = env_write_api(self, *env_action.args, **env_action.kwargs) return res + @abstractmethod + def reset( + self, + *, + seed: Optional[int] = None, + options: Optional[dict[str, Any]] = None, + ) -> tuple[dict[str, Any], dict[str, Any]]: + """Implement this to get init observation""" + + @abstractmethod + def observe(self, obs_params: Optional[BaseEnvObsParams] = None) -> Any: + """Implement this if you want to get partial observation from the env""" + + @abstractmethod + def step(self, action: BaseEnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]: + """Implement this to feed a action and then get new observation from the env""" + class Environment(ExtEnv): """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 diff --git a/metagpt/environment/base_env_space.py b/metagpt/environment/base_env_space.py new file mode 100644 index 000000000..fd0cfa399 --- /dev/null +++ b/metagpt/environment/base_env_space.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : + +from enum import IntEnum + +from pydantic import BaseModel, ConfigDict, Field + + +class BaseEnvActionType(IntEnum): + # # NONE = 0 # no action to run, just get observation + pass + + +class BaseEnvAction(BaseModel): + """env action type and its related params of action functions/apis""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + action_type: int = Field(default=0, description="action type") + + +class BaseEnvObsType(IntEnum): + # # NONE = 0 # get whole observation from env + pass + + +class BaseEnvObsParams(BaseModel): + """observation params for different EnvObsType to get its observe result""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + obs_type: int = Field(default=0, description="observation type") diff --git a/metagpt/environment/stanford_town_env/env_space.py b/metagpt/environment/stanford_town_env/env_space.py new file mode 100644 index 000000000..e100a2952 --- /dev/null +++ b/metagpt/environment/stanford_town_env/env_space.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : + +from typing import Any, Optional, Union + +import numpy as np +import numpy.typing as npt +from gymnasium import spaces +from pydantic import ConfigDict, Field, field_validator + +from metagpt.environment.base_env_space import ( + BaseEnvAction, + BaseEnvActionType, + BaseEnvObsParams, + BaseEnvObsType, +) + + +class EnvActionType(BaseEnvActionType): + NONE = 0 # no action to run, just get observation + + ADD_TILE_EVENT = 1 # Add an event triple to a tile + RM_TILE_EVENT = 2 # Remove an event triple from a tile + TURN_TILE_EVENT_IDLE = 3 # Turn an event triple from a tile into idle + RM_TITLE_SUB_EVENT = 4 # Remove an event triple that has the input subject from a tile + + +class EnvAction(BaseEnvAction): + """env action type and its related params of action functions/apis""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + action_type: int = Field(default=EnvActionType.NONE, description="action type") + coord: npt.NDArray[np.int64] = Field( + default_factory=lambda: np.zeros(2, dtype=np.int64), description="tile coordinate" + ) + subject: str = Field(default="", description="subject name of first element in event") + event: tuple[str, Optional[str], Optional[str], Optional[str]] = Field( + default=["", None, None, None], description="tile event" + ) + + @field_validator("coord", mode="before") + @classmethod + def check_coord(cls, coord) -> npt.NDArray[np.int64]: + if not isinstance(coord, np.ndarray): + return np.array(coord) + + +class EnvObsType(BaseEnvObsType): + """get part observation with specific params""" + + NONE = 0 # get whole observation from env + + GET_TITLE = 1 # get the tile detail dictionary with given tile coord + TILE_PATH = 2 # get the tile address with given tile coord + TILE_NBR = 3 # get the neighbors of given tile coord and its vision radius + + +class EnvObsParams(BaseEnvObsParams): + """observation params for different EnvObsType""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + obs_type: int = Field(default=EnvObsType.NONE, description="observation type") + coord: npt.NDArray[np.int64] = Field( + default_factory=lambda: np.zeros(2, dtype=np.int64), description="tile coordinate" + ) + level: str = Field(default="", description="different level of title") + vision_radius: int = Field(default=0, description="the vision radius of current tile") + + @field_validator("coord", mode="before") + @classmethod + def check_coord(cls, coord) -> npt.NDArray[np.int64]: + if not isinstance(coord, np.ndarray): + return np.array(coord) + + +EnvObsValType = Union[list[list[str]], dict[str, set[tuple[int, int]]], list[list[dict[str, Any]]]] + + +def get_observation_space() -> spaces.Dict: + # it's a + space = spaces.Dict( + {"collision_maze": spaces.Discrete(2), "tiles": spaces.Discrete(2), "address_tiles": spaces.Discrete(2)} + ) + + return space + + +def get_action_space(maze_shape: tuple[int, int]) -> spaces.Dict: + """The fields defined by the space correspond to the input parameters of the action except `action_type`""" + space = spaces.Dict( + { + "action_type": spaces.Discrete(len(EnvActionType)), + "coord": spaces.Box( + np.array([0, 0], dtype=np.int64), np.array([maze_shape[0], maze_shape[1]], dtype=np.int64) + ), # coord of the tile + "subject": spaces.Text(256), # the first element of an tile event + "event": spaces.Tuple( + (spaces.Text(256), spaces.Text(256), spaces.Text(256), spaces.Text(256)) + ), # event is a tuple of four str + } + ) + return space diff --git a/metagpt/environment/stanford_town_env/stanford_town_ext_env.py b/metagpt/environment/stanford_town_env/stanford_town_ext_env.py index 8a9a65965..b41ae375c 100644 --- a/metagpt/environment/stanford_town_env/stanford_town_ext_env.py +++ b/metagpt/environment/stanford_town_env/stanford_town_ext_env.py @@ -5,11 +5,20 @@ import math from pathlib import Path -from typing import Optional, Tuple +from typing import Any, Optional from pydantic import ConfigDict, Field, model_validator from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable +from metagpt.environment.stanford_town_env.env_space import ( + EnvAction, + EnvActionType, + EnvObsParams, + EnvObsType, + EnvObsValType, + get_action_space, + get_observation_space, +) from metagpt.utils.common import read_csv_to_list, read_json_file @@ -197,15 +206,82 @@ class StanfordTownExtEnv(ExtEnv): else: address_tiles[add] = set([(j, i)]) values["address_tiles"] = address_tiles + + values["action_space"] = get_action_space((maze_width, maze_height)) + values["observation_space"] = get_observation_space() return values + def reset( + self, + *, + seed: Optional[int] = None, + options: Optional[dict[str, Any]] = None, + ) -> tuple[dict[str, EnvObsValType], dict[str, Any]]: + """reset env and get the init observation + Return results corresponding to `observation, info` + """ + super().reset(seed=seed, options=options) + + obs = self._get_obs() + + return obs, {} + + def _get_obs(self) -> dict[str, EnvObsValType]: + """Get observation""" + return { + "collision_maze": self.get_collision_maze(), + "tiles": self.tiles, + "address_tiles": self.get_address_tiles(), + } + + def observe(self, obs_params: Optional[EnvObsParams] = None) -> Any: + """Get partial or full observation from the env""" + obs_type = obs_params.obs_type if obs_params else EnvObsType.NONE + if obs_type == EnvObsType.NONE: + obs = self._get_obs() + elif obs_type == EnvObsType.GET_TITLE: + obs = self.access_tile(tile=obs_params.coord) + elif obs_type == EnvObsType.TILE_PATH: + obs = self.get_tile_path(tile=obs_params.coord, level=obs_params.level) + elif obs_type == EnvObsType.TILE_NBR: + obs = self.get_nearby_tiles(tile=obs_params.coord, vision_r=obs_params.vision_radius) + return obs + + def step(self, action: EnvAction) -> tuple[dict[str, EnvObsValType], float, bool, bool, dict[str, Any]]: + """Execute action and then return observation + Return results corresponding to `observation, reward, terminated, truncated, info` + """ + terminated = False + try: + self._execute_env_action(action) + except Exception: + terminated = True + + obs = self._get_obs() + + ret = (obs, 1.0, terminated, False, {}) + return ret + + def _execute_env_action(self, action: EnvAction): + action_type = action.action_type + if action_type == EnvActionType.NONE: + pass + elif action_type == EnvActionType.ADD_TILE_EVENT: + self.add_event_from_tile(curr_event=action.event, tile=action.coord) + elif action_type == EnvActionType.RM_TILE_EVENT: + self.remove_event_from_tile(curr_event=action.event, tile=action.coord) + elif action_type == EnvActionType.TURN_TILE_EVENT_IDLE: + self.turn_event_from_tile_idle(curr_event=action.event, tile=action.coord) + elif action_type == EnvActionType.RM_TITLE_SUB_EVENT: + self.remove_subject_events_from_tile(subject=action.subject, tile=action.coord) + def turn_coordinate_to_tile(self, px_coordinate: tuple[int, int]) -> tuple[int, int]: """ Turns a pixel coordinate to a tile coordinate. """ x = math.ceil(px_coordinate[0] / self.sq_tile_size) y = math.ceil(px_coordinate[1] / self.sq_tile_size) - return (x, y) + return x, y @mark_as_readable def get_collision_maze(self) -> list: @@ -316,10 +392,6 @@ class StanfordTownExtEnv(ExtEnv): nearby_tiles += [(i, j)] return nearby_tiles - @mark_as_writeable - def add_tiles_event(self, pt_y: int, pt_x: int, event: Tuple[str, str, str, str]): - self.tiles[pt_y][pt_x]["events"].add(event) - @mark_as_writeable def add_event_from_tile(self, curr_event: tuple[str], tile: tuple[int, int]) -> None: """ diff --git a/requirements.txt b/requirements.txt index a0ce1d1ac..36de775db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -69,4 +69,5 @@ imap_tools==1.5.0 # Used by metagpt/tools/libs/email_login.py qianfan==0.3.2 dashscope==1.14.1 rank-bm25==0.2.2 # for tool recommendation -jieba==0.42.1 # for tool recommendation \ No newline at end of file +jieba==0.42.1 # for tool recommendation +gymnasium==0.29.1 \ No newline at end of file diff --git a/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py b/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py index b167f83bb..63e88cf32 100644 --- a/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py +++ b/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py @@ -4,6 +4,12 @@ from pathlib import Path +from metagpt.environment.stanford_town_env.env_space import ( + EnvAction, + EnvActionType, + EnvObsParams, + EnvObsType, +) from metagpt.environment.stanford_town_env.stanford_town_ext_env import ( StanfordTownExtEnv, ) @@ -27,7 +33,6 @@ def test_stanford_town_ext_env(): assert len(ext_env.get_nearby_tiles(tile=tile, vision_r=5)) == 121 event = ("double studio:double studio:bedroom 2:bed", None, None, None) - ext_env.add_tiles_event(tile[1], tile[0], event=event) ext_env.add_event_from_tile(event, tile) assert len(ext_env.tiles[tile[1]][tile[0]]["events"]) == 1 @@ -38,3 +43,22 @@ def test_stanford_town_ext_env(): ext_env.remove_subject_events_from_tile(subject=event[0], tile=tile) assert len(ext_env.tiles[tile[1]][tile[0]]["events"]) == 0 + + +def test_stanford_town_ext_env_observe_step(): + ext_env = StanfordTownExtEnv(maze_asset_path=maze_asset_path) + obs, info = ext_env.reset() + assert len(info) == 0 + assert len(obs["address_tiles"]) == 306 + + tile = (58, 9) + obs = ext_env.observe(obs_params=EnvObsParams(obs_type=EnvObsType.TILE_PATH, coord=tile, level="world")) + assert obs == "the Ville" + + action = ext_env.action_space.sample() + assert len(action) == 4 + assert len(action["event"]) == 4 + + event = ("double studio:double studio:bedroom 2:bed", None, None, None) + obs, _, _, _, _ = ext_env.step(action=EnvAction(action_type=EnvActionType.ADD_TILE_EVENT, coord=tile, event=event)) + assert len(ext_env.tiles[tile[1]][tile[0]]["events"]) == 1 diff --git a/tests/metagpt/environment/test_base_env.py b/tests/metagpt/environment/test_base_env.py index fd73679d8..28815a874 100644 --- a/tests/metagpt/environment/test_base_env.py +++ b/tests/metagpt/environment/test_base_env.py @@ -44,11 +44,11 @@ async def test_ext_env(): assert len(apis) > 0 assert len(apis["read_api"]) == 3 - _ = await env.step(EnvAPIAbstract(api_name="write_api", kwargs={"a": 5, "b": 10})) + _ = await env.write_thru_api(EnvAPIAbstract(api_name="write_api", kwargs={"a": 5, "b": 10})) assert env.value == 15 with pytest.raises(ValueError): - await env.observe("not_exist_api") + await env.read_from_api("not_exist_api") - assert await env.observe("read_api_no_param") == 15 - assert await env.observe(EnvAPIAbstract(api_name="read_api", kwargs={"a": 5, "b": 5})) == 10 + assert await env.read_from_api("read_api_no_param") == 15 + assert await env.read_from_api(EnvAPIAbstract(api_name="read_api", kwargs={"a": 5, "b": 5})) == 10 From b5bfa4b8a71b3d345cc92d9329cd7ff7fc0b31ae Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 26 Mar 2024 20:35:47 +0800 Subject: [PATCH 064/127] add inherited funcs and then implement --- .../android_env/android_ext_env.py | 15 +++++++++++++ .../minecraft_env/minecraft_env.py | 12 +++++------ .../minecraft_env/minecraft_ext_env.py | 21 ++++++++++++++++--- .../werewolf_env/werewolf_ext_env.py | 17 ++++++++++++++- tests/metagpt/environment/test_base_env.py | 17 +++++++++++++++ 5 files changed, 72 insertions(+), 10 deletions(-) diff --git a/metagpt/environment/android_env/android_ext_env.py b/metagpt/environment/android_env/android_ext_env.py index b81b2cd26..01a24c5b9 100644 --- a/metagpt/environment/android_env/android_ext_env.py +++ b/metagpt/environment/android_env/android_ext_env.py @@ -10,6 +10,7 @@ from pydantic import Field from metagpt.environment.android_env.const import ADB_EXEC_FAIL from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable +from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams class AndroidExtEnv(ExtEnv): @@ -19,6 +20,20 @@ class AndroidExtEnv(ExtEnv): width: int = Field(default=720, description="device screen width") height: int = Field(default=1080, description="device screen height") + def reset( + self, + *, + seed: Optional[int] = None, + options: Optional[dict[str, Any]] = None, + ) -> tuple[dict[str, Any], dict[str, Any]]: + pass + + def observe(self, obs_params: Optional[BaseEnvObsParams] = None) -> Any: + pass + + def step(self, action: BaseEnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]: + pass + def __init__(self, **data: Any): super().__init__(**data) if data.get("device_id"): diff --git a/metagpt/environment/minecraft_env/minecraft_env.py b/metagpt/environment/minecraft_env/minecraft_env.py index 26d4d03a8..bba35ce21 100644 --- a/metagpt/environment/minecraft_env/minecraft_env.py +++ b/metagpt/environment/minecraft_env/minecraft_env.py @@ -282,7 +282,7 @@ class MinecraftEnv(Environment, MinecraftExtEnv): position = event["status"]["position"] blocks.append(block) positions.append(position) - new_events = self.step( + new_events = self._step( f"await givePlacedItemBack(bot, {json.dumps(blocks)}, {json.dumps(positions)})", programs=self.programs, ) @@ -323,7 +323,7 @@ class MinecraftEnv(Environment, MinecraftExtEnv): Exception: If there is an issue retrieving events. """ try: - self.reset( + self._reset( options={ "mode": "soft", "wait_ticks": 20, @@ -332,13 +332,13 @@ class MinecraftEnv(Environment, MinecraftExtEnv): # difficulty = "easy" if len(self.completed_tasks) > 15 else "peaceful" difficulty = "peaceful" - events = self.step("bot.chat(`/time set ${getNextTime()}`);\n" + f"bot.chat('/difficulty {difficulty}');") + events = self._step("bot.chat(`/time set ${getNextTime()}`);\n" + f"bot.chat('/difficulty {difficulty}');") self.update_event(events) return events except Exception as e: time.sleep(3) # wait for mineflayer to exit # reset bot status here - events = self.reset( + events = self._reset( options={ "mode": "hard", "wait_ticks": 20, @@ -365,7 +365,7 @@ class MinecraftEnv(Environment, MinecraftExtEnv): Exception: If there is an issue retrieving events. """ try: - events = self.step( + events = self._step( code=self.code, programs=self.programs, ) @@ -374,7 +374,7 @@ class MinecraftEnv(Environment, MinecraftExtEnv): except Exception as e: time.sleep(3) # wait for mineflayer to exit # reset bot status here - events = self.reset( + events = self._reset( options={ "mode": "hard", "wait_ticks": 20, diff --git a/metagpt/environment/minecraft_env/minecraft_ext_env.py b/metagpt/environment/minecraft_env/minecraft_ext_env.py index 3b793079f..74f417eb0 100644 --- a/metagpt/environment/minecraft_env/minecraft_ext_env.py +++ b/metagpt/environment/minecraft_env/minecraft_ext_env.py @@ -5,12 +5,13 @@ import json import time -from typing import Optional +from typing import Any, Optional import requests from pydantic import ConfigDict, Field, model_validator from metagpt.environment.base_env import ExtEnv, mark_as_writeable +from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams from metagpt.environment.minecraft_env.const import ( MC_CKPT_DIR, MC_CORE_INVENTORY_ITEMS, @@ -38,6 +39,20 @@ class MinecraftExtEnv(ExtEnv): server_paused: bool = Field(default=False) warm_up: dict = Field(default=dict()) + def reset( + self, + *, + seed: Optional[int] = None, + options: Optional[dict[str, Any]] = None, + ) -> tuple[dict[str, Any], dict[str, Any]]: + pass + + def observe(self, obs_params: Optional[BaseEnvObsParams] = None) -> Any: + pass + + def step(self, action: BaseEnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]: + pass + @property def server(self) -> str: return f"{self.server_host}:{self.server_port}" @@ -115,7 +130,7 @@ class MinecraftExtEnv(ExtEnv): return res.json() @mark_as_writeable - def reset(self, *, seed=None, options=None) -> dict: + def _reset(self, *, seed=None, options=None) -> dict: if options is None: options = {} if options.get("inventory", {}) and options.get("mode", "hard") != "hard": @@ -145,7 +160,7 @@ class MinecraftExtEnv(ExtEnv): return json.loads(returned_data) @mark_as_writeable - def step(self, code: str, programs: str = "") -> dict: + def _step(self, code: str, programs: str = "") -> dict: if not self.has_reset: raise RuntimeError("Environment has not been reset yet") self.check_process() diff --git a/metagpt/environment/werewolf_env/werewolf_ext_env.py b/metagpt/environment/werewolf_env/werewolf_ext_env.py index 7c4b4c475..3f2508b06 100644 --- a/metagpt/environment/werewolf_env/werewolf_ext_env.py +++ b/metagpt/environment/werewolf_env/werewolf_ext_env.py @@ -5,11 +5,12 @@ import random from collections import Counter from enum import Enum -from typing import Callable, Optional +from typing import Any, Callable, Optional from pydantic import ConfigDict, Field from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable +from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams from metagpt.logs import logger @@ -128,6 +129,20 @@ class WerewolfExtEnv(ExtEnv): player_poisoned: Optional[str] = Field(default=None) player_current_dead: list[str] = Field(default=[]) + def reset( + self, + *, + seed: Optional[int] = None, + options: Optional[dict[str, Any]] = None, + ) -> tuple[dict[str, Any], dict[str, Any]]: + pass + + def observe(self, obs_params: Optional[BaseEnvObsParams] = None) -> Any: + pass + + def step(self, action: BaseEnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]: + pass + @property def living_players(self) -> list[str]: player_names = [] diff --git a/tests/metagpt/environment/test_base_env.py b/tests/metagpt/environment/test_base_env.py index 28815a874..404f1c206 100644 --- a/tests/metagpt/environment/test_base_env.py +++ b/tests/metagpt/environment/test_base_env.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- # @Desc : the unittest of ExtEnv&Env +from typing import Any, Optional + import pytest from metagpt.environment.api.env_api import EnvAPIAbstract @@ -12,11 +14,26 @@ from metagpt.environment.base_env import ( mark_as_readable, mark_as_writeable, ) +from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams class ForTestEnv(Environment): value: int = 0 + def reset( + self, + *, + seed: Optional[int] = None, + options: Optional[dict[str, Any]] = None, + ) -> tuple[dict[str, Any], dict[str, Any]]: + pass + + def observe(self, obs_params: Optional[BaseEnvObsParams] = None) -> Any: + pass + + def step(self, action: BaseEnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]: + pass + @mark_as_readable def read_api_no_param(self): return self.value From bc5cd8dd981b1193f8de660d481a83e9f167e1c3 Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 26 Mar 2024 20:59:29 +0800 Subject: [PATCH 065/127] fix Environment missing abstract methods implement --- metagpt/environment/base_env.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/metagpt/environment/base_env.py b/metagpt/environment/base_env.py index c6bfcbc12..024c46877 100644 --- a/metagpt/environment/base_env.py +++ b/metagpt/environment/base_env.py @@ -134,6 +134,20 @@ class Environment(ExtEnv): history: str = "" # For debug context: Context = Field(default_factory=Context, exclude=True) + def reset( + self, + *, + seed: Optional[int] = None, + options: Optional[dict[str, Any]] = None, + ) -> tuple[dict[str, Any], dict[str, Any]]: + pass + + def observe(self, obs_params: Optional[BaseEnvObsParams] = None) -> Any: + pass + + def step(self, action: BaseEnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]: + pass + @model_validator(mode="after") def init_roles(self): self.add_roles(self.roles.values()) From 8d98ce34e54eb6250f1f2cf60f5d4dd66d462a5d Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Wed, 27 Mar 2024 11:15:10 +0800 Subject: [PATCH 066/127] fix by cr --- examples/rag_pipeline.py | 29 +++++++++-------------------- requirements.txt | 10 ---------- setup.py | 2 ++ 3 files changed, 11 insertions(+), 30 deletions(-) diff --git a/examples/rag_pipeline.py b/examples/rag_pipeline.py index ae6e7b7bc..47137c0a4 100644 --- a/examples/rag_pipeline.py +++ b/examples/rag_pipeline.py @@ -1,7 +1,6 @@ """RAG pipeline""" import asyncio -from functools import wraps from pydantic import BaseModel @@ -18,6 +17,7 @@ from metagpt.rag.schema import ( FAISSRetrieverConfig, LLMRankerConfig, ) +from metagpt.utils.exceptions import handle_exception DOC_PATH = EXAMPLE_DATA_PATH / "rag/writer.txt" QUESTION = "What are key qualities to be a good writer?" @@ -28,17 +28,6 @@ TRAVEL_QUESTION = "What does Bob like?" LLM_TIP = "If you not sure, just answer I don't know." -def catch_exception(func): - @wraps(func) - async def wrapper(*args, **kwargs): - try: - return await func(*args, **kwargs) - except Exception as e: - logger.error(f"{func.__name__} exception: {e}") - - return wrapper - - class Player(BaseModel): """To demonstrate rag add objs.""" @@ -122,7 +111,7 @@ class RAGExample: self.engine.add_docs([travel_filepath]) await self.run_pipeline(question=travel_question, print_title=False) - @catch_exception + @handle_exception async def add_objects(self, print_title=True): """This example show how to add objects. @@ -180,21 +169,21 @@ class RAGExample: """ self._print_title("Init And Query ChromaDB") - # 1.save index + # 1. save index output_dir = DATA_PATH / "rag" SimpleEngine.from_docs( input_files=[TRAVEL_DOC_PATH], retriever_configs=[ChromaRetrieverConfig(persist_path=output_dir)], ) - # 2.load index + # 2. load index engine = SimpleEngine.from_index(index_config=ChromaIndexConfig(persist_path=output_dir)) - # 3.query + # 3. query answer = await engine.aquery(TRAVEL_QUESTION) self._print_query_result(answer) - @catch_exception + @handle_exception async def init_and_query_es(self): """This example show how to use es. how to save and load index. will print something like: @@ -205,17 +194,17 @@ class RAGExample: """ self._print_title("Init And Query Elasticsearch") - # 1.create es index and save docs + # 1. create es index and save docs store_config = ElasticsearchStoreConfig(index_name="travel", es_url="http://127.0.0.1:9200") engine = SimpleEngine.from_docs( input_files=[TRAVEL_DOC_PATH], retriever_configs=[ElasticsearchRetrieverConfig(store_config=store_config)], ) - # 2.load index + # 2. load index engine = SimpleEngine.from_index(index_config=ElasticsearchIndexConfig(store_config=store_config)) - # 3.query + # 3. query answer = await engine.aquery(TRAVEL_QUESTION) self._print_query_result(answer) diff --git a/requirements.txt b/requirements.txt index fef56e810..da8aa26b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,16 +10,6 @@ typer==0.9.0 # godot==0.1.1 # google_api_python_client==2.93.0 # Used by search_engine.py lancedb==0.4.0 -llama-index-core==0.10.15 -llama-index-embeddings-azure-openai==0.1.6 -llama-index-embeddings-openai==0.1.5 -llama-index-llms-azure-openai==0.1.4 -llama-index-readers-file==0.1.4 -llama-index-retrievers-bm25==0.1.3 -llama-index-vector-stores-faiss==0.1.1 -llama-index-vector-stores-elasticsearch==0.1.6 -llama-index-postprocessor-colbert-rerank==0.1.1 -chromadb==0.4.23 loguru==0.6.0 meilisearch==0.21.0 numpy==1.24.3 diff --git a/setup.py b/setup.py index f834b4c44..c728872ef 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,8 @@ extras_require = { "llama-index-readers-file==0.1.4", "llama-index-retrievers-bm25==0.1.3", "llama-index-vector-stores-faiss==0.1.1", + "llama-index-vector-stores-elasticsearch==0.1.6", + "llama-index-postprocessor-colbert-rerank==0.1.1", "chromadb==0.4.23", ], } From 90e1b629341abbf12a8b6f16910d4548c8ea2c79 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Wed, 27 Mar 2024 11:28:23 +0800 Subject: [PATCH 067/127] rm unnecessary comment --- examples/rag_pipeline.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/rag_pipeline.py b/examples/rag_pipeline.py index 47137c0a4..b5111b75c 100644 --- a/examples/rag_pipeline.py +++ b/examples/rag_pipeline.py @@ -189,8 +189,6 @@ class RAGExample: Query Result: Bob likes traveling. - - If `Unclosed client session`, it's llamaindex elasticsearch problem, maybe fixed later. """ self._print_title("Init And Query Elasticsearch") From 695ffca5fa8bd01ad32a81a98d3b01a68566290d Mon Sep 17 00:00:00 2001 From: wiley Date: Wed, 27 Mar 2024 14:42:44 +0800 Subject: [PATCH 068/127] :sparkles: Add bing search engine --- metagpt/tools/__init__.py | 1 + metagpt/tools/search_engine.py | 3 + metagpt/tools/search_engine_bing.py | 103 ++++++++++++++++++++++ tests/metagpt/tools/test_search_engine.py | 1 + 4 files changed, 108 insertions(+) create mode 100644 metagpt/tools/search_engine_bing.py diff --git a/metagpt/tools/__init__.py b/metagpt/tools/__init__.py index 8d265e9f3..4b27be287 100644 --- a/metagpt/tools/__init__.py +++ b/metagpt/tools/__init__.py @@ -19,6 +19,7 @@ class SearchEngineType(Enum): DIRECT_GOOGLE = "google" DUCK_DUCK_GO = "ddg" CUSTOM_ENGINE = "custom" + Bing = "bing" class WebBrowserEngineType(Enum): diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 1e540bd0e..7c463605c 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -88,6 +88,9 @@ class SearchEngine(BaseModel): run_func = importlib.import_module(module).DDGAPIWrapper(**kwargs).run elif self.engine == SearchEngineType.CUSTOM_ENGINE: run_func = self.run_func + elif self.engine == SearchEngineType.Bing: + module = "metagpt.tools.search_engine_bing" + run_func = importlib.import_module(module).BingAPIWrapper(**kwargs).run else: raise NotImplementedError self.run_func = run_func diff --git a/metagpt/tools/search_engine_bing.py b/metagpt/tools/search_engine_bing.py new file mode 100644 index 000000000..831cf1648 --- /dev/null +++ b/metagpt/tools/search_engine_bing.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import annotations + +import json +import warnings +from typing import Optional + +import aiohttp + +from pydantic import BaseModel, ConfigDict, model_validator + + +class BingAPIWrapper(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + api_key: str + bing_url: str = "https://api.bing.microsoft.com/v7.0/search" + aiosession: Optional[aiohttp.ClientSession] = None + proxy: Optional[str] = None + + @model_validator(mode="before") + @classmethod + def validate_api_key(cls, values: dict) -> dict: + if "api_key" in values: + values.setdefault("api_key", values["api_key"]) + warnings.warn("`api_key` is deprecated, use `api_key` instead", DeprecationWarning, stacklevel=2) + return values + + @property + def header(self): + return {"Ocp-Apim-Subscription-Key": self.api_key} + + async def run( + self, + query: str, + max_results: int = 8, + as_string: bool = True, + focus: list[str] | None = None, + ) -> str | list[dict]: + """Return the results of a Google search using the official Bing API. + + Args: + query: The search query. + max_results: The number of results to return. + as_string: A boolean flag to determine the return type of the results. If True, the function will + return a formatted string with the search results. If False, it will return a list of dictionaries + containing detailed information about each search result. + focus: Specific information to be focused on from each search result. + + Returns: + The results of the search. + """ + params = { + "q": query, + "count": max_results, + "textFormat": "HTML", + } + result = await self.results(params) + search_results = result["webPages"]["value"] + focus = focus or ["snippet", "url", "name"] + details = [{i: j for i, j in item_dict.items() if i in focus} for item_dict in search_results] + if as_string: + return safe_results(details) + return details + + async def results(self, params: dict) -> dict: + """Use aiohttp to run query and return the results async.""" + + if not self.aiosession: + async with aiohttp.ClientSession() as session: + async with session.get(self.bing_url, params=params, headers=self.header, proxy=self.proxy) as response: + response.raise_for_status() + res = await response.json() + else: + async with self.aiosession.get(self.bing_url, params=params, headers=self.header, + proxy=self.proxy) as response: + response.raise_for_status() + res = await response.json() + + return res + + +def safe_results(results: str | list) -> str: + """Return the results of a bing search in a safe format. + + Args: + results: The search results. + + Returns: + The results of the search. + """ + if isinstance(results, list): + safe_message = json.dumps([result for result in results]) + else: + safe_message = results.encode("utf-8", "ignore").decode("utf-8") + return safe_message + + +if __name__ == "__main__": + import fire + + fire.Fire(BingAPIWrapper().run) diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index 964ead02f..4877e250b 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -37,6 +37,7 @@ class MockSearchEnine: (SearchEngineType.SERPER_GOOGLE, None, 6, False), (SearchEngineType.DUCK_DUCK_GO, None, 8, True), (SearchEngineType.DUCK_DUCK_GO, None, 6, False), + (SearchEngineType.Bing, None, 6, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 8, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 6, False), ], From ba6fa497a9d2eea4ae207012d6bba900b9e7f2ca Mon Sep 17 00:00:00 2001 From: wiley Date: Wed, 27 Mar 2024 15:38:16 +0800 Subject: [PATCH 069/127] :sparkles: Add bing search engine --- metagpt/tools/search_engine_bing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/tools/search_engine_bing.py b/metagpt/tools/search_engine_bing.py index 831cf1648..4ca9ce32d 100644 --- a/metagpt/tools/search_engine_bing.py +++ b/metagpt/tools/search_engine_bing.py @@ -58,7 +58,10 @@ class BingAPIWrapper(BaseModel): } result = await self.results(params) search_results = result["webPages"]["value"] - focus = focus or ["snippet", "url", "name"] + focus = focus or ["snippet", "link", "title"] + for item_dict in search_results: + item_dict["link"] = item_dict["url"] + item_dict["title"] = item_dict["name"] details = [{i: j for i, j in item_dict.items() if i in focus} for item_dict in search_results] if as_string: return safe_results(details) From c8fe6db931f8ac4347e81549b927d55b5ccb4445 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Wed, 27 Mar 2024 19:53:50 +0800 Subject: [PATCH 070/127] upgrade llama-index-vector-stores-chroma and rag test coverage 100% --- .../minecraft_env/minecraft_env.py | 2 +- metagpt/rag/factories/index.py | 2 +- metagpt/rag/factories/retriever.py | 2 +- metagpt/rag/vector_stores/__init__.py | 0 metagpt/rag/vector_stores/chroma/__init__.py | 3 - metagpt/rag/vector_stores/chroma/base.py | 290 ------------------ setup.py | 2 +- tests/metagpt/rag/engines/test_simple.py | 168 +++++++++- tests/metagpt/rag/factories/test_embedding.py | 43 +++ tests/metagpt/rag/factories/test_index.py | 89 ++++++ tests/metagpt/rag/factories/test_llm.py | 71 +++++ tests/metagpt/rag/factories/test_ranker.py | 58 ++-- tests/metagpt/rag/factories/test_retriever.py | 80 +++-- tests/metagpt/rag/rankers/test_base_ranker.py | 23 ++ .../metagpt/rag/rankers/test_object_ranker.py | 37 ++- .../rag/retrievers/test_base_retriever.py | 21 ++ .../rag/retrievers/test_bm25_retriever.py | 12 +- .../rag/retrievers/test_chroma_retriever.py | 20 ++ .../rag/retrievers/test_es_retriever.py | 20 ++ .../rag/retrievers/test_faiss_retriever.py | 11 +- .../rag/retrievers/test_hybrid_retriever.py | 28 +- 21 files changed, 600 insertions(+), 382 deletions(-) delete mode 100644 metagpt/rag/vector_stores/__init__.py delete mode 100644 metagpt/rag/vector_stores/chroma/__init__.py delete mode 100644 metagpt/rag/vector_stores/chroma/base.py create mode 100644 tests/metagpt/rag/factories/test_embedding.py create mode 100644 tests/metagpt/rag/factories/test_index.py create mode 100644 tests/metagpt/rag/factories/test_llm.py create mode 100644 tests/metagpt/rag/rankers/test_base_ranker.py create mode 100644 tests/metagpt/rag/retrievers/test_base_retriever.py create mode 100644 tests/metagpt/rag/retrievers/test_chroma_retriever.py create mode 100644 tests/metagpt/rag/retrievers/test_es_retriever.py diff --git a/metagpt/environment/minecraft_env/minecraft_env.py b/metagpt/environment/minecraft_env/minecraft_env.py index 26d4d03a8..6e1800b32 100644 --- a/metagpt/environment/minecraft_env/minecraft_env.py +++ b/metagpt/environment/minecraft_env/minecraft_env.py @@ -8,6 +8,7 @@ import re import time from typing import Any, Iterable +from llama_index.vector_stores.chroma import ChromaVectorStore from pydantic import ConfigDict, Field from metagpt.config2 import config as CONFIG @@ -15,7 +16,6 @@ from metagpt.environment.base_env import Environment from metagpt.environment.minecraft_env.const import MC_CKPT_DIR from metagpt.environment.minecraft_env.minecraft_ext_env import MinecraftExtEnv from metagpt.logs import logger -from metagpt.rag.vector_stores.chroma import ChromaVectorStore from metagpt.utils.common import load_mc_skills_code, read_json_file, write_json_file diff --git a/metagpt/rag/factories/index.py b/metagpt/rag/factories/index.py index f200fc94f..a56471359 100644 --- a/metagpt/rag/factories/index.py +++ b/metagpt/rag/factories/index.py @@ -5,6 +5,7 @@ from llama_index.core import StorageContext, VectorStoreIndex, load_index_from_s from llama_index.core.embeddings import BaseEmbedding from llama_index.core.indices.base import BaseIndex from llama_index.core.vector_stores.types import BasePydanticVectorStore +from llama_index.vector_stores.chroma import ChromaVectorStore from llama_index.vector_stores.elasticsearch import ElasticsearchStore from llama_index.vector_stores.faiss import FaissVectorStore @@ -17,7 +18,6 @@ from metagpt.rag.schema import ( ElasticsearchKeywordIndexConfig, FAISSIndexConfig, ) -from metagpt.rag.vector_stores.chroma import ChromaVectorStore class RAGIndexFactory(ConfigBasedFactory): diff --git a/metagpt/rag/factories/retriever.py b/metagpt/rag/factories/retriever.py index a107d9573..65729002e 100644 --- a/metagpt/rag/factories/retriever.py +++ b/metagpt/rag/factories/retriever.py @@ -6,6 +6,7 @@ import chromadb import faiss from llama_index.core import StorageContext, VectorStoreIndex from llama_index.core.vector_stores.types import BasePydanticVectorStore +from llama_index.vector_stores.chroma import ChromaVectorStore from llama_index.vector_stores.elasticsearch import ElasticsearchStore from llama_index.vector_stores.faiss import FaissVectorStore @@ -25,7 +26,6 @@ from metagpt.rag.schema import ( FAISSRetrieverConfig, IndexRetrieverConfig, ) -from metagpt.rag.vector_stores.chroma import ChromaVectorStore class RetrieverFactory(ConfigBasedFactory): diff --git a/metagpt/rag/vector_stores/__init__.py b/metagpt/rag/vector_stores/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/metagpt/rag/vector_stores/chroma/__init__.py b/metagpt/rag/vector_stores/chroma/__init__.py deleted file mode 100644 index 87ba4d8a7..000000000 --- a/metagpt/rag/vector_stores/chroma/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from metagpt.rag.vector_stores.chroma.base import ChromaVectorStore - -__all__ = ["ChromaVectorStore"] diff --git a/metagpt/rag/vector_stores/chroma/base.py b/metagpt/rag/vector_stores/chroma/base.py deleted file mode 100644 index 55e5bd40d..000000000 --- a/metagpt/rag/vector_stores/chroma/base.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Chroma vector store. - -Refs to https://github.com/run-llama/llama_index/blob/v0.10.12/llama-index-integrations/vector_stores/llama-index-vector-stores-chroma/llama_index/vector_stores/chroma/base.py. -The repo requires onnxruntime = "^1.17.0", which is too new for many OS systems, such as CentOS7. -""" - -import math -from typing import Any, Dict, Generator, List, Optional, cast - -import chromadb -from chromadb.api.models.Collection import Collection -from llama_index.core.bridge.pydantic import Field, PrivateAttr -from llama_index.core.schema import BaseNode, MetadataMode, TextNode -from llama_index.core.utils import truncate_text -from llama_index.core.vector_stores.types import ( - BasePydanticVectorStore, - MetadataFilters, - VectorStoreQuery, - VectorStoreQueryResult, -) -from llama_index.core.vector_stores.utils import ( - legacy_metadata_dict_to_node, - metadata_dict_to_node, - node_to_metadata_dict, -) - -from metagpt.logs import logger - - -def _transform_chroma_filter_condition(condition: str) -> str: - """Translate standard metadata filter op to Chroma specific spec.""" - if condition == "and": - return "$and" - elif condition == "or": - return "$or" - else: - raise ValueError(f"Filter condition {condition} not supported") - - -def _transform_chroma_filter_operator(operator: str) -> str: - """Translate standard metadata filter operator to Chroma specific spec.""" - if operator == "!=": - return "$ne" - elif operator == "==": - return "$eq" - elif operator == ">": - return "$gt" - elif operator == "<": - return "$lt" - elif operator == ">=": - return "$gte" - elif operator == "<=": - return "$lte" - else: - raise ValueError(f"Filter operator {operator} not supported") - - -def _to_chroma_filter( - standard_filters: MetadataFilters, -) -> dict: - """Translate standard metadata filters to Chroma specific spec.""" - filters = {} - filters_list = [] - condition = standard_filters.condition or "and" - condition = _transform_chroma_filter_condition(condition) - if standard_filters.filters: - for filter in standard_filters.filters: - if filter.operator: - filters_list.append({filter.key: {_transform_chroma_filter_operator(filter.operator): filter.value}}) - else: - filters_list.append({filter.key: filter.value}) - if len(filters_list) == 1: - # If there is only one filter, return it directly - return filters_list[0] - elif len(filters_list) > 1: - filters[condition] = filters_list - return filters - - -import_err_msg = "`chromadb` package not found, please run `pip install chromadb`" -MAX_CHUNK_SIZE = 41665 # One less than the max chunk size for ChromaDB - - -def chunk_list(lst: List[BaseNode], max_chunk_size: int) -> Generator[List[BaseNode], None, None]: - """Yield successive max_chunk_size-sized chunks from lst. - Args: - lst (List[BaseNode]): list of nodes with embeddings - max_chunk_size (int): max chunk size - Yields: - Generator[List[BaseNode], None, None]: list of nodes with embeddings - """ - for i in range(0, len(lst), max_chunk_size): - yield lst[i : i + max_chunk_size] - - -class ChromaVectorStore(BasePydanticVectorStore): - """Chroma vector store. - In this vector store, embeddings are stored within a ChromaDB collection. - During query time, the index uses ChromaDB to query for the top - k most similar nodes. - Args: - chroma_collection (chromadb.api.models.Collection.Collection): - ChromaDB collection instance - """ - - stores_text: bool = True - flat_metadata: bool = True - collection_name: Optional[str] - host: Optional[str] - port: Optional[str] - ssl: bool - headers: Optional[Dict[str, str]] - persist_dir: Optional[str] - collection_kwargs: Dict[str, Any] = Field(default_factory=dict) - _collection: Any = PrivateAttr() - - def __init__( - self, - chroma_collection: Optional[Any] = None, - collection_name: Optional[str] = None, - host: Optional[str] = None, - port: Optional[str] = None, - ssl: bool = False, - headers: Optional[Dict[str, str]] = None, - persist_dir: Optional[str] = None, - collection_kwargs: Optional[dict] = None, - **kwargs: Any, - ) -> None: - """Init params.""" - collection_kwargs = collection_kwargs or {} - if chroma_collection is None: - client = chromadb.HttpClient(host=host, port=port, ssl=ssl, headers=headers) - self._collection = client.get_or_create_collection(name=collection_name, **collection_kwargs) - else: - self._collection = cast(Collection, chroma_collection) - super().__init__( - host=host, - port=port, - ssl=ssl, - headers=headers, - collection_name=collection_name, - persist_dir=persist_dir, - collection_kwargs=collection_kwargs or {}, - ) - - @classmethod - def from_collection(cls, collection: Any) -> "ChromaVectorStore": - try: - from chromadb import Collection - except ImportError: - raise ImportError(import_err_msg) - if not isinstance(collection, Collection): - raise Exception("argument is not chromadb collection instance") - return cls(chroma_collection=collection) - - @classmethod - def from_params( - cls, - collection_name: str, - host: Optional[str] = None, - port: Optional[str] = None, - ssl: bool = False, - headers: Optional[Dict[str, str]] = None, - persist_dir: Optional[str] = None, - collection_kwargs: dict = {}, - **kwargs: Any, - ) -> "ChromaVectorStore": - if persist_dir: - client = chromadb.PersistentClient(path=persist_dir) - collection = client.get_or_create_collection(name=collection_name, **collection_kwargs) - elif host and port: - client = chromadb.HttpClient(host=host, port=port, ssl=ssl, headers=headers) - collection = client.get_or_create_collection(name=collection_name, **collection_kwargs) - else: - raise ValueError("Either `persist_dir` or (`host`,`port`) must be specified") - return cls( - chroma_collection=collection, - host=host, - port=port, - ssl=ssl, - headers=headers, - persist_dir=persist_dir, - collection_kwargs=collection_kwargs, - **kwargs, - ) - - @classmethod - def class_name(cls) -> str: - return "ChromaVectorStore" - - def add(self, nodes: List[BaseNode], **add_kwargs: Any) -> List[str]: - """Add nodes to index. - Args: - nodes: List[BaseNode]: list of nodes with embeddings - """ - if not self._collection: - raise ValueError("Collection not initialized") - max_chunk_size = MAX_CHUNK_SIZE - node_chunks = chunk_list(nodes, max_chunk_size) - all_ids = [] - for node_chunk in node_chunks: - embeddings = [] - metadatas = [] - ids = [] - documents = [] - for node in node_chunk: - embeddings.append(node.get_embedding()) - metadata_dict = node_to_metadata_dict(node, remove_text=True, flat_metadata=self.flat_metadata) - for key in metadata_dict: - if metadata_dict[key] is None: - metadata_dict[key] = "" - metadatas.append(metadata_dict) - ids.append(node.node_id) - documents.append(node.get_content(metadata_mode=MetadataMode.NONE)) - self._collection.add( - embeddings=embeddings, - ids=ids, - metadatas=metadatas, - documents=documents, - ) - all_ids.extend(ids) - return all_ids - - def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None: - """ - Delete nodes using with ref_doc_id. - Args: - ref_doc_id (str): The doc_id of the document to delete. - """ - self._collection.delete(where={"document_id": ref_doc_id}) - - @property - def client(self) -> Any: - """Return client.""" - return self._collection - - def query(self, query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResult: - """Query index for top k most similar nodes. - Args: - query_embedding (List[float]): query embedding - similarity_top_k (int): top k most similar nodes - """ - if query.filters is not None: - if "where" in kwargs: - raise ValueError( - "Cannot specify metadata filters via both query and kwargs. " - "Use kwargs only for chroma specific items that are " - "not supported via the generic query interface." - ) - where = _to_chroma_filter(query.filters) - else: - where = kwargs.pop("where", {}) - results = self._collection.query( - query_embeddings=query.query_embedding, - n_results=query.similarity_top_k, - where=where, - **kwargs, - ) - logger.debug(f"> Top {len(results['documents'])} nodes:") - nodes = [] - similarities = [] - ids = [] - for node_id, text, metadata, distance in zip( - results["ids"][0], - results["documents"][0], - results["metadatas"][0], - results["distances"][0], - ): - try: - node = metadata_dict_to_node(metadata) - node.set_content(text) - except Exception: - # NOTE: deprecated legacy logic for backward compatibility - metadata, node_info, relationships = legacy_metadata_dict_to_node(metadata) - node = TextNode( - text=text, - id_=node_id, - metadata=metadata, - start_char_idx=node_info.get("start", None), - end_char_idx=node_info.get("end", None), - relationships=relationships, - ) - nodes.append(node) - similarity_score = math.exp(-distance) - similarities.append(similarity_score) - logger.debug( - f"> [Node {node_id}] [Similarity score: {similarity_score}] " f"{truncate_text(str(text), 100)}" - ) - ids.append(node_id) - return VectorStoreQueryResult(nodes=nodes, similarities=similarities, ids=ids) diff --git a/setup.py b/setup.py index c728872ef..4fa5499da 100644 --- a/setup.py +++ b/setup.py @@ -37,8 +37,8 @@ extras_require = { "llama-index-retrievers-bm25==0.1.3", "llama-index-vector-stores-faiss==0.1.1", "llama-index-vector-stores-elasticsearch==0.1.6", + "llama-index-vector-stores-chroma==0.1.6", "llama-index-postprocessor-colbert-rerank==0.1.1", - "chromadb==0.4.23", ], } diff --git a/tests/metagpt/rag/engines/test_simple.py b/tests/metagpt/rag/engines/test_simple.py index 5627957c7..9262ccb07 100644 --- a/tests/metagpt/rag/engines/test_simple.py +++ b/tests/metagpt/rag/engines/test_simple.py @@ -1,12 +1,26 @@ +import json + import pytest from llama_index.core import VectorStoreIndex -from llama_index.core.schema import Document, TextNode +from llama_index.core.embeddings import MockEmbedding +from llama_index.core.llms import MockLLM +from llama_index.core.schema import Document, NodeWithScore, TextNode from metagpt.rag.engines import SimpleEngine -from metagpt.rag.retrievers.base import ModifiableRAGRetriever +from metagpt.rag.retrievers import SimpleHybridRetriever +from metagpt.rag.retrievers.base import ModifiableRAGRetriever, PersistableRAGRetriever +from metagpt.rag.schema import BM25RetrieverConfig, ObjectNode class TestSimpleEngine: + @pytest.fixture + def mock_llm(self): + return MockLLM() + + @pytest.fixture + def mock_embedding(self): + return MockEmbedding(embed_dim=1) + @pytest.fixture def mock_simple_directory_reader(self, mocker): return mocker.patch("metagpt.rag.engines.simple.SimpleDirectoryReader") @@ -54,7 +68,7 @@ class TestSimpleEngine: retriever_configs = [mocker.MagicMock()] ranker_configs = [mocker.MagicMock()] - # Execute + # Exec engine = SimpleEngine.from_docs( input_dir=input_dir, input_files=input_files, @@ -65,7 +79,7 @@ class TestSimpleEngine: ranker_configs=ranker_configs, ) - # Assertions + # Assert mock_simple_directory_reader.assert_called_once_with(input_dir=input_dir, input_files=input_files) mock_vector_store_index.assert_called_once() mock_get_retriever.assert_called_once_with( @@ -75,6 +89,68 @@ class TestSimpleEngine: mock_get_response_synthesizer.assert_called_once_with(llm=llm) assert isinstance(engine, SimpleEngine) + def test_from_docs_without_file(self): + with pytest.raises(ValueError): + SimpleEngine.from_docs() + + def test_from_objs(self, mock_llm, mock_embedding): + # Mock + class MockRAGObject: + def rag_key(self): + return "key" + + def model_dump_json(self): + return "{}" + + objs = [MockRAGObject()] + + # Setup + retriever_configs = [] + ranker_configs = [] + + # Exec + engine = SimpleEngine.from_objs( + objs=objs, + llm=mock_llm, + embed_model=mock_embedding, + retriever_configs=retriever_configs, + ranker_configs=ranker_configs, + ) + + # Assert + assert isinstance(engine, SimpleEngine) + assert engine.index is not None + + def test_from_objs_with_bm25_config(self): + # Setup + retriever_configs = [BM25RetrieverConfig()] + + # Exec + with pytest.raises(ValueError): + SimpleEngine.from_objs( + objs=[], + llm=MockLLM(), + retriever_configs=retriever_configs, + ranker_configs=[], + ) + + def test_from_index(self, mocker, mock_llm, mock_embedding): + # Mock + mock_index = mocker.MagicMock(spec=VectorStoreIndex) + mock_get_index = mocker.patch("metagpt.rag.engines.simple.get_index") + mock_get_index.return_value = mock_index + + # Exec + engine = SimpleEngine.from_index( + index_config=mock_index, + embed_model=mock_embedding, + llm=mock_llm, + ) + + # Assert + assert isinstance(engine, SimpleEngine) + assert engine.index is mock_index + @pytest.mark.asyncio async def test_asearch(self, mocker): # Mock @@ -86,10 +162,10 @@ class TestSimpleEngine: engine = SimpleEngine(retriever=mocker.MagicMock()) engine.aquery = mock_aquery - # Execute + # Exec result = await engine.asearch(test_query) - # Assertions + # Assert mock_aquery.assert_called_once_with(test_query) assert result == expected_result @@ -106,10 +182,10 @@ class TestSimpleEngine: engine = SimpleEngine(retriever=mocker.MagicMock()) test_query = "test query" - # Execute + # Exec result = await engine.aretrieve(test_query) - # Assertions + # Assert mock_query_bundle.assert_called_once_with(test_query) mock_super_aretrieve.assert_called_once_with("query_bundle") assert result[0].text == "node_with_score" @@ -134,10 +210,10 @@ class TestSimpleEngine: engine = SimpleEngine(retriever=mock_retriever, index=mock_index) input_files = ["test_file1", "test_file2"] - # Execute + # Exec engine.add_docs(input_files=input_files) - # Assertions + # Assert mock_simple_directory_reader.assert_called_once_with(input_files=input_files) mock_retriever.add_nodes.assert_called_once_with(["node1", "node2"]) @@ -156,11 +232,79 @@ class TestSimpleEngine: objs = [CustomTextNode(text=f"text_{i}", metadata={"obj": f"obj_{i}"}) for i in range(2)] engine = SimpleEngine(retriever=mock_retriever, index=mocker.MagicMock()) - # Execute + # Exec engine.add_objs(objs=objs) - # Assertions + # Assert assert mock_retriever.add_nodes.call_count == 1 for node in mock_retriever.add_nodes.call_args[0][0]: assert isinstance(node, TextNode) assert "is_obj" in node.metadata + + def test_persist_successfully(self, mocker): + # Mock + mock_retriever = mocker.MagicMock(spec=PersistableRAGRetriever) + mock_retriever.persist.return_value = mocker.MagicMock() + + # Setup + engine = SimpleEngine(retriever=mock_retriever) + + # Exec + engine.persist(persist_dir="") + + def test_ensure_retriever_of_type(self, mocker): + # Mock + class MyRetriever: + def add_nodes(self): + ... + + mock_retriever = mocker.MagicMock(spec=SimpleHybridRetriever) + mock_retriever.retrievers = [MyRetriever()] + + # Setup + engine = SimpleEngine(retriever=mock_retriever) + + # Assert + engine._ensure_retriever_of_type(ModifiableRAGRetriever) + + with pytest.raises(TypeError): + engine._ensure_retriever_of_type(PersistableRAGRetriever) + + with pytest.raises(TypeError): + other_engine = SimpleEngine(retriever=mocker.MagicMock(spec=ModifiableRAGRetriever)) + other_engine._ensure_retriever_of_type(PersistableRAGRetriever) + + def test_with_obj_metadata(self, mocker): + # Mock + node = NodeWithScore( + node=ObjectNode( + text="example", + metadata={ + "is_obj": True, + "obj_cls_name": "ExampleObject", + "obj_mod_name": "__main__", + "obj_json": json.dumps({"key": "test_key", "value": "test_value"}), + }, + ) + ) + + class ExampleObject: + def __init__(self, key, value): + self.key = key + self.value = value + + def __eq__(self, other): + return self.key == other.key and self.value == other.value + + mock_import_class = mocker.patch("metagpt.rag.engines.simple.import_class") + mock_import_class.return_value = ExampleObject + + # Setup + SimpleEngine._try_reconstruct_obj([node]) + + # Exec + expected_obj = ExampleObject(key="test_key", value="test_value") + + # Assert + assert "obj" in node.node.metadata + assert node.node.metadata["obj"] == expected_obj diff --git a/tests/metagpt/rag/factories/test_embedding.py b/tests/metagpt/rag/factories/test_embedding.py new file mode 100644 index 000000000..1ded6b4a8 --- /dev/null +++ b/tests/metagpt/rag/factories/test_embedding.py @@ -0,0 +1,43 @@ +import pytest + +from metagpt.configs.llm_config import LLMType +from metagpt.rag.factories.embedding import RAGEmbeddingFactory + + +class TestRAGEmbeddingFactory: + @pytest.fixture(autouse=True) + def mock_embedding_factory(self): + self.embedding_factory = RAGEmbeddingFactory() + + @pytest.fixture + def mock_openai_embedding(self, mocker): + return mocker.patch("metagpt.rag.factories.embedding.OpenAIEmbedding") + + @pytest.fixture + def mock_azure_embedding(self, mocker): + return mocker.patch("metagpt.rag.factories.embedding.AzureOpenAIEmbedding") + + def test_get_rag_embedding_openai(self, mock_openai_embedding): + # Exec + self.embedding_factory.get_rag_embedding(LLMType.OPENAI) + + # Assert + mock_openai_embedding.assert_called_once() + + def test_get_rag_embedding_azure(self, mock_azure_embedding): + # Exec + self.embedding_factory.get_rag_embedding(LLMType.AZURE) + + # Assert + mock_azure_embedding.assert_called_once() + + def test_get_rag_embedding_default(self, mocker, mock_openai_embedding): + # Mock + mock_config = mocker.patch("metagpt.rag.factories.embedding.config") + mock_config.llm.api_type = LLMType.OPENAI + + # Exec + self.embedding_factory.get_rag_embedding() + + # Assert + mock_openai_embedding.assert_called_once() diff --git a/tests/metagpt/rag/factories/test_index.py b/tests/metagpt/rag/factories/test_index.py new file mode 100644 index 000000000..9dc5bfb6b --- /dev/null +++ b/tests/metagpt/rag/factories/test_index.py @@ -0,0 +1,89 @@ +import pytest +from llama_index.core.embeddings import MockEmbedding + +from metagpt.rag.factories.index import RAGIndexFactory +from metagpt.rag.schema import ( + BM25IndexConfig, + ChromaIndexConfig, + ElasticsearchIndexConfig, + ElasticsearchStoreConfig, + FAISSIndexConfig, +) + + +class TestRAGIndexFactory: + @pytest.fixture(autouse=True) + def setup(self): + self.index_factory = RAGIndexFactory() + + @pytest.fixture + def faiss_config(self): + return FAISSIndexConfig(persist_path="") + + @pytest.fixture + def chroma_config(self): + return ChromaIndexConfig(persist_path="", collection_name="") + + @pytest.fixture + def bm25_config(self): + return BM25IndexConfig(persist_path="") + + @pytest.fixture + def es_config(self, mocker): + return ElasticsearchIndexConfig(store_config=ElasticsearchStoreConfig()) + + @pytest.fixture + def mock_storage_context(self, mocker): + return mocker.patch("metagpt.rag.factories.index.StorageContext.from_defaults") + + @pytest.fixture + def mock_load_index_from_storage(self, mocker): + return mocker.patch("metagpt.rag.factories.index.load_index_from_storage") + + @pytest.fixture + def mock_from_vector_store(self, mocker): + return mocker.patch("metagpt.rag.factories.index.VectorStoreIndex.from_vector_store") + + @pytest.fixture + def mock_embedding(self): + return MockEmbedding(embed_dim=1) + + def test_create_faiss_index( + self, mocker, faiss_config, mock_storage_context, mock_load_index_from_storage, mock_embedding + ): + # Mock + mock_faiss_store = mocker.patch("metagpt.rag.factories.index.FaissVectorStore.from_persist_dir") + + # Exec + self.index_factory.get_index(faiss_config, embed_model=mock_embedding) + + # Assert + mock_faiss_store.assert_called_once() + + def test_create_bm25_index( + self, mocker, bm25_config, mock_storage_context, mock_load_index_from_storage, mock_embedding + ): + self.index_factory.get_index(bm25_config, embed_model=mock_embedding) + + def test_create_chroma_index(self, mocker, chroma_config, mock_from_vector_store, mock_embedding): + # Mock + mock_chroma_db = mocker.patch("metagpt.rag.factories.index.chromadb.PersistentClient") + mock_chroma_db.get_or_create_collection.return_value = mocker.MagicMock() + + mock_chroma_store = mocker.patch("metagpt.rag.factories.index.ChromaVectorStore") + + # Exec + self.index_factory.get_index(chroma_config, embed_model=mock_embedding) + + # Assert + mock_chroma_store.assert_called_once() + + def test_create_es_index(self, mocker, es_config, mock_from_vector_store, mock_embedding): + # Mock + mock_es_store = mocker.patch("metagpt.rag.factories.index.ElasticsearchStore") + + # Exec + self.index_factory.get_index(es_config, embed_model=mock_embedding) + + # Assert + mock_es_store.assert_called_once() diff --git a/tests/metagpt/rag/factories/test_llm.py b/tests/metagpt/rag/factories/test_llm.py new file mode 100644 index 000000000..e11b87076 --- /dev/null +++ b/tests/metagpt/rag/factories/test_llm.py @@ -0,0 +1,71 @@ +from typing import Optional, Union + +import pytest +from llama_index.core.llms import LLMMetadata + +from metagpt.configs.llm_config import LLMConfig +from metagpt.const import USE_CONFIG_TIMEOUT +from metagpt.provider.base_llm import BaseLLM +from metagpt.rag.factories.llm import RAGLLM, get_rag_llm + + +class MockLLM(BaseLLM): + def __init__(self, config: LLMConfig): + ... + + async def _achat_completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): + """_achat_completion implemented by inherited class""" + + async def acompletion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): + return "ok" + + def completion(self, messages: list[dict], timeout=USE_CONFIG_TIMEOUT): + return "ok" + + async def _achat_completion_stream(self, messages: list[dict], timeout: int = USE_CONFIG_TIMEOUT) -> str: + """_achat_completion_stream implemented by inherited class""" + + async def aask( + self, + msg: Union[str, list[dict[str, str]]], + system_msgs: Optional[list[str]] = None, + format_msgs: Optional[list[dict[str, str]]] = None, + images: Optional[Union[str, list[str]]] = None, + timeout=USE_CONFIG_TIMEOUT, + stream=True, + ) -> str: + return "ok" + + +class TestRAGLLM: + @pytest.fixture + def mock_model_infer(self): + return MockLLM(config=LLMConfig()) + + @pytest.fixture + def rag_llm(self, mock_model_infer): + return RAGLLM(model_infer=mock_model_infer) + + def test_metadata(self, rag_llm): + metadata = rag_llm.metadata + assert isinstance(metadata, LLMMetadata) + assert metadata.context_window == rag_llm.context_window + assert metadata.num_output == rag_llm.num_output + assert metadata.model_name == rag_llm.model_name + + @pytest.mark.asyncio + async def test_acomplete(self, rag_llm, mock_model_infer): + response = await rag_llm.acomplete("question") + assert response.text == "ok" + + def test_complete(self, rag_llm, mock_model_infer): + response = rag_llm.complete("question") + assert response.text == "ok" + + def test_stream_complete(self, rag_llm, mock_model_infer): + rag_llm.stream_complete("question") + + +def test_get_rag_llm(): + result = get_rag_llm(MockLLM(config=LLMConfig())) + assert isinstance(result, RAGLLM) diff --git a/tests/metagpt/rag/factories/test_ranker.py b/tests/metagpt/rag/factories/test_ranker.py index 563cffa73..3f6b94b47 100644 --- a/tests/metagpt/rag/factories/test_ranker.py +++ b/tests/metagpt/rag/factories/test_ranker.py @@ -1,41 +1,57 @@ import pytest -from llama_index.core.llms import LLM +from llama_index.core.llms import MockLLM from llama_index.core.postprocessor import LLMRerank from metagpt.rag.factories.ranker import RankerFactory -from metagpt.rag.schema import LLMRankerConfig +from metagpt.rag.schema import ColbertRerankConfig, LLMRankerConfig, ObjectRankerConfig class TestRankerFactory: - @pytest.fixture - def ranker_factory(self) -> RankerFactory: - return RankerFactory() + @pytest.fixture(autouse=True) + def ranker_factory(self): + self.ranker_factory: RankerFactory = RankerFactory() @pytest.fixture - def mock_llm(self, mocker): - return mocker.MagicMock(spec=LLM) + def mock_llm(self): + return MockLLM() - def test_get_rankers_with_no_configs(self, ranker_factory: RankerFactory, mock_llm, mocker): - mocker.patch.object(ranker_factory, "_extract_llm", return_value=mock_llm) - default_rankers = ranker_factory.get_rankers() + def test_get_rankers_with_no_configs(self, mock_llm, mocker): + mocker.patch.object(self.ranker_factory, "_extract_llm", return_value=mock_llm) + default_rankers = self.ranker_factory.get_rankers() assert len(default_rankers) == 0 - def test_get_rankers_with_configs(self, ranker_factory: RankerFactory, mock_llm): + def test_get_rankers_with_configs(self, mock_llm): mock_config = LLMRankerConfig(llm=mock_llm) - rankers = ranker_factory.get_rankers(configs=[mock_config]) + rankers = self.ranker_factory.get_rankers(configs=[mock_config]) assert len(rankers) == 1 assert isinstance(rankers[0], LLMRerank) - def test_create_llm_ranker_creates_correct_instance(self, ranker_factory: RankerFactory, mock_llm): + def test_extract_llm_from_config(self, mock_llm): mock_config = LLMRankerConfig(llm=mock_llm) - ranker = ranker_factory._create_llm_ranker(mock_config) + extracted_llm = self.ranker_factory._extract_llm(config=mock_config) + assert extracted_llm == mock_llm + + def test_extract_llm_from_kwargs(self, mock_llm): + extracted_llm = self.ranker_factory._extract_llm(llm=mock_llm) + assert extracted_llm == mock_llm + + def test_create_llm_ranker(self, mock_llm): + mock_config = LLMRankerConfig(llm=mock_llm) + ranker = self.ranker_factory._create_llm_ranker(mock_config) assert isinstance(ranker, LLMRerank) - def test_extract_llm_from_config(self, ranker_factory: RankerFactory, mock_llm): - mock_config = LLMRankerConfig(llm=mock_llm) - extracted_llm = ranker_factory._extract_llm(config=mock_config) - assert extracted_llm == mock_llm + def test_create_colbert_ranker(self, mocker, mock_llm): + mocker.patch("metagpt.rag.factories.ranker.ColbertRerank", return_value="colbert") - def test_extract_llm_from_kwargs(self, ranker_factory: RankerFactory, mock_llm): - extracted_llm = ranker_factory._extract_llm(llm=mock_llm) - assert extracted_llm == mock_llm + mock_config = ColbertRerankConfig(llm=mock_llm) + ranker = self.ranker_factory._create_colbert_ranker(mock_config) + + assert ranker == "colbert" + + def test_create_object_ranker(self, mocker, mock_llm): + mocker.patch("metagpt.rag.factories.ranker.ObjectSortPostprocessor", return_value="object") + + mock_config = ObjectRankerConfig(field_name="fake", llm=mock_llm) + ranker = self.ranker_factory._create_object_ranker(mock_config) + + assert ranker == "object" diff --git a/tests/metagpt/rag/factories/test_retriever.py b/tests/metagpt/rag/factories/test_retriever.py index ac8926d46..ef1cef7e0 100644 --- a/tests/metagpt/rag/factories/test_retriever.py +++ b/tests/metagpt/rag/factories/test_retriever.py @@ -1,18 +1,28 @@ import faiss import pytest from llama_index.core import VectorStoreIndex +from llama_index.vector_stores.chroma import ChromaVectorStore +from llama_index.vector_stores.elasticsearch import ElasticsearchStore from metagpt.rag.factories.retriever import RetrieverFactory from metagpt.rag.retrievers.bm25_retriever import DynamicBM25Retriever +from metagpt.rag.retrievers.chroma_retriever import ChromaRetriever +from metagpt.rag.retrievers.es_retriever import ElasticsearchRetriever from metagpt.rag.retrievers.faiss_retriever import FAISSRetriever from metagpt.rag.retrievers.hybrid_retriever import SimpleHybridRetriever -from metagpt.rag.schema import BM25RetrieverConfig, FAISSRetrieverConfig +from metagpt.rag.schema import ( + BM25RetrieverConfig, + ChromaRetrieverConfig, + ElasticsearchRetrieverConfig, + ElasticsearchStoreConfig, + FAISSRetrieverConfig, +) class TestRetrieverFactory: - @pytest.fixture + @pytest.fixture(autouse=True) def retriever_factory(self): - return RetrieverFactory() + self.retriever_factory: RetrieverFactory = RetrieverFactory() @pytest.fixture def mock_faiss_index(self, mocker): @@ -25,55 +35,79 @@ class TestRetrieverFactory: mock.docstore.docs.values.return_value = [] return mock - def test_get_retriever_with_faiss_config( - self, retriever_factory: RetrieverFactory, mock_faiss_index, mocker, mock_vector_store_index - ): + @pytest.fixture + def mock_chroma_vector_store(self, mocker): + return mocker.MagicMock(spec=ChromaVectorStore) + + @pytest.fixture + def mock_es_vector_store(self, mocker): + return mocker.MagicMock(spec=ElasticsearchStore) + + def test_get_retriever_with_faiss_config(self, mock_faiss_index, mocker, mock_vector_store_index): mock_config = FAISSRetrieverConfig(dimensions=128) mocker.patch("faiss.IndexFlatL2", return_value=mock_faiss_index) - mocker.patch.object(retriever_factory, "_extract_index", return_value=mock_vector_store_index) + mocker.patch.object(self.retriever_factory, "_extract_index", return_value=mock_vector_store_index) - retriever = retriever_factory.get_retriever(configs=[mock_config]) + retriever = self.retriever_factory.get_retriever(configs=[mock_config]) assert isinstance(retriever, FAISSRetriever) - def test_get_retriever_with_bm25_config(self, retriever_factory: RetrieverFactory, mocker, mock_vector_store_index): + def test_get_retriever_with_bm25_config(self, mocker, mock_vector_store_index): mock_config = BM25RetrieverConfig() mocker.patch("rank_bm25.BM25Okapi.__init__", return_value=None) - mocker.patch.object(retriever_factory, "_extract_index", return_value=mock_vector_store_index) + mocker.patch.object(self.retriever_factory, "_extract_index", return_value=mock_vector_store_index) - retriever = retriever_factory.get_retriever(configs=[mock_config]) + retriever = self.retriever_factory.get_retriever(configs=[mock_config]) assert isinstance(retriever, DynamicBM25Retriever) - def test_get_retriever_with_multiple_configs_returns_hybrid( - self, retriever_factory: RetrieverFactory, mocker, mock_vector_store_index - ): + def test_get_retriever_with_multiple_configs_returns_hybrid(self, mocker, mock_vector_store_index): mock_faiss_config = FAISSRetrieverConfig(dimensions=128) mock_bm25_config = BM25RetrieverConfig() mocker.patch("rank_bm25.BM25Okapi.__init__", return_value=None) - mocker.patch.object(retriever_factory, "_extract_index", return_value=mock_vector_store_index) + mocker.patch.object(self.retriever_factory, "_extract_index", return_value=mock_vector_store_index) - retriever = retriever_factory.get_retriever(configs=[mock_faiss_config, mock_bm25_config]) + retriever = self.retriever_factory.get_retriever(configs=[mock_faiss_config, mock_bm25_config]) assert isinstance(retriever, SimpleHybridRetriever) - def test_create_default_retriever(self, retriever_factory: RetrieverFactory, mocker, mock_vector_store_index): - mocker.patch.object(retriever_factory, "_extract_index", return_value=mock_vector_store_index) + def test_get_retriever_with_chroma_config(self, mocker, mock_vector_store_index, mock_chroma_vector_store): + mock_config = ChromaRetrieverConfig(persist_path="/path/to/chroma", collection_name="test_collection") + mock_chromadb = mocker.patch("metagpt.rag.factories.retriever.chromadb.PersistentClient") + mock_chromadb.get_or_create_collection.return_value = mocker.MagicMock() + mocker.patch("metagpt.rag.factories.retriever.ChromaVectorStore", return_value=mock_chroma_vector_store) + mocker.patch.object(self.retriever_factory, "_extract_index", return_value=mock_vector_store_index) + + retriever = self.retriever_factory.get_retriever(configs=[mock_config]) + + assert isinstance(retriever, ChromaRetriever) + + def test_get_retriever_with_es_config(self, mocker, mock_vector_store_index, mock_es_vector_store): + mock_config = ElasticsearchRetrieverConfig(store_config=ElasticsearchStoreConfig()) + mocker.patch("metagpt.rag.factories.retriever.ElasticsearchStore", return_value=mock_es_vector_store) + mocker.patch.object(self.retriever_factory, "_extract_index", return_value=mock_vector_store_index) + + retriever = self.retriever_factory.get_retriever(configs=[mock_config]) + + assert isinstance(retriever, ElasticsearchRetriever) + + def test_create_default_retriever(self, mocker, mock_vector_store_index): + mocker.patch.object(self.retriever_factory, "_extract_index", return_value=mock_vector_store_index) mock_vector_store_index.as_retriever = mocker.MagicMock() - retriever = retriever_factory.get_retriever() + retriever = self.retriever_factory.get_retriever() mock_vector_store_index.as_retriever.assert_called_once() assert retriever is mock_vector_store_index.as_retriever.return_value - def test_extract_index_from_config(self, retriever_factory: RetrieverFactory, mock_vector_store_index): + def test_extract_index_from_config(self, mock_vector_store_index): mock_config = FAISSRetrieverConfig(index=mock_vector_store_index) - extracted_index = retriever_factory._extract_index(config=mock_config) + extracted_index = self.retriever_factory._extract_index(config=mock_config) assert extracted_index == mock_vector_store_index - def test_extract_index_from_kwargs(self, retriever_factory: RetrieverFactory, mock_vector_store_index): - extracted_index = retriever_factory._extract_index(index=mock_vector_store_index) + def test_extract_index_from_kwargs(self, mock_vector_store_index): + extracted_index = self.retriever_factory._extract_index(index=mock_vector_store_index) assert extracted_index == mock_vector_store_index diff --git a/tests/metagpt/rag/rankers/test_base_ranker.py b/tests/metagpt/rag/rankers/test_base_ranker.py new file mode 100644 index 000000000..9755949f6 --- /dev/null +++ b/tests/metagpt/rag/rankers/test_base_ranker.py @@ -0,0 +1,23 @@ +import pytest +from llama_index.core.schema import NodeWithScore, QueryBundle, TextNode + +from metagpt.rag.rankers.base import RAGRanker + + +class SimpleRAGRanker(RAGRanker): + def _postprocess_nodes(self, nodes, query_bundle=None): + return [NodeWithScore(node=node.node, score=node.score + 1) for node in nodes] + + +class TestSimpleRAGRanker: + @pytest.fixture + def ranker(self): + return SimpleRAGRanker() + + def test_postprocess_nodes_increases_scores(self, ranker): + nodes = [NodeWithScore(node=TextNode(text="a"), score=10), NodeWithScore(node=TextNode(text="b"), score=20)] + query_bundle = QueryBundle(query_str="test query") + + processed_nodes = ranker._postprocess_nodes(nodes, query_bundle) + + assert all(node.score == original_node.score + 1 for node, original_node in zip(processed_nodes, nodes)) diff --git a/tests/metagpt/rag/rankers/test_object_ranker.py b/tests/metagpt/rag/rankers/test_object_ranker.py index 7ea6b7488..4a9f66a42 100644 --- a/tests/metagpt/rag/rankers/test_object_ranker.py +++ b/tests/metagpt/rag/rankers/test_object_ranker.py @@ -14,7 +14,7 @@ class Record(BaseModel): class TestObjectSortPostprocessor: @pytest.fixture - def nodes_with_scores(self): + def mock_nodes_with_scores(self): nodes = [ NodeWithScore(node=ObjectNode(metadata={"obj_json": Record(score=10).model_dump_json()}), score=10), NodeWithScore(node=ObjectNode(metadata={"obj_json": Record(score=20).model_dump_json()}), score=20), @@ -23,38 +23,47 @@ class TestObjectSortPostprocessor: return nodes @pytest.fixture - def query_bundle(self, mocker): + def mock_query_bundle(self, mocker): return mocker.MagicMock(spec=QueryBundle) - def test_sort_descending(self, nodes_with_scores, query_bundle): + def test_sort_descending(self, mock_nodes_with_scores, mock_query_bundle): postprocessor = ObjectSortPostprocessor(field_name="score", order="desc") - sorted_nodes = postprocessor._postprocess_nodes(nodes_with_scores, query_bundle) + sorted_nodes = postprocessor._postprocess_nodes(mock_nodes_with_scores, mock_query_bundle) assert [node.score for node in sorted_nodes] == [20, 10, 5] - def test_sort_ascending(self, nodes_with_scores, query_bundle): + def test_sort_ascending(self, mock_nodes_with_scores, mock_query_bundle): postprocessor = ObjectSortPostprocessor(field_name="score", order="asc") - sorted_nodes = postprocessor._postprocess_nodes(nodes_with_scores, query_bundle) + sorted_nodes = postprocessor._postprocess_nodes(mock_nodes_with_scores, mock_query_bundle) assert [node.score for node in sorted_nodes] == [5, 10, 20] - def test_top_n_limit(self, nodes_with_scores, query_bundle): + def test_top_n_limit(self, mock_nodes_with_scores, mock_query_bundle): postprocessor = ObjectSortPostprocessor(field_name="score", order="desc", top_n=2) - sorted_nodes = postprocessor._postprocess_nodes(nodes_with_scores, query_bundle) + sorted_nodes = postprocessor._postprocess_nodes(mock_nodes_with_scores, mock_query_bundle) assert len(sorted_nodes) == 2 assert [node.score for node in sorted_nodes] == [20, 10] - def test_invalid_json_metadata(self, query_bundle): + def test_invalid_json_metadata(self, mock_query_bundle): nodes = [NodeWithScore(node=ObjectNode(metadata={"obj_json": "invalid_json"}), score=10)] postprocessor = ObjectSortPostprocessor(field_name="score", order="desc") with pytest.raises(ValueError): - postprocessor._postprocess_nodes(nodes, query_bundle) + postprocessor._postprocess_nodes(nodes, mock_query_bundle) - def test_missing_query_bundle(self, nodes_with_scores): + def test_missing_query_bundle(self, mock_nodes_with_scores): postprocessor = ObjectSortPostprocessor(field_name="score", order="desc") with pytest.raises(ValueError): - postprocessor._postprocess_nodes(nodes_with_scores, query_bundle=None) + postprocessor._postprocess_nodes(mock_nodes_with_scores, query_bundle=None) - def test_field_not_found_in_object(self): + def test_field_not_found_in_object(self, mock_query_bundle): nodes = [NodeWithScore(node=ObjectNode(metadata={"obj_json": json.dumps({"not_score": 10})}), score=10)] postprocessor = ObjectSortPostprocessor(field_name="score", order="desc") with pytest.raises(ValueError): - postprocessor._postprocess_nodes(nodes) + postprocessor._postprocess_nodes(nodes, query_bundle=mock_query_bundle) + + def test_not_nodes(self, mock_query_bundle): + nodes = [] + postprocessor = ObjectSortPostprocessor(field_name="score", order="desc") + result = postprocessor._postprocess_nodes(nodes, mock_query_bundle) + assert result == [] + + def test_class_name(self): + assert ObjectSortPostprocessor.class_name() == "ObjectSortPostprocessor" diff --git a/tests/metagpt/rag/retrievers/test_base_retriever.py b/tests/metagpt/rag/retrievers/test_base_retriever.py new file mode 100644 index 000000000..1065b9731 --- /dev/null +++ b/tests/metagpt/rag/retrievers/test_base_retriever.py @@ -0,0 +1,21 @@ +from metagpt.rag.retrievers.base import ModifiableRAGRetriever, PersistableRAGRetriever + + +class SubModifiableRAGRetriever(ModifiableRAGRetriever): + ... + + +class SubPersistableRAGRetriever(PersistableRAGRetriever): + ... + + +class TestModifiableRAGRetriever: + def test_subclasshook(self): + result = SubModifiableRAGRetriever.__subclasshook__(SubModifiableRAGRetriever) + assert result is NotImplemented + + +class TestPersistableRAGRetriever: + def test_subclasshook(self): + result = SubPersistableRAGRetriever.__subclasshook__(SubPersistableRAGRetriever) + assert result is NotImplemented diff --git a/tests/metagpt/rag/retrievers/test_bm25_retriever.py b/tests/metagpt/rag/retrievers/test_bm25_retriever.py index 28b37c86b..5a569f103 100644 --- a/tests/metagpt/rag/retrievers/test_bm25_retriever.py +++ b/tests/metagpt/rag/retrievers/test_bm25_retriever.py @@ -8,30 +8,30 @@ from metagpt.rag.retrievers.bm25_retriever import DynamicBM25Retriever class TestDynamicBM25Retriever: @pytest.fixture(autouse=True) def setup(self, mocker): - # 创建模拟的Document对象 self.doc1 = mocker.MagicMock(spec=Node) self.doc1.get_content.return_value = "Document content 1" self.doc2 = mocker.MagicMock(spec=Node) self.doc2.get_content.return_value = "Document content 2" self.mock_nodes = [self.doc1, self.doc2] - # 模拟index index = mocker.MagicMock(spec=VectorStoreIndex) + index.storage_context.persist.return_value = "ok" - # 模拟nodes和tokenizer参数 mock_nodes = [] mock_tokenizer = mocker.MagicMock() self.mock_bm25okapi = mocker.patch("rank_bm25.BM25Okapi.__init__", return_value=None) - # 初始化DynamicBM25Retriever对象，并提供必需的参数 self.retriever = DynamicBM25Retriever(nodes=mock_nodes, tokenizer=mock_tokenizer, index=index) def test_add_docs_updates_nodes_and_corpus(self): - # Execute + # Exec self.retriever.add_nodes(self.mock_nodes) - # Assertions + # Assert assert len(self.retriever._nodes) == len(self.mock_nodes) assert len(self.retriever._corpus) == len(self.mock_nodes) self.retriever._tokenizer.assert_called() self.mock_bm25okapi.assert_called() + + def test_persist(self): + self.retriever.persist("") diff --git a/tests/metagpt/rag/retrievers/test_chroma_retriever.py b/tests/metagpt/rag/retrievers/test_chroma_retriever.py new file mode 100644 index 000000000..cf07903cf --- /dev/null +++ b/tests/metagpt/rag/retrievers/test_chroma_retriever.py @@ -0,0 +1,20 @@ +import pytest +from llama_index.core.schema import Node + +from metagpt.rag.retrievers.chroma_retriever import ChromaRetriever + + +class TestChromaRetriever: + @pytest.fixture(autouse=True) + def setup(self, mocker): + self.doc1 = mocker.MagicMock(spec=Node) + self.doc2 = mocker.MagicMock(spec=Node) + self.mock_nodes = [self.doc1, self.doc2] + + self.mock_index = mocker.MagicMock() + self.retriever = ChromaRetriever(self.mock_index) + + def test_add_nodes(self): + self.retriever.add_nodes(self.mock_nodes) + + self.mock_index.insert_nodes.assert_called() diff --git a/tests/metagpt/rag/retrievers/test_es_retriever.py b/tests/metagpt/rag/retrievers/test_es_retriever.py new file mode 100644 index 000000000..1824bfbd2 --- /dev/null +++ b/tests/metagpt/rag/retrievers/test_es_retriever.py @@ -0,0 +1,20 @@ +import pytest +from llama_index.core.schema import Node + +from metagpt.rag.retrievers.es_retriever import ElasticsearchRetriever + + +class TestElasticsearchRetriever: + @pytest.fixture(autouse=True) + def setup(self, mocker): + self.doc1 = mocker.MagicMock(spec=Node) + self.doc2 = mocker.MagicMock(spec=Node) + self.mock_nodes = [self.doc1, self.doc2] + + self.mock_index = mocker.MagicMock() + self.retriever = ElasticsearchRetriever(self.mock_index) + + def test_add_nodes(self): + self.retriever.add_nodes(self.mock_nodes) + + self.mock_index.insert_nodes.assert_called() diff --git a/tests/metagpt/rag/retrievers/test_faiss_retriever.py b/tests/metagpt/rag/retrievers/test_faiss_retriever.py index 9113f110c..854673215 100644 --- a/tests/metagpt/rag/retrievers/test_faiss_retriever.py +++ b/tests/metagpt/rag/retrievers/test_faiss_retriever.py @@ -7,16 +7,19 @@ from metagpt.rag.retrievers.faiss_retriever import FAISSRetriever class TestFAISSRetriever: @pytest.fixture(autouse=True) def setup(self, mocker): - # 创建模拟的Document对象 self.doc1 = mocker.MagicMock(spec=Node) self.doc2 = mocker.MagicMock(spec=Node) self.mock_nodes = [self.doc1, self.doc2] - # 模拟FAISSRetriever的_index属性 self.mock_index = mocker.MagicMock() self.retriever = FAISSRetriever(self.mock_index) - def test_add_docs_calls_insert_for_each_document(self, mocker): + def test_add_docs_calls_insert_for_each_document(self): self.retriever.add_nodes(self.mock_nodes) - assert self.mock_index.insert_nodes.assert_called + self.mock_index.insert_nodes.assert_called() + + def test_persist(self): + self.retriever.persist("") + + self.mock_index.storage_context.persist.assert_called() diff --git a/tests/metagpt/rag/retrievers/test_hybrid_retriever.py b/tests/metagpt/rag/retrievers/test_hybrid_retriever.py index 8cc3087c8..da150d879 100644 --- a/tests/metagpt/rag/retrievers/test_hybrid_retriever.py +++ b/tests/metagpt/rag/retrievers/test_hybrid_retriever.py @@ -1,5 +1,3 @@ -from unittest.mock import AsyncMock - import pytest from llama_index.core.schema import NodeWithScore, TextNode @@ -7,18 +5,30 @@ from metagpt.rag.retrievers import SimpleHybridRetriever class TestSimpleHybridRetriever: + @pytest.fixture + def mock_retriever(self, mocker): + return mocker.MagicMock() + + @pytest.fixture + def mock_hybrid_retriever(self, mock_retriever) -> SimpleHybridRetriever: + return SimpleHybridRetriever(mock_retriever) + + @pytest.fixture + def mock_node(self): + return NodeWithScore(node=TextNode(id_="2"), score=0.95) + @pytest.mark.asyncio - async def test_aretrieve(self): + async def test_aretrieve(self, mocker): question = "test query" # Create mock retrievers - mock_retriever1 = AsyncMock() + mock_retriever1 = mocker.AsyncMock() mock_retriever1.aretrieve.return_value = [ NodeWithScore(node=TextNode(id_="1"), score=1.0), NodeWithScore(node=TextNode(id_="2"), score=0.95), ] - mock_retriever2 = AsyncMock() + mock_retriever2 = mocker.AsyncMock() mock_retriever2.aretrieve.return_value = [ NodeWithScore(node=TextNode(id_="2"), score=0.95), NodeWithScore(node=TextNode(id_="3"), score=0.8), @@ -37,3 +47,11 @@ class TestSimpleHybridRetriever: # Check if the scores are correct (assuming you want the highest score) node_scores = {node.node.node_id: node.score for node in results} assert node_scores["2"] == 0.95 + + def test_add_nodes(self, mock_hybrid_retriever: SimpleHybridRetriever, mock_node): + mock_hybrid_retriever.add_nodes([mock_node]) + mock_hybrid_retriever.retrievers[0].add_nodes.assert_called_once() + + def test_persist(self, mock_hybrid_retriever: SimpleHybridRetriever): + mock_hybrid_retriever.persist("") + mock_hybrid_retriever.retrievers[0].persist.assert_called_once() From b828f9d8ed911937d528462742ee2b2d2f40912b Mon Sep 17 00:00:00 2001 From: wiley Date: Wed, 27 Mar 2024 21:53:51 +0800 Subject: [PATCH 071/127] Update __init__.py. BING="bing" --- metagpt/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/tools/__init__.py b/metagpt/tools/__init__.py index 4b27be287..35fa04658 100644 --- a/metagpt/tools/__init__.py +++ b/metagpt/tools/__init__.py @@ -19,7 +19,7 @@ class SearchEngineType(Enum): DIRECT_GOOGLE = "google" DUCK_DUCK_GO = "ddg" CUSTOM_ENGINE = "custom" - Bing = "bing" + BING = "bing" class WebBrowserEngineType(Enum): From 3d555cabc1b87ba13ffe34f1cb7f293dad8fea45 Mon Sep 17 00:00:00 2001 From: wiley Date: Wed, 27 Mar 2024 21:54:34 +0800 Subject: [PATCH 072/127] Update test_search_engine.py BING="bing" --- tests/metagpt/tools/test_search_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index 4877e250b..498d3974d 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -37,7 +37,7 @@ class MockSearchEnine: (SearchEngineType.SERPER_GOOGLE, None, 6, False), (SearchEngineType.DUCK_DUCK_GO, None, 8, True), (SearchEngineType.DUCK_DUCK_GO, None, 6, False), - (SearchEngineType.Bing, None, 6, False), + (SearchEngineType.BING, None, 6, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 8, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 6, False), ], From 9793f08d071ab38a5d7dc608400a895b81d69d06 Mon Sep 17 00:00:00 2001 From: wiley Date: Wed, 27 Mar 2024 21:57:24 +0800 Subject: [PATCH 073/127] Update search_engine.py BING="bing" --- metagpt/tools/search_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 7c463605c..767f4aaba 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -88,7 +88,7 @@ class SearchEngine(BaseModel): run_func = importlib.import_module(module).DDGAPIWrapper(**kwargs).run elif self.engine == SearchEngineType.CUSTOM_ENGINE: run_func = self.run_func - elif self.engine == SearchEngineType.Bing: + elif self.engine == SearchEngineType.BING: module = "metagpt.tools.search_engine_bing" run_func = importlib.import_module(module).BingAPIWrapper(**kwargs).run else: From af3296a6f52bc4dfbadd03f4d9f325e914e8238b Mon Sep 17 00:00:00 2001 From: better629 Date: Wed, 27 Mar 2024 23:07:46 +0800 Subject: [PATCH 074/127] rename folder --- .../test_memory/kw_strength.json | 4 --- .../associative_memory/embeddings.json | 1 - .../associative_memory/nodes.json | 1 - .../{st_game => stanford_town}/.gitignore | 0 examples/{st_game => stanford_town}/README.md | 4 +-- .../{st_game => stanford_town}/README_CN.md | 4 +-- .../{st_game => stanford_town}/__init__.py | 0 .../actions/__init__.py | 0 .../actions/agent_chat_sum_rel.py | 2 +- .../actions/decide_to_talk.py | 2 +- .../actions/dummy_action.py | 0 .../actions/gen_action_details.py | 2 +- .../actions/gen_daily_schedule.py | 0 .../actions/gen_hourly_schedule.py | 0 .../actions/gen_iter_chat_utt.py | 6 ++-- .../actions/inner_voice_action.py | 2 +- .../actions/new_decomp_schedule.py | 2 +- .../actions/run_reflect_action.py | 2 +- .../actions/st_action.py | 2 +- .../actions/summarize_conv.py | 2 +- .../actions/task_decomp.py | 0 .../actions/wake_up.py | 0 .../memory/__init__.py | 0 .../memory/agent_memory.py | 0 .../memory/retrieve.py | 4 +-- .../memory/scratch.py | 0 .../memory/spatial_memory.py | 0 .../plan/__init__.py | 0 .../plan/converse.py | 6 ++-- .../plan/st_plan.py | 0 .../prompts/__init__.py | 0 .../prompts/action_location_object_vMar11.txt | 0 .../prompts/action_location_sector_v1.txt | 0 .../prompts/action_object_v2.txt | 0 .../prompts/daily_planning_v6.txt | 0 .../prompts/decide_to_talk_v2.txt | 0 .../prompts/generate_event_triple_v1.txt | 0 .../prompts/generate_focal_pt_v1.txt | 0 .../prompts/generate_hourly_schedule_v2.txt | 0 .../prompts/generate_obj_event_v1.txt | 0 .../prompts/generate_pronunciatio_v1.txt | 0 .../prompts/insight_and_evidence_v1.txt | 0 .../prompts/iterative_convo_v1.txt | 0 .../prompts/memo_on_convo_v1.txt | 0 .../prompts/new_decomp_schedule_v1.txt | 0 .../prompts/planning_thought_on_convo_v1.txt | 0 .../prompts/poignancy_action_v1.txt | 0 .../prompts/poignancy_chat_v1.txt | 0 .../prompts/poignancy_event_v1.txt | 0 .../prompts/poignancy_thought_v1.txt | 0 .../summarize_chat_relationship_v2.txt | 0 .../prompts/summarize_conversation_v1.txt | 0 .../prompts/task_decomp_v3.txt | 0 .../prompts/wake_up_hour_v1.txt | 0 .../prompts/whisper_inner_thought_v1.txt | 0 .../reflect/__init__.py | 0 .../reflect/reflect.py | 6 ++-- .../requirements.txt | 0 .../roles/__init__.py | 0 .../roles/st_role.py | 28 +++++++++---------- .../{st_game => stanford_town}/run_st_game.py | 10 +++---- .../stanford_town.py | 6 ++-- .../the_ville/agent_history_init_n25.csv | 0 .../the_ville/agent_history_init_n3.csv | 0 .../the_ville/matrix/maze/arena_maze.csv | 0 .../the_ville/matrix/maze/collision_maze.csv | 0 .../matrix/maze/game_object_maze.csv | 0 .../the_ville/matrix/maze/sector_maze.csv | 0 .../matrix/maze/spawning_location_maze.csv | 0 .../the_ville/matrix/maze_meta_info.json | 0 .../matrix/special_blocks/arena_blocks.csv | 0 .../special_blocks/game_object_blocks.csv | 0 .../matrix/special_blocks/sector_blocks.csv | 0 .../spawning_location_blocks.csv | 0 .../matrix/special_blocks/world_blocks.csv | 0 .../storage/.gitignore | 0 .../environment/0.json | 0 .../associative_memory/embeddings.json | 0 .../associative_memory/kw_strength.json | 0 .../associative_memory/nodes.json | 0 .../bootstrap_memory/scratch.json | 0 .../bootstrap_memory/spatial_memory.json | 0 .../associative_memory}/embeddings.json | 0 .../associative_memory/kw_strength.json | 0 .../associative_memory}/nodes.json | 0 .../bootstrap_memory/scratch.json | 0 .../bootstrap_memory/spatial_memory.json | 0 .../associative_memory/embeddings.json | 0 .../associative_memory/kw_strength.json | 0 .../associative_memory/nodes.json | 0 .../Maria Lopez/bootstrap_memory/scratch.json | 0 .../bootstrap_memory/spatial_memory.json | 0 .../reverie/meta.json | 0 .../tests/__init__.py | 0 .../tests/actions/__init__.py | 0 .../tests/actions/test_gen_action_details.py | 8 +++--- .../tests/actions/test_summarize_conv.py | 2 +- .../tests/plan/__init__.py | 0 .../tests/plan/test_conversation.py | 12 ++++---- .../tests/plan/test_st_plan.py | 8 ++++-- .../tests/roles/__init__.py | 0 .../tests/roles/test_st_role.py | 8 +++--- .../tests/test_agent_memory.py | 6 ++-- .../tests/test_basic_memory.py | 2 +- .../tests/test_reflect.py | 8 +++--- .../tests/test_spatial_memory.py | 0 .../utils/__init__.py | 0 .../{st_game => stanford_town}/utils/const.py | 0 .../utils/mg_ga_transform.py | 2 +- .../{st_game => stanford_town}/utils/utils.py | 0 metagpt/environment/__init__.py | 8 +++--- .../{android_env => android}/__init__.py | 0 .../{android_env => android}/android_env.py | 2 +- .../android_ext_env.py | 2 +- .../{android_env => android}/const.py | 0 .../{minecraft_env => minecraft}/__init__.py | 0 .../{minecraft_env => minecraft}/const.py | 0 .../minecraft_env.py | 4 +-- .../minecraft_ext_env.py | 6 ++-- .../mineflayer/.gitignore | 0 .../mineflayer/.prettierignore | 0 .../mineflayer/.prettierrc.json | 0 .../mineflayer/index.js | 0 .../mineflayer/lib/observation/base.js | 0 .../mineflayer/lib/observation/chests.js | 0 .../mineflayer/lib/observation/inventory.js | 0 .../mineflayer/lib/observation/onChat.js | 0 .../mineflayer/lib/observation/onError.js | 0 .../mineflayer/lib/observation/onSave.js | 0 .../mineflayer/lib/observation/status.js | 0 .../mineflayer/lib/observation/voxels.js | 0 .../mineflayer/lib/skillLoader.js | 0 .../mineflayer/lib/utils.js | 0 .../mineflayer-collectblock/.gitignore | 0 .../mineflayer-collectblock/LICENSE | 0 .../mineflayer-collectblock/README.md | 0 .../mineflayer-collectblock/_config.yml | 0 .../mineflayer-collectblock/docs/api.md | 0 .../examples/collector.js | 0 .../examples/oreMiner.js | 0 .../examples/storageBot.js | 0 .../mineflayer-collectblock/package.json | 0 .../mineflayer-collectblock/src/BlockVeins.ts | 0 .../src/CollectBlock.ts | 0 .../mineflayer-collectblock/src/Inventory.ts | 0 .../mineflayer-collectblock/src/Targets.ts | 0 .../mineflayer-collectblock/src/TaskQueue.ts | 0 .../src/TemporarySubscriber.ts | 0 .../mineflayer-collectblock/src/Util.ts | 0 .../mineflayer-collectblock/src/index.ts | 0 .../mineflayer-collectblock/tsconfig.json | 0 .../mineflayer/package.json | 0 .../process_monitor.py | 0 .../{software_env => software}/__init__.py | 0 .../software_env.py | 0 .../__init__.py | 0 .../env_space.py | 0 .../stanford_town_env.py | 4 +-- .../stanford_town_ext_env.py | 2 +- .../{werewolf_env => werewolf}/__init__.py | 0 .../werewolf_env.py | 2 +- .../werewolf_ext_env.py | 0 .../android_env/test_android_ext_env.py | 22 ++++++--------- .../minecraft_env/test_minecraft_ext_env.py | 4 +-- .../test_stanford_town_ext_env.py | 10 +++---- .../werewolf_env/test_werewolf_ext_env.py | 2 +- 166 files changed, 105 insertions(+), 115 deletions(-) delete mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/kw_strength.json delete mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/embeddings.json delete mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/nodes.json rename examples/{st_game => stanford_town}/.gitignore (100%) rename examples/{st_game => stanford_town}/README.md (77%) rename examples/{st_game => stanford_town}/README_CN.md (77%) rename examples/{st_game => stanford_town}/__init__.py (100%) rename examples/{st_game => stanford_town}/actions/__init__.py (100%) rename examples/{st_game => stanford_town}/actions/agent_chat_sum_rel.py (95%) rename examples/{st_game => stanford_town}/actions/decide_to_talk.py (98%) rename examples/{st_game => stanford_town}/actions/dummy_action.py (100%) rename examples/{st_game => stanford_town}/actions/gen_action_details.py (99%) rename examples/{st_game => stanford_town}/actions/gen_daily_schedule.py (100%) rename examples/{st_game => stanford_town}/actions/gen_hourly_schedule.py (100%) rename examples/{st_game => stanford_town}/actions/gen_iter_chat_utt.py (95%) rename examples/{st_game => stanford_town}/actions/inner_voice_action.py (94%) rename examples/{st_game => stanford_town}/actions/new_decomp_schedule.py (98%) rename examples/{st_game => stanford_town}/actions/run_reflect_action.py (99%) rename examples/{st_game => stanford_town}/actions/st_action.py (98%) rename examples/{st_game => stanford_town}/actions/summarize_conv.py (96%) rename examples/{st_game => stanford_town}/actions/task_decomp.py (100%) rename examples/{st_game => stanford_town}/actions/wake_up.py (100%) rename examples/{st_game => stanford_town}/memory/__init__.py (100%) rename examples/{st_game => stanford_town}/memory/agent_memory.py (100%) rename examples/{st_game => stanford_town}/memory/retrieve.py (97%) rename examples/{st_game => stanford_town}/memory/scratch.py (100%) rename examples/{st_game => stanford_town}/memory/spatial_memory.py (100%) rename examples/{st_game => stanford_town}/plan/__init__.py (100%) rename examples/{st_game => stanford_town}/plan/converse.py (94%) rename examples/{st_game => stanford_town}/plan/st_plan.py (100%) rename examples/{st_game => stanford_town}/prompts/__init__.py (100%) rename examples/{st_game => stanford_town}/prompts/action_location_object_vMar11.txt (100%) rename examples/{st_game => stanford_town}/prompts/action_location_sector_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/action_object_v2.txt (100%) rename examples/{st_game => stanford_town}/prompts/daily_planning_v6.txt (100%) rename examples/{st_game => stanford_town}/prompts/decide_to_talk_v2.txt (100%) rename examples/{st_game => stanford_town}/prompts/generate_event_triple_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/generate_focal_pt_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/generate_hourly_schedule_v2.txt (100%) rename examples/{st_game => stanford_town}/prompts/generate_obj_event_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/generate_pronunciatio_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/insight_and_evidence_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/iterative_convo_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/memo_on_convo_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/new_decomp_schedule_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/planning_thought_on_convo_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/poignancy_action_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/poignancy_chat_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/poignancy_event_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/poignancy_thought_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/summarize_chat_relationship_v2.txt (100%) rename examples/{st_game => stanford_town}/prompts/summarize_conversation_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/task_decomp_v3.txt (100%) rename examples/{st_game => stanford_town}/prompts/wake_up_hour_v1.txt (100%) rename examples/{st_game => stanford_town}/prompts/whisper_inner_thought_v1.txt (100%) rename examples/{st_game => stanford_town}/reflect/__init__.py (100%) rename examples/{st_game => stanford_town}/reflect/reflect.py (97%) rename examples/{st_game => stanford_town}/requirements.txt (100%) rename examples/{st_game => stanford_town}/roles/__init__.py (100%) rename examples/{st_game => stanford_town}/roles/st_role.py (96%) rename examples/{st_game => stanford_town}/run_st_game.py (89%) rename examples/{st_game => stanford_town}/stanford_town.py (87%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/agent_history_init_n25.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/agent_history_init_n3.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/maze/arena_maze.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/maze/collision_maze.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/maze/game_object_maze.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/maze/sector_maze.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/maze/spawning_location_maze.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/maze_meta_info.json (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/special_blocks/arena_blocks.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/special_blocks/game_object_blocks.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/special_blocks/sector_blocks.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/special_blocks/spawning_location_blocks.csv (100%) rename examples/{st_game => stanford_town}/static_dirs/assets/the_ville/matrix/special_blocks/world_blocks.csv (100%) rename examples/{st_game => stanford_town}/storage/.gitignore (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/environment/0.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/embeddings.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/kw_strength.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/nodes.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/scratch.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/spatial_memory.json (100%) rename examples/{st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory => stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory}/embeddings.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/kw_strength.json (100%) rename examples/{st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory => stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory}/nodes.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/scratch.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/spatial_memory.json (100%) rename examples/{st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller => stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez}/bootstrap_memory/associative_memory/embeddings.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/kw_strength.json (100%) rename examples/{st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller => stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez}/bootstrap_memory/associative_memory/nodes.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/scratch.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/spatial_memory.json (100%) rename examples/{st_game => stanford_town}/storage/base_the_ville_isabella_maria_klaus/reverie/meta.json (100%) rename examples/{st_game => stanford_town}/tests/__init__.py (100%) rename examples/{st_game => stanford_town}/tests/actions/__init__.py (100%) rename examples/{st_game => stanford_town}/tests/actions/test_gen_action_details.py (90%) rename examples/{st_game => stanford_town}/tests/actions/test_summarize_conv.py (83%) rename examples/{st_game => stanford_town}/tests/plan/__init__.py (100%) rename examples/{st_game => stanford_town}/tests/plan/test_conversation.py (80%) rename examples/{st_game => stanford_town}/tests/plan/test_st_plan.py (87%) rename examples/{st_game => stanford_town}/tests/roles/__init__.py (100%) rename examples/{st_game => stanford_town}/tests/roles/test_st_role.py (67%) rename examples/{st_game => stanford_town}/tests/test_agent_memory.py (94%) rename examples/{st_game => stanford_town}/tests/test_basic_memory.py (96%) rename examples/{st_game => stanford_town}/tests/test_reflect.py (93%) rename examples/{st_game => stanford_town}/tests/test_spatial_memory.py (100%) rename examples/{st_game => stanford_town}/utils/__init__.py (100%) rename examples/{st_game => stanford_town}/utils/const.py (100%) rename examples/{st_game => stanford_town}/utils/mg_ga_transform.py (96%) rename examples/{st_game => stanford_town}/utils/utils.py (100%) rename metagpt/environment/{android_env => android}/__init__.py (100%) rename metagpt/environment/{android_env => android}/android_env.py (83%) rename metagpt/environment/{android_env => android}/android_ext_env.py (99%) rename metagpt/environment/{android_env => android}/const.py (100%) rename metagpt/environment/{minecraft_env => minecraft}/__init__.py (100%) rename metagpt/environment/{minecraft_env => minecraft}/const.py (100%) rename metagpt/environment/{minecraft_env => minecraft}/minecraft_env.py (99%) rename metagpt/environment/{minecraft_env => minecraft}/minecraft_ext_env.py (97%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/.gitignore (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/.prettierignore (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/.prettierrc.json (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/index.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/lib/observation/base.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/lib/observation/chests.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/lib/observation/inventory.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/lib/observation/onChat.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/lib/observation/onError.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/lib/observation/onSave.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/lib/observation/status.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/lib/observation/voxels.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/lib/skillLoader.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/lib/utils.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/.gitignore (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/LICENSE (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/README.md (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/_config.yml (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/docs/api.md (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/examples/collector.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/examples/oreMiner.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/examples/storageBot.js (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/package.json (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/src/BlockVeins.ts (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/src/CollectBlock.ts (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/src/Inventory.ts (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/src/Targets.ts (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/src/TaskQueue.ts (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/src/TemporarySubscriber.ts (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/src/Util.ts (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/src/index.ts (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/mineflayer-collectblock/tsconfig.json (100%) rename metagpt/environment/{minecraft_env => minecraft}/mineflayer/package.json (100%) rename metagpt/environment/{minecraft_env => minecraft}/process_monitor.py (100%) rename metagpt/environment/{software_env => software}/__init__.py (100%) rename metagpt/environment/{software_env => software}/software_env.py (100%) rename metagpt/environment/{stanford_town_env => stanford_town}/__init__.py (100%) rename metagpt/environment/{stanford_town_env => stanford_town}/env_space.py (100%) rename metagpt/environment/{stanford_town_env => stanford_town}/stanford_town_env.py (66%) rename metagpt/environment/{stanford_town_env => stanford_town}/stanford_town_ext_env.py (99%) rename metagpt/environment/{werewolf_env => werewolf}/__init__.py (100%) rename metagpt/environment/{werewolf_env => werewolf}/werewolf_env.py (93%) rename metagpt/environment/{werewolf_env => werewolf}/werewolf_ext_env.py (100%) diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/kw_strength.json b/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/kw_strength.json deleted file mode 100644 index 041a8bd0c..000000000 --- a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/kw_strength.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "kw_strength_event": {}, - "kw_strength_thought": {} -} \ No newline at end of file diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/embeddings.json b/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/embeddings.json deleted file mode 100644 index 9e26dfeeb..000000000 --- a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/embeddings.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/nodes.json b/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/nodes.json deleted file mode 100644 index 9e26dfeeb..000000000 --- a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/nodes.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/examples/st_game/.gitignore b/examples/stanford_town/.gitignore similarity index 100% rename from examples/st_game/.gitignore rename to examples/stanford_town/.gitignore diff --git a/examples/st_game/README.md b/examples/stanford_town/README.md similarity index 77% rename from examples/st_game/README.md rename to examples/stanford_town/README.md index 114f7f672..915c61356 100644 --- a/examples/st_game/README.md +++ b/examples/stanford_town/README.md @@ -1,7 +1,7 @@ ## Stanford Town Game ### Pre-Description -The path configured in `examples/st_game/utils/const.py` is the storage path of the current project. In order to facilitate GA( [generative_agents](https://github.com/joonspk-research/generative_agents) )'s frontend docking data (to avoid changing its code), you can set the value `temp_storage_path` to `temp_storage` of `generative_agents` when start `run_st_game.py`. like +The path configured in `examples/stanford_town/utils/const.py` is the storage path of the current project. In order to facilitate GA( [generative_agents](https://github.com/joonspk-research/generative_agents) )'s frontend docking data (to avoid changing its code), you can set the value `temp_storage_path` to `temp_storage` of `generative_agents` when start `run_st_game.py`. like `python3 run_st_game.py --temp_storage_path path/to/ga/temp_storage xxx` @@ -17,7 +17,7 @@ # updated This can be used to achieve docking of simulation data without changing the GA code. Otherwise, the GA code must be modified to adapt to the MG output path. -If you don't want to start from 0, copy other simulation directories under `generative_agents/environment/frontend_server/storage/` to `examples/st_game/storage`, and select a directory named `fork_sim_code`. +If you don't want to start from 0, copy other simulation directories under `generative_agents/environment/frontend_server/storage/` to `examples/stanford_town/storage`, and select a directory named `fork_sim_code`. ### Backend service startup The execution entry is `python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` diff --git a/examples/st_game/README_CN.md b/examples/stanford_town/README_CN.md similarity index 77% rename from examples/st_game/README_CN.md rename to examples/stanford_town/README_CN.md index 84470f5c2..445a5b1b3 100644 --- a/examples/st_game/README_CN.md +++ b/examples/stanford_town/README_CN.md @@ -1,7 +1,7 @@ ## Stanford Town Game ### 前置 -`examples/st_game/utils/const.py`配置的路径为当前项目的存储路径，为了方便GA（ [generative_agents](https://github.com/joonspk-research/generative_agents) ）的前端对接数据（避免改动它那块的代码），可在启动`run_st_game.py`加上`temp_storage_path`指向`generative_agents`对应的`temp_storage`路径。比如 +`examples/stanford_town/utils/const.py`配置的路径为当前项目的存储路径，为了方便GA（ [generative_agents](https://github.com/joonspk-research/generative_agents) ）的前端对接数据（避免改动它那块的代码），可在启动`run_st_game.py`加上`temp_storage_path`指向`generative_agents`对应的`temp_storage`路径。比如 `python3 run_st_game.py --temp_storage_path path/to/ga/temp_storage xxx` @@ -16,7 +16,7 @@ # 更新为 ``` 这样可用实现不改变GA代码情况下，实现仿真数据的对接。不然得修改GA的代码来适配MG的输出路径。 -如果你不想从0开始启动，拷贝`generative_agents/environment/frontend_server/storage/`下的其他仿真目录到`examples/st_game/storage`，并选择一个目录名作为`fork_sim_code`。 +如果你不想从0开始启动，拷贝`generative_agents/environment/frontend_server/storage/`下的其他仿真目录到`examples/stanford_town/storage`，并选择一个目录名作为`fork_sim_code`。 ### 后端服务启动 执行入口为：`python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` diff --git a/examples/st_game/__init__.py b/examples/stanford_town/__init__.py similarity index 100% rename from examples/st_game/__init__.py rename to examples/stanford_town/__init__.py diff --git a/examples/st_game/actions/__init__.py b/examples/stanford_town/actions/__init__.py similarity index 100% rename from examples/st_game/actions/__init__.py rename to examples/stanford_town/actions/__init__.py diff --git a/examples/st_game/actions/agent_chat_sum_rel.py b/examples/stanford_town/actions/agent_chat_sum_rel.py similarity index 95% rename from examples/st_game/actions/agent_chat_sum_rel.py rename to examples/stanford_town/actions/agent_chat_sum_rel.py index 00f15e82f..3e564a60c 100644 --- a/examples/st_game/actions/agent_chat_sum_rel.py +++ b/examples/stanford_town/actions/agent_chat_sum_rel.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # @Desc : summarize relationship in a agent chat -from examples.st_game.actions.st_action import STAction +from examples.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/st_game/actions/decide_to_talk.py b/examples/stanford_town/actions/decide_to_talk.py similarity index 98% rename from examples/st_game/actions/decide_to_talk.py rename to examples/stanford_town/actions/decide_to_talk.py index cabb5ceff..414ee7e36 100644 --- a/examples/st_game/actions/decide_to_talk.py +++ b/examples/stanford_town/actions/decide_to_talk.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # @Desc : device to talk to another role, return yes or no -from examples.st_game.actions.st_action import STAction +from examples.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/st_game/actions/dummy_action.py b/examples/stanford_town/actions/dummy_action.py similarity index 100% rename from examples/st_game/actions/dummy_action.py rename to examples/stanford_town/actions/dummy_action.py diff --git a/examples/st_game/actions/gen_action_details.py b/examples/stanford_town/actions/gen_action_details.py similarity index 99% rename from examples/st_game/actions/gen_action_details.py rename to examples/stanford_town/actions/gen_action_details.py index 6af2cb338..0eb7cb701 100644 --- a/examples/st_game/actions/gen_action_details.py +++ b/examples/stanford_town/actions/gen_action_details.py @@ -4,7 +4,7 @@ import random -from metagpt.environment.stanford_town_env.env_space import EnvObsParams, EnvObsType +from metagpt.environment.stanford_town.env_space import EnvObsParams, EnvObsType from metagpt.logs import logger from .st_action import STAction diff --git a/examples/st_game/actions/gen_daily_schedule.py b/examples/stanford_town/actions/gen_daily_schedule.py similarity index 100% rename from examples/st_game/actions/gen_daily_schedule.py rename to examples/stanford_town/actions/gen_daily_schedule.py diff --git a/examples/st_game/actions/gen_hourly_schedule.py b/examples/stanford_town/actions/gen_hourly_schedule.py similarity index 100% rename from examples/st_game/actions/gen_hourly_schedule.py rename to examples/stanford_town/actions/gen_hourly_schedule.py diff --git a/examples/st_game/actions/gen_iter_chat_utt.py b/examples/stanford_town/actions/gen_iter_chat_utt.py similarity index 95% rename from examples/st_game/actions/gen_iter_chat_utt.py rename to examples/stanford_town/actions/gen_iter_chat_utt.py index eb5f569c7..82104b6ca 100644 --- a/examples/st_game/actions/gen_iter_chat_utt.py +++ b/examples/stanford_town/actions/gen_iter_chat_utt.py @@ -2,9 +2,9 @@ # -*- coding: utf-8 -*- # @Desc : generate_iterative_chat_utt -from examples.st_game.actions.st_action import STAction -from examples.st_game.utils.utils import extract_first_json_dict -from metagpt.environment.stanford_town_env.env_space import EnvObsParams, EnvObsType +from examples.stanford_town.actions.st_action import STAction +from examples.stanford_town.utils.utils import extract_first_json_dict +from metagpt.environment.stanford_town.env_space import EnvObsParams, EnvObsType from metagpt.logs import logger diff --git a/examples/st_game/actions/inner_voice_action.py b/examples/stanford_town/actions/inner_voice_action.py similarity index 94% rename from examples/st_game/actions/inner_voice_action.py rename to examples/stanford_town/actions/inner_voice_action.py index cfbdf0303..121f1dcee 100644 --- a/examples/st_game/actions/inner_voice_action.py +++ b/examples/stanford_town/actions/inner_voice_action.py @@ -1,4 +1,4 @@ -from examples.st_game.actions.st_action import STAction +from examples.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/st_game/actions/new_decomp_schedule.py b/examples/stanford_town/actions/new_decomp_schedule.py similarity index 98% rename from examples/st_game/actions/new_decomp_schedule.py rename to examples/stanford_town/actions/new_decomp_schedule.py index 8e5c79401..c7078be9c 100644 --- a/examples/st_game/actions/new_decomp_schedule.py +++ b/examples/stanford_town/actions/new_decomp_schedule.py @@ -4,7 +4,7 @@ import datetime -from examples.st_game.actions.st_action import STAction +from examples.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/st_game/actions/run_reflect_action.py b/examples/stanford_town/actions/run_reflect_action.py similarity index 99% rename from examples/st_game/actions/run_reflect_action.py rename to examples/stanford_town/actions/run_reflect_action.py index d3bf008cb..055c03db4 100644 --- a/examples/st_game/actions/run_reflect_action.py +++ b/examples/stanford_town/actions/run_reflect_action.py @@ -4,7 +4,7 @@ import re -from examples.st_game.actions.st_action import STAction +from examples.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/st_game/actions/st_action.py b/examples/stanford_town/actions/st_action.py similarity index 98% rename from examples/st_game/actions/st_action.py rename to examples/stanford_town/actions/st_action.py index 81f66c080..72ef851e0 100644 --- a/examples/st_game/actions/st_action.py +++ b/examples/stanford_town/actions/st_action.py @@ -7,7 +7,7 @@ from abc import abstractmethod from pathlib import Path from typing import Any, Optional, Union -from examples.st_game.utils.const import PROMPTS_DIR +from examples.stanford_town.utils.const import PROMPTS_DIR from metagpt.actions.action import Action from metagpt.config2 import config from metagpt.logs import logger diff --git a/examples/st_game/actions/summarize_conv.py b/examples/stanford_town/actions/summarize_conv.py similarity index 96% rename from examples/st_game/actions/summarize_conv.py rename to examples/stanford_town/actions/summarize_conv.py index f35651b35..f01fb36ad 100644 --- a/examples/st_game/actions/summarize_conv.py +++ b/examples/stanford_town/actions/summarize_conv.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # @Desc : summarize the content of agents' conversation -from examples.st_game.actions.st_action import STAction +from examples.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/st_game/actions/task_decomp.py b/examples/stanford_town/actions/task_decomp.py similarity index 100% rename from examples/st_game/actions/task_decomp.py rename to examples/stanford_town/actions/task_decomp.py diff --git a/examples/st_game/actions/wake_up.py b/examples/stanford_town/actions/wake_up.py similarity index 100% rename from examples/st_game/actions/wake_up.py rename to examples/stanford_town/actions/wake_up.py diff --git a/examples/st_game/memory/__init__.py b/examples/stanford_town/memory/__init__.py similarity index 100% rename from examples/st_game/memory/__init__.py rename to examples/stanford_town/memory/__init__.py diff --git a/examples/st_game/memory/agent_memory.py b/examples/stanford_town/memory/agent_memory.py similarity index 100% rename from examples/st_game/memory/agent_memory.py rename to examples/stanford_town/memory/agent_memory.py diff --git a/examples/st_game/memory/retrieve.py b/examples/stanford_town/memory/retrieve.py similarity index 97% rename from examples/st_game/memory/retrieve.py rename to examples/stanford_town/memory/retrieve.py index eeeecf85a..db3a87bea 100644 --- a/examples/st_game/memory/retrieve.py +++ b/examples/stanford_town/memory/retrieve.py @@ -7,8 +7,8 @@ import datetime from numpy import dot from numpy.linalg import norm -from examples.st_game.memory.agent_memory import BasicMemory -from examples.st_game.utils.utils import get_embedding +from examples.stanford_town.memory.agent_memory import BasicMemory +from examples.stanford_town.utils.utils import get_embedding def agent_retrieve( diff --git a/examples/st_game/memory/scratch.py b/examples/stanford_town/memory/scratch.py similarity index 100% rename from examples/st_game/memory/scratch.py rename to examples/stanford_town/memory/scratch.py diff --git a/examples/st_game/memory/spatial_memory.py b/examples/stanford_town/memory/spatial_memory.py similarity index 100% rename from examples/st_game/memory/spatial_memory.py rename to examples/stanford_town/memory/spatial_memory.py diff --git a/examples/st_game/plan/__init__.py b/examples/stanford_town/plan/__init__.py similarity index 100% rename from examples/st_game/plan/__init__.py rename to examples/stanford_town/plan/__init__.py diff --git a/examples/st_game/plan/converse.py b/examples/stanford_town/plan/converse.py similarity index 94% rename from examples/st_game/plan/converse.py rename to examples/stanford_town/plan/converse.py index 0371b3d44..b0ff54b68 100644 --- a/examples/st_game/plan/converse.py +++ b/examples/stanford_town/plan/converse.py @@ -4,9 +4,9 @@ from typing import Tuple -from examples.st_game.actions.agent_chat_sum_rel import AgentChatSumRel -from examples.st_game.actions.gen_iter_chat_utt import GenIterChatUTT -from examples.st_game.memory.retrieve import new_agent_retrieve +from examples.stanford_town.actions.agent_chat_sum_rel import AgentChatSumRel +from examples.stanford_town.actions.gen_iter_chat_utt import GenIterChatUTT +from examples.stanford_town.memory.retrieve import new_agent_retrieve from metagpt.logs import logger diff --git a/examples/st_game/plan/st_plan.py b/examples/stanford_town/plan/st_plan.py similarity index 100% rename from examples/st_game/plan/st_plan.py rename to examples/stanford_town/plan/st_plan.py diff --git a/examples/st_game/prompts/__init__.py b/examples/stanford_town/prompts/__init__.py similarity index 100% rename from examples/st_game/prompts/__init__.py rename to examples/stanford_town/prompts/__init__.py diff --git a/examples/st_game/prompts/action_location_object_vMar11.txt b/examples/stanford_town/prompts/action_location_object_vMar11.txt similarity index 100% rename from examples/st_game/prompts/action_location_object_vMar11.txt rename to examples/stanford_town/prompts/action_location_object_vMar11.txt diff --git a/examples/st_game/prompts/action_location_sector_v1.txt b/examples/stanford_town/prompts/action_location_sector_v1.txt similarity index 100% rename from examples/st_game/prompts/action_location_sector_v1.txt rename to examples/stanford_town/prompts/action_location_sector_v1.txt diff --git a/examples/st_game/prompts/action_object_v2.txt b/examples/stanford_town/prompts/action_object_v2.txt similarity index 100% rename from examples/st_game/prompts/action_object_v2.txt rename to examples/stanford_town/prompts/action_object_v2.txt diff --git a/examples/st_game/prompts/daily_planning_v6.txt b/examples/stanford_town/prompts/daily_planning_v6.txt similarity index 100% rename from examples/st_game/prompts/daily_planning_v6.txt rename to examples/stanford_town/prompts/daily_planning_v6.txt diff --git a/examples/st_game/prompts/decide_to_talk_v2.txt b/examples/stanford_town/prompts/decide_to_talk_v2.txt similarity index 100% rename from examples/st_game/prompts/decide_to_talk_v2.txt rename to examples/stanford_town/prompts/decide_to_talk_v2.txt diff --git a/examples/st_game/prompts/generate_event_triple_v1.txt b/examples/stanford_town/prompts/generate_event_triple_v1.txt similarity index 100% rename from examples/st_game/prompts/generate_event_triple_v1.txt rename to examples/stanford_town/prompts/generate_event_triple_v1.txt diff --git a/examples/st_game/prompts/generate_focal_pt_v1.txt b/examples/stanford_town/prompts/generate_focal_pt_v1.txt similarity index 100% rename from examples/st_game/prompts/generate_focal_pt_v1.txt rename to examples/stanford_town/prompts/generate_focal_pt_v1.txt diff --git a/examples/st_game/prompts/generate_hourly_schedule_v2.txt b/examples/stanford_town/prompts/generate_hourly_schedule_v2.txt similarity index 100% rename from examples/st_game/prompts/generate_hourly_schedule_v2.txt rename to examples/stanford_town/prompts/generate_hourly_schedule_v2.txt diff --git a/examples/st_game/prompts/generate_obj_event_v1.txt b/examples/stanford_town/prompts/generate_obj_event_v1.txt similarity index 100% rename from examples/st_game/prompts/generate_obj_event_v1.txt rename to examples/stanford_town/prompts/generate_obj_event_v1.txt diff --git a/examples/st_game/prompts/generate_pronunciatio_v1.txt b/examples/stanford_town/prompts/generate_pronunciatio_v1.txt similarity index 100% rename from examples/st_game/prompts/generate_pronunciatio_v1.txt rename to examples/stanford_town/prompts/generate_pronunciatio_v1.txt diff --git a/examples/st_game/prompts/insight_and_evidence_v1.txt b/examples/stanford_town/prompts/insight_and_evidence_v1.txt similarity index 100% rename from examples/st_game/prompts/insight_and_evidence_v1.txt rename to examples/stanford_town/prompts/insight_and_evidence_v1.txt diff --git a/examples/st_game/prompts/iterative_convo_v1.txt b/examples/stanford_town/prompts/iterative_convo_v1.txt similarity index 100% rename from examples/st_game/prompts/iterative_convo_v1.txt rename to examples/stanford_town/prompts/iterative_convo_v1.txt diff --git a/examples/st_game/prompts/memo_on_convo_v1.txt b/examples/stanford_town/prompts/memo_on_convo_v1.txt similarity index 100% rename from examples/st_game/prompts/memo_on_convo_v1.txt rename to examples/stanford_town/prompts/memo_on_convo_v1.txt diff --git a/examples/st_game/prompts/new_decomp_schedule_v1.txt b/examples/stanford_town/prompts/new_decomp_schedule_v1.txt similarity index 100% rename from examples/st_game/prompts/new_decomp_schedule_v1.txt rename to examples/stanford_town/prompts/new_decomp_schedule_v1.txt diff --git a/examples/st_game/prompts/planning_thought_on_convo_v1.txt b/examples/stanford_town/prompts/planning_thought_on_convo_v1.txt similarity index 100% rename from examples/st_game/prompts/planning_thought_on_convo_v1.txt rename to examples/stanford_town/prompts/planning_thought_on_convo_v1.txt diff --git a/examples/st_game/prompts/poignancy_action_v1.txt b/examples/stanford_town/prompts/poignancy_action_v1.txt similarity index 100% rename from examples/st_game/prompts/poignancy_action_v1.txt rename to examples/stanford_town/prompts/poignancy_action_v1.txt diff --git a/examples/st_game/prompts/poignancy_chat_v1.txt b/examples/stanford_town/prompts/poignancy_chat_v1.txt similarity index 100% rename from examples/st_game/prompts/poignancy_chat_v1.txt rename to examples/stanford_town/prompts/poignancy_chat_v1.txt diff --git a/examples/st_game/prompts/poignancy_event_v1.txt b/examples/stanford_town/prompts/poignancy_event_v1.txt similarity index 100% rename from examples/st_game/prompts/poignancy_event_v1.txt rename to examples/stanford_town/prompts/poignancy_event_v1.txt diff --git a/examples/st_game/prompts/poignancy_thought_v1.txt b/examples/stanford_town/prompts/poignancy_thought_v1.txt similarity index 100% rename from examples/st_game/prompts/poignancy_thought_v1.txt rename to examples/stanford_town/prompts/poignancy_thought_v1.txt diff --git a/examples/st_game/prompts/summarize_chat_relationship_v2.txt b/examples/stanford_town/prompts/summarize_chat_relationship_v2.txt similarity index 100% rename from examples/st_game/prompts/summarize_chat_relationship_v2.txt rename to examples/stanford_town/prompts/summarize_chat_relationship_v2.txt diff --git a/examples/st_game/prompts/summarize_conversation_v1.txt b/examples/stanford_town/prompts/summarize_conversation_v1.txt similarity index 100% rename from examples/st_game/prompts/summarize_conversation_v1.txt rename to examples/stanford_town/prompts/summarize_conversation_v1.txt diff --git a/examples/st_game/prompts/task_decomp_v3.txt b/examples/stanford_town/prompts/task_decomp_v3.txt similarity index 100% rename from examples/st_game/prompts/task_decomp_v3.txt rename to examples/stanford_town/prompts/task_decomp_v3.txt diff --git a/examples/st_game/prompts/wake_up_hour_v1.txt b/examples/stanford_town/prompts/wake_up_hour_v1.txt similarity index 100% rename from examples/st_game/prompts/wake_up_hour_v1.txt rename to examples/stanford_town/prompts/wake_up_hour_v1.txt diff --git a/examples/st_game/prompts/whisper_inner_thought_v1.txt b/examples/stanford_town/prompts/whisper_inner_thought_v1.txt similarity index 100% rename from examples/st_game/prompts/whisper_inner_thought_v1.txt rename to examples/stanford_town/prompts/whisper_inner_thought_v1.txt diff --git a/examples/st_game/reflect/__init__.py b/examples/stanford_town/reflect/__init__.py similarity index 100% rename from examples/st_game/reflect/__init__.py rename to examples/stanford_town/reflect/__init__.py diff --git a/examples/st_game/reflect/reflect.py b/examples/stanford_town/reflect/reflect.py similarity index 97% rename from examples/st_game/reflect/reflect.py rename to examples/stanford_town/reflect/reflect.py index 0c855b8f1..beec53dc2 100644 --- a/examples/st_game/reflect/reflect.py +++ b/examples/stanford_town/reflect/reflect.py @@ -5,7 +5,7 @@ import datetime import time -from examples.st_game.actions.run_reflect_action import ( +from examples.stanford_town.actions.run_reflect_action import ( AgentChatPoignancy, AgentEventPoignancy, AgentEventTriple, @@ -14,8 +14,8 @@ from examples.st_game.actions.run_reflect_action import ( AgentMemoryOnConvo, AgentPlanThoughtOnConvo, ) -from examples.st_game.memory.retrieve import new_agent_retrieve -from examples.st_game.utils.utils import get_embedding +from examples.stanford_town.memory.retrieve import new_agent_retrieve +from examples.stanford_town.utils.utils import get_embedding from metagpt.logs import logger diff --git a/examples/st_game/requirements.txt b/examples/stanford_town/requirements.txt similarity index 100% rename from examples/st_game/requirements.txt rename to examples/stanford_town/requirements.txt diff --git a/examples/st_game/roles/__init__.py b/examples/stanford_town/roles/__init__.py similarity index 100% rename from examples/st_game/roles/__init__.py rename to examples/stanford_town/roles/__init__.py diff --git a/examples/st_game/roles/st_role.py b/examples/stanford_town/roles/st_role.py similarity index 96% rename from examples/st_game/roles/st_role.py rename to examples/stanford_town/roles/st_role.py index 48de34f15..414fd40b8 100644 --- a/examples/st_game/roles/st_role.py +++ b/examples/stanford_town/roles/st_role.py @@ -20,23 +20,23 @@ from typing import TYPE_CHECKING, Optional from pydantic import ConfigDict, Field, field_validator, model_validator -from examples.st_game.actions.dummy_action import DummyAction, DummyMessage -from examples.st_game.actions.inner_voice_action import AgentWhisperThoughtAction -from examples.st_game.actions.run_reflect_action import AgentEventTriple -from examples.st_game.memory.agent_memory import AgentMemory, BasicMemory -from examples.st_game.memory.scratch import Scratch -from examples.st_game.memory.spatial_memory import MemoryTree -from examples.st_game.plan.st_plan import plan -from examples.st_game.reflect.reflect import generate_poig_score, role_reflect -from examples.st_game.utils.const import STORAGE_PATH, collision_block_id -from examples.st_game.utils.mg_ga_transform import ( +from examples.stanford_town.actions.dummy_action import DummyAction, DummyMessage +from examples.stanford_town.actions.inner_voice_action import AgentWhisperThoughtAction +from examples.stanford_town.actions.run_reflect_action import AgentEventTriple +from examples.stanford_town.memory.agent_memory import AgentMemory, BasicMemory +from examples.stanford_town.memory.scratch import Scratch +from examples.stanford_town.memory.spatial_memory import MemoryTree +from examples.stanford_town.plan.st_plan import plan +from examples.stanford_town.reflect.reflect import generate_poig_score, role_reflect +from examples.stanford_town.utils.const import STORAGE_PATH, collision_block_id +from examples.stanford_town.utils.mg_ga_transform import ( get_role_environment, save_environment, save_movement, ) -from examples.st_game.utils.utils import get_embedding, path_finder +from examples.stanford_town.utils.utils import get_embedding, path_finder from metagpt.actions.add_requirement import UserRequirement -from metagpt.environment.stanford_town_env.env_space import ( +from metagpt.environment.stanford_town.env_space import ( EnvAction, EnvActionType, EnvObsParams, @@ -48,7 +48,7 @@ from metagpt.schema import Message from metagpt.utils.common import any_to_str if TYPE_CHECKING: - from metagpt.environment.stanford_town_env.stanford_town_env import ( # noqa: F401 + from metagpt.environment.stanford_town.stanford_town_env import ( # noqa: F401 StanfordTownEnv, ) @@ -63,7 +63,7 @@ class STRoleContext(RoleContext): @classmethod def model_rebuild(cls, **kwargs): - from metagpt.environment.stanford_town_env.stanford_town_env import ( # noqa: F401 + from metagpt.environment.stanford_town.stanford_town_env import ( # noqa: F401 StanfordTownEnv, ) diff --git a/examples/st_game/run_st_game.py b/examples/stanford_town/run_st_game.py similarity index 89% rename from examples/st_game/run_st_game.py rename to examples/stanford_town/run_st_game.py index 304bf24ba..4c3b3bb0e 100644 --- a/examples/st_game/run_st_game.py +++ b/examples/stanford_town/run_st_game.py @@ -7,15 +7,15 @@ from typing import Optional import fire -from examples.st_game.roles.st_role import STRole -from examples.st_game.stanford_town import StanfordTown -from examples.st_game.utils.const import STORAGE_PATH -from examples.st_game.utils.mg_ga_transform import ( +from examples.stanford_town.roles.st_role import STRole +from examples.stanford_town.stanford_town import StanfordTown +from examples.stanford_town.utils.const import STORAGE_PATH +from examples.stanford_town.utils.mg_ga_transform import ( get_reverie_meta, write_curr_sim_code, write_curr_step, ) -from examples.st_game.utils.utils import copy_folder +from examples.stanford_town.utils.utils import copy_folder from metagpt.logs import logger diff --git a/examples/st_game/stanford_town.py b/examples/stanford_town/stanford_town.py similarity index 87% rename from examples/st_game/stanford_town.py rename to examples/stanford_town/stanford_town.py index f10332b60..7c39cbf04 100644 --- a/examples/st_game/stanford_town.py +++ b/examples/stanford_town/stanford_town.py @@ -4,10 +4,10 @@ from typing import Any, Optional -from examples.st_game.roles.st_role import STRole -from examples.st_game.utils.const import MAZE_ASSET_PATH +from examples.stanford_town.roles.st_role import STRole +from examples.stanford_town.utils.const import MAZE_ASSET_PATH from metagpt.context import Context -from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv +from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv from metagpt.logs import logger from metagpt.team import Team diff --git a/examples/st_game/static_dirs/assets/the_ville/agent_history_init_n25.csv b/examples/stanford_town/static_dirs/assets/the_ville/agent_history_init_n25.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/agent_history_init_n25.csv rename to examples/stanford_town/static_dirs/assets/the_ville/agent_history_init_n25.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/agent_history_init_n3.csv b/examples/stanford_town/static_dirs/assets/the_ville/agent_history_init_n3.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/agent_history_init_n3.csv rename to examples/stanford_town/static_dirs/assets/the_ville/agent_history_init_n3.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/maze/arena_maze.csv b/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/arena_maze.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/maze/arena_maze.csv rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/arena_maze.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/maze/collision_maze.csv b/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/collision_maze.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/maze/collision_maze.csv rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/collision_maze.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/maze/game_object_maze.csv b/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/game_object_maze.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/maze/game_object_maze.csv rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/game_object_maze.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/maze/sector_maze.csv b/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/sector_maze.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/maze/sector_maze.csv rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/sector_maze.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/maze/spawning_location_maze.csv b/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/spawning_location_maze.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/maze/spawning_location_maze.csv rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/spawning_location_maze.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/maze_meta_info.json b/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze_meta_info.json similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/maze_meta_info.json rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/maze_meta_info.json diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/special_blocks/arena_blocks.csv b/examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/arena_blocks.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/special_blocks/arena_blocks.csv rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/arena_blocks.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/special_blocks/game_object_blocks.csv b/examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/game_object_blocks.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/special_blocks/game_object_blocks.csv rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/game_object_blocks.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/special_blocks/sector_blocks.csv b/examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/sector_blocks.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/special_blocks/sector_blocks.csv rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/sector_blocks.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/special_blocks/spawning_location_blocks.csv b/examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/spawning_location_blocks.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/special_blocks/spawning_location_blocks.csv rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/spawning_location_blocks.csv diff --git a/examples/st_game/static_dirs/assets/the_ville/matrix/special_blocks/world_blocks.csv b/examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/world_blocks.csv similarity index 100% rename from examples/st_game/static_dirs/assets/the_ville/matrix/special_blocks/world_blocks.csv rename to examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/world_blocks.csv diff --git a/examples/st_game/storage/.gitignore b/examples/stanford_town/storage/.gitignore similarity index 100% rename from examples/st_game/storage/.gitignore rename to examples/stanford_town/storage/.gitignore diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/environment/0.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/environment/0.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/environment/0.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/environment/0.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/embeddings.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/embeddings.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/embeddings.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/embeddings.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/kw_strength.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/kw_strength.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/kw_strength.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/kw_strength.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/nodes.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/nodes.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/nodes.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/nodes.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/scratch.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/scratch.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/scratch.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/scratch.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/spatial_memory.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/spatial_memory.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/spatial_memory.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/spatial_memory.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/embeddings.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/embeddings.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/embeddings.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/embeddings.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/kw_strength.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/kw_strength.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/kw_strength.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/kw_strength.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/nodes.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/nodes.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/nodes.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/nodes.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/scratch.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/scratch.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/scratch.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/scratch.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/spatial_memory.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/spatial_memory.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/spatial_memory.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/spatial_memory.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/embeddings.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/embeddings.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/embeddings.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/embeddings.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/kw_strength.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/kw_strength.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/kw_strength.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/kw_strength.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/nodes.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/nodes.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/nodes.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/nodes.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/scratch.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/scratch.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/scratch.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/scratch.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/spatial_memory.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/spatial_memory.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/spatial_memory.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/spatial_memory.json diff --git a/examples/st_game/storage/base_the_ville_isabella_maria_klaus/reverie/meta.json b/examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/reverie/meta.json similarity index 100% rename from examples/st_game/storage/base_the_ville_isabella_maria_klaus/reverie/meta.json rename to examples/stanford_town/storage/base_the_ville_isabella_maria_klaus/reverie/meta.json diff --git a/examples/st_game/tests/__init__.py b/examples/stanford_town/tests/__init__.py similarity index 100% rename from examples/st_game/tests/__init__.py rename to examples/stanford_town/tests/__init__.py diff --git a/examples/st_game/tests/actions/__init__.py b/examples/stanford_town/tests/actions/__init__.py similarity index 100% rename from examples/st_game/tests/actions/__init__.py rename to examples/stanford_town/tests/actions/__init__.py diff --git a/examples/st_game/tests/actions/test_gen_action_details.py b/examples/stanford_town/tests/actions/test_gen_action_details.py similarity index 90% rename from examples/st_game/tests/actions/test_gen_action_details.py rename to examples/stanford_town/tests/actions/test_gen_action_details.py index 49e24481d..bf6b75c04 100644 --- a/examples/st_game/tests/actions/test_gen_action_details.py +++ b/examples/stanford_town/tests/actions/test_gen_action_details.py @@ -4,17 +4,17 @@ import pytest -from examples.st_game.actions.gen_action_details import ( +from examples.stanford_town.actions.gen_action_details import ( GenActionArena, GenActionDetails, GenActionObject, GenActionSector, GenActObjDescription, ) -from examples.st_game.roles.st_role import STRole -from examples.st_game.utils.const import MAZE_ASSET_PATH +from examples.stanford_town.roles.st_role import STRole +from examples.stanford_town.utils.const import MAZE_ASSET_PATH from metagpt.environment.api.env_api import EnvAPIAbstract -from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv +from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv @pytest.mark.asyncio diff --git a/examples/st_game/tests/actions/test_summarize_conv.py b/examples/stanford_town/tests/actions/test_summarize_conv.py similarity index 83% rename from examples/st_game/tests/actions/test_summarize_conv.py rename to examples/stanford_town/tests/actions/test_summarize_conv.py index 9174a558a..fa0de2031 100644 --- a/examples/st_game/tests/actions/test_summarize_conv.py +++ b/examples/stanford_town/tests/actions/test_summarize_conv.py @@ -4,7 +4,7 @@ import pytest -from examples.st_game.actions.summarize_conv import SummarizeConv +from examples.stanford_town.actions.summarize_conv import SummarizeConv @pytest.mark.asyncio diff --git a/examples/st_game/tests/plan/__init__.py b/examples/stanford_town/tests/plan/__init__.py similarity index 100% rename from examples/st_game/tests/plan/__init__.py rename to examples/stanford_town/tests/plan/__init__.py diff --git a/examples/st_game/tests/plan/test_conversation.py b/examples/stanford_town/tests/plan/test_conversation.py similarity index 80% rename from examples/st_game/tests/plan/test_conversation.py rename to examples/stanford_town/tests/plan/test_conversation.py index 3a158fc5f..86c2f0b94 100644 --- a/examples/st_game/tests/plan/test_conversation.py +++ b/examples/stanford_town/tests/plan/test_conversation.py @@ -6,12 +6,12 @@ from typing import Tuple import pytest -from examples.st_game.plan.converse import agent_conversation -from examples.st_game.roles.st_role import STRole -from examples.st_game.utils.const import MAZE_ASSET_PATH, STORAGE_PATH -from examples.st_game.utils.mg_ga_transform import get_reverie_meta -from examples.st_game.utils.utils import copy_folder -from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv +from examples.stanford_town.plan.converse import agent_conversation +from examples.stanford_town.roles.st_role import STRole +from examples.stanford_town.utils.const import MAZE_ASSET_PATH, STORAGE_PATH +from examples.stanford_town.utils.mg_ga_transform import get_reverie_meta +from examples.stanford_town.utils.utils import copy_folder +from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv async def init_two_roles(fork_sim_code: str = "base_the_ville_isabella_maria_klaus") -> Tuple["STRole"]: diff --git a/examples/st_game/tests/plan/test_st_plan.py b/examples/stanford_town/tests/plan/test_st_plan.py similarity index 87% rename from examples/st_game/tests/plan/test_st_plan.py rename to examples/stanford_town/tests/plan/test_st_plan.py index 82e3c56c2..ad20d3068 100644 --- a/examples/st_game/tests/plan/test_st_plan.py +++ b/examples/stanford_town/tests/plan/test_st_plan.py @@ -4,8 +4,12 @@ import pytest -from examples.st_game.plan.st_plan import _choose_retrieved, _should_react, _wait_react -from examples.st_game.tests.plan.test_converse import init_two_roles +from examples.stanford_town.plan.st_plan import ( + _choose_retrieved, + _should_react, + _wait_react, +) +from examples.stanford_town.tests.plan.test_converse import init_two_roles def test_should_react(): diff --git a/examples/st_game/tests/roles/__init__.py b/examples/stanford_town/tests/roles/__init__.py similarity index 100% rename from examples/st_game/tests/roles/__init__.py rename to examples/stanford_town/tests/roles/__init__.py diff --git a/examples/st_game/tests/roles/test_st_role.py b/examples/stanford_town/tests/roles/test_st_role.py similarity index 67% rename from examples/st_game/tests/roles/test_st_role.py rename to examples/stanford_town/tests/roles/test_st_role.py index c0844d10c..b8feecb2a 100644 --- a/examples/st_game/tests/roles/test_st_role.py +++ b/examples/stanford_town/tests/roles/test_st_role.py @@ -4,10 +4,10 @@ import pytest -from examples.st_game.memory.agent_memory import BasicMemory -from examples.st_game.roles.st_role import STRole -from examples.st_game.utils.const import MAZE_ASSET_PATH -from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv +from examples.stanford_town.memory.agent_memory import BasicMemory +from examples.stanford_town.roles.st_role import STRole +from examples.stanford_town.utils.const import MAZE_ASSET_PATH +from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv @pytest.mark.asyncio diff --git a/examples/st_game/tests/test_agent_memory.py b/examples/stanford_town/tests/test_agent_memory.py similarity index 94% rename from examples/st_game/tests/test_agent_memory.py rename to examples/stanford_town/tests/test_agent_memory.py index d5a75163c..e7ad3fc43 100644 --- a/examples/st_game/tests/test_agent_memory.py +++ b/examples/stanford_town/tests/test_agent_memory.py @@ -6,9 +6,9 @@ from datetime import datetime, timedelta import pytest -from examples.st_game.memory.agent_memory import AgentMemory -from examples.st_game.memory.retrieve import agent_retrieve -from examples.st_game.utils.const import STORAGE_PATH +from examples.stanford_town.memory.agent_memory import AgentMemory +from examples.stanford_town.memory.retrieve import agent_retrieve +from examples.stanford_town.utils.const import STORAGE_PATH from metagpt.logs import logger """ diff --git a/examples/st_game/tests/test_basic_memory.py b/examples/stanford_town/tests/test_basic_memory.py similarity index 96% rename from examples/st_game/tests/test_basic_memory.py rename to examples/stanford_town/tests/test_basic_memory.py index 25d350263..fa2a5c487 100644 --- a/examples/st_game/tests/test_basic_memory.py +++ b/examples/stanford_town/tests/test_basic_memory.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta import pytest -from examples.st_game.memory.agent_memory import BasicMemory +from examples.stanford_town.memory.agent_memory import BasicMemory from metagpt.logs import logger """ diff --git a/examples/st_game/tests/test_reflect.py b/examples/stanford_town/tests/test_reflect.py similarity index 93% rename from examples/st_game/tests/test_reflect.py rename to examples/stanford_town/tests/test_reflect.py index 6672b0545..48422d073 100644 --- a/examples/st_game/tests/test_reflect.py +++ b/examples/stanford_town/tests/test_reflect.py @@ -4,14 +4,14 @@ import pytest -from examples.st_game.actions.run_reflect_action import ( +from examples.stanford_town.actions.run_reflect_action import ( AgentEventTriple, AgentFocusPt, AgentInsightAndGuidance, ) -from examples.st_game.roles.st_role import STRole -from examples.st_game.utils.const import MAZE_ASSET_PATH -from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv +from examples.stanford_town.roles.st_role import STRole +from examples.stanford_town.utils.const import MAZE_ASSET_PATH +from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv @pytest.mark.asyncio diff --git a/examples/st_game/tests/test_spatial_memory.py b/examples/stanford_town/tests/test_spatial_memory.py similarity index 100% rename from examples/st_game/tests/test_spatial_memory.py rename to examples/stanford_town/tests/test_spatial_memory.py diff --git a/examples/st_game/utils/__init__.py b/examples/stanford_town/utils/__init__.py similarity index 100% rename from examples/st_game/utils/__init__.py rename to examples/stanford_town/utils/__init__.py diff --git a/examples/st_game/utils/const.py b/examples/stanford_town/utils/const.py similarity index 100% rename from examples/st_game/utils/const.py rename to examples/stanford_town/utils/const.py diff --git a/examples/st_game/utils/mg_ga_transform.py b/examples/stanford_town/utils/mg_ga_transform.py similarity index 96% rename from examples/st_game/utils/mg_ga_transform.py rename to examples/stanford_town/utils/mg_ga_transform.py index 9ff57a3b1..521199a05 100644 --- a/examples/st_game/utils/mg_ga_transform.py +++ b/examples/stanford_town/utils/mg_ga_transform.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Optional -from examples.st_game.utils.const import STORAGE_PATH, TEMP_STORAGE_PATH +from examples.stanford_town.utils.const import STORAGE_PATH, TEMP_STORAGE_PATH from metagpt.logs import logger from metagpt.utils.common import read_json_file, write_json_file diff --git a/examples/st_game/utils/utils.py b/examples/stanford_town/utils/utils.py similarity index 100% rename from examples/st_game/utils/utils.py rename to examples/stanford_town/utils/utils.py diff --git a/metagpt/environment/__init__.py b/metagpt/environment/__init__.py index 28981f2f8..0621fff16 100644 --- a/metagpt/environment/__init__.py +++ b/metagpt/environment/__init__.py @@ -3,10 +3,10 @@ # @Desc : from metagpt.environment.base_env import Environment -from metagpt.environment.android_env.android_env import AndroidEnv -from metagpt.environment.werewolf_env.werewolf_env import WerewolfEnv -from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv -from metagpt.environment.software_env.software_env import SoftwareEnv +from metagpt.environment.android.android_env import AndroidEnv +from metagpt.environment.werewolf.werewolf_env import WerewolfEnv +from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv +from metagpt.environment.software.software_env import SoftwareEnv __all__ = ["AndroidEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"] diff --git a/metagpt/environment/android_env/__init__.py b/metagpt/environment/android/__init__.py similarity index 100% rename from metagpt/environment/android_env/__init__.py rename to metagpt/environment/android/__init__.py diff --git a/metagpt/environment/android_env/android_env.py b/metagpt/environment/android/android_env.py similarity index 83% rename from metagpt/environment/android_env/android_env.py rename to metagpt/environment/android/android_env.py index c27e20541..b3227df0f 100644 --- a/metagpt/environment/android_env/android_env.py +++ b/metagpt/environment/android/android_env.py @@ -4,7 +4,7 @@ from pydantic import Field -from metagpt.environment.android_env.android_ext_env import AndroidExtEnv +from metagpt.environment.android.android_ext_env import AndroidExtEnv from metagpt.environment.base_env import Environment diff --git a/metagpt/environment/android_env/android_ext_env.py b/metagpt/environment/android/android_ext_env.py similarity index 99% rename from metagpt/environment/android_env/android_ext_env.py rename to metagpt/environment/android/android_ext_env.py index 01a24c5b9..d2344fa1f 100644 --- a/metagpt/environment/android_env/android_ext_env.py +++ b/metagpt/environment/android/android_ext_env.py @@ -8,7 +8,7 @@ from typing import Any, Optional from pydantic import Field -from metagpt.environment.android_env.const import ADB_EXEC_FAIL +from metagpt.environment.android.const import ADB_EXEC_FAIL from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams diff --git a/metagpt/environment/android_env/const.py b/metagpt/environment/android/const.py similarity index 100% rename from metagpt/environment/android_env/const.py rename to metagpt/environment/android/const.py diff --git a/metagpt/environment/minecraft_env/__init__.py b/metagpt/environment/minecraft/__init__.py similarity index 100% rename from metagpt/environment/minecraft_env/__init__.py rename to metagpt/environment/minecraft/__init__.py diff --git a/metagpt/environment/minecraft_env/const.py b/metagpt/environment/minecraft/const.py similarity index 100% rename from metagpt/environment/minecraft_env/const.py rename to metagpt/environment/minecraft/const.py diff --git a/metagpt/environment/minecraft_env/minecraft_env.py b/metagpt/environment/minecraft/minecraft_env.py similarity index 99% rename from metagpt/environment/minecraft_env/minecraft_env.py rename to metagpt/environment/minecraft/minecraft_env.py index bba35ce21..edc4aca5b 100644 --- a/metagpt/environment/minecraft_env/minecraft_env.py +++ b/metagpt/environment/minecraft/minecraft_env.py @@ -12,8 +12,8 @@ from pydantic import ConfigDict, Field from metagpt.config2 import config as CONFIG from metagpt.environment.base_env import Environment -from metagpt.environment.minecraft_env.const import MC_CKPT_DIR -from metagpt.environment.minecraft_env.minecraft_ext_env import MinecraftExtEnv +from metagpt.environment.minecraft.const import MC_CKPT_DIR +from metagpt.environment.minecraft.minecraft_ext_env import MinecraftExtEnv from metagpt.logs import logger from metagpt.rag.vector_stores.chroma import ChromaVectorStore from metagpt.utils.common import load_mc_skills_code, read_json_file, write_json_file diff --git a/metagpt/environment/minecraft_env/minecraft_ext_env.py b/metagpt/environment/minecraft/minecraft_ext_env.py similarity index 97% rename from metagpt/environment/minecraft_env/minecraft_ext_env.py rename to metagpt/environment/minecraft/minecraft_ext_env.py index 74f417eb0..0436bc3aa 100644 --- a/metagpt/environment/minecraft_env/minecraft_ext_env.py +++ b/metagpt/environment/minecraft/minecraft_ext_env.py @@ -12,14 +12,14 @@ from pydantic import ConfigDict, Field, model_validator from metagpt.environment.base_env import ExtEnv, mark_as_writeable from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams -from metagpt.environment.minecraft_env.const import ( +from metagpt.environment.minecraft.const import ( MC_CKPT_DIR, MC_CORE_INVENTORY_ITEMS, MC_CURRICULUM_OB, MC_DEFAULT_WARMUP, METAGPT_ROOT, ) -from metagpt.environment.minecraft_env.process_monitor import SubprocessMonitor +from metagpt.environment.minecraft.process_monitor import SubprocessMonitor from metagpt.logs import logger @@ -63,7 +63,7 @@ class MinecraftExtEnv(ExtEnv): self.mineflayer = SubprocessMonitor( commands=[ "node", - METAGPT_ROOT.joinpath("metagpt", "environment", "minecraft_env", "mineflayer", "index.js"), + METAGPT_ROOT.joinpath("metagpt", "environment", "minecraft", "mineflayer", "index.js"), str(self.server_port), ], name="mineflayer", diff --git a/metagpt/environment/minecraft_env/mineflayer/.gitignore b/metagpt/environment/minecraft/mineflayer/.gitignore similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/.gitignore rename to metagpt/environment/minecraft/mineflayer/.gitignore diff --git a/metagpt/environment/minecraft_env/mineflayer/.prettierignore b/metagpt/environment/minecraft/mineflayer/.prettierignore similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/.prettierignore rename to metagpt/environment/minecraft/mineflayer/.prettierignore diff --git a/metagpt/environment/minecraft_env/mineflayer/.prettierrc.json b/metagpt/environment/minecraft/mineflayer/.prettierrc.json similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/.prettierrc.json rename to metagpt/environment/minecraft/mineflayer/.prettierrc.json diff --git a/metagpt/environment/minecraft_env/mineflayer/index.js b/metagpt/environment/minecraft/mineflayer/index.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/index.js rename to metagpt/environment/minecraft/mineflayer/index.js diff --git a/metagpt/environment/minecraft_env/mineflayer/lib/observation/base.js b/metagpt/environment/minecraft/mineflayer/lib/observation/base.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/lib/observation/base.js rename to metagpt/environment/minecraft/mineflayer/lib/observation/base.js diff --git a/metagpt/environment/minecraft_env/mineflayer/lib/observation/chests.js b/metagpt/environment/minecraft/mineflayer/lib/observation/chests.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/lib/observation/chests.js rename to metagpt/environment/minecraft/mineflayer/lib/observation/chests.js diff --git a/metagpt/environment/minecraft_env/mineflayer/lib/observation/inventory.js b/metagpt/environment/minecraft/mineflayer/lib/observation/inventory.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/lib/observation/inventory.js rename to metagpt/environment/minecraft/mineflayer/lib/observation/inventory.js diff --git a/metagpt/environment/minecraft_env/mineflayer/lib/observation/onChat.js b/metagpt/environment/minecraft/mineflayer/lib/observation/onChat.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/lib/observation/onChat.js rename to metagpt/environment/minecraft/mineflayer/lib/observation/onChat.js diff --git a/metagpt/environment/minecraft_env/mineflayer/lib/observation/onError.js b/metagpt/environment/minecraft/mineflayer/lib/observation/onError.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/lib/observation/onError.js rename to metagpt/environment/minecraft/mineflayer/lib/observation/onError.js diff --git a/metagpt/environment/minecraft_env/mineflayer/lib/observation/onSave.js b/metagpt/environment/minecraft/mineflayer/lib/observation/onSave.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/lib/observation/onSave.js rename to metagpt/environment/minecraft/mineflayer/lib/observation/onSave.js diff --git a/metagpt/environment/minecraft_env/mineflayer/lib/observation/status.js b/metagpt/environment/minecraft/mineflayer/lib/observation/status.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/lib/observation/status.js rename to metagpt/environment/minecraft/mineflayer/lib/observation/status.js diff --git a/metagpt/environment/minecraft_env/mineflayer/lib/observation/voxels.js b/metagpt/environment/minecraft/mineflayer/lib/observation/voxels.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/lib/observation/voxels.js rename to metagpt/environment/minecraft/mineflayer/lib/observation/voxels.js diff --git a/metagpt/environment/minecraft_env/mineflayer/lib/skillLoader.js b/metagpt/environment/minecraft/mineflayer/lib/skillLoader.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/lib/skillLoader.js rename to metagpt/environment/minecraft/mineflayer/lib/skillLoader.js diff --git a/metagpt/environment/minecraft_env/mineflayer/lib/utils.js b/metagpt/environment/minecraft/mineflayer/lib/utils.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/lib/utils.js rename to metagpt/environment/minecraft/mineflayer/lib/utils.js diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/.gitignore b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/.gitignore similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/.gitignore rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/.gitignore diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/LICENSE b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/LICENSE similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/LICENSE rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/LICENSE diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/README.md b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/README.md similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/README.md rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/README.md diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/_config.yml b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/_config.yml similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/_config.yml rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/_config.yml diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/docs/api.md b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/docs/api.md similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/docs/api.md rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/docs/api.md diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/collector.js b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/examples/collector.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/collector.js rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/examples/collector.js diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/oreMiner.js b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/examples/oreMiner.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/oreMiner.js rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/examples/oreMiner.js diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/storageBot.js b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/examples/storageBot.js similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/examples/storageBot.js rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/examples/storageBot.js diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/package.json b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/package.json similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/package.json rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/package.json diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/BlockVeins.ts b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/BlockVeins.ts similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/BlockVeins.ts rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/BlockVeins.ts diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/CollectBlock.ts b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/CollectBlock.ts similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/CollectBlock.ts rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/CollectBlock.ts diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Inventory.ts b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/Inventory.ts similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Inventory.ts rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/Inventory.ts diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Targets.ts b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/Targets.ts similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Targets.ts rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/Targets.ts diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/TaskQueue.ts b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/TaskQueue.ts similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/TaskQueue.ts rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/TaskQueue.ts diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/TemporarySubscriber.ts b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/TemporarySubscriber.ts similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/TemporarySubscriber.ts rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/TemporarySubscriber.ts diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Util.ts b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/Util.ts similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/Util.ts rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/Util.ts diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/index.ts b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/index.ts similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/src/index.ts rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/src/index.ts diff --git a/metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/tsconfig.json b/metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/tsconfig.json similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/mineflayer-collectblock/tsconfig.json rename to metagpt/environment/minecraft/mineflayer/mineflayer-collectblock/tsconfig.json diff --git a/metagpt/environment/minecraft_env/mineflayer/package.json b/metagpt/environment/minecraft/mineflayer/package.json similarity index 100% rename from metagpt/environment/minecraft_env/mineflayer/package.json rename to metagpt/environment/minecraft/mineflayer/package.json diff --git a/metagpt/environment/minecraft_env/process_monitor.py b/metagpt/environment/minecraft/process_monitor.py similarity index 100% rename from metagpt/environment/minecraft_env/process_monitor.py rename to metagpt/environment/minecraft/process_monitor.py diff --git a/metagpt/environment/software_env/__init__.py b/metagpt/environment/software/__init__.py similarity index 100% rename from metagpt/environment/software_env/__init__.py rename to metagpt/environment/software/__init__.py diff --git a/metagpt/environment/software_env/software_env.py b/metagpt/environment/software/software_env.py similarity index 100% rename from metagpt/environment/software_env/software_env.py rename to metagpt/environment/software/software_env.py diff --git a/metagpt/environment/stanford_town_env/__init__.py b/metagpt/environment/stanford_town/__init__.py similarity index 100% rename from metagpt/environment/stanford_town_env/__init__.py rename to metagpt/environment/stanford_town/__init__.py diff --git a/metagpt/environment/stanford_town_env/env_space.py b/metagpt/environment/stanford_town/env_space.py similarity index 100% rename from metagpt/environment/stanford_town_env/env_space.py rename to metagpt/environment/stanford_town/env_space.py diff --git a/metagpt/environment/stanford_town_env/stanford_town_env.py b/metagpt/environment/stanford_town/stanford_town_env.py similarity index 66% rename from metagpt/environment/stanford_town_env/stanford_town_env.py rename to metagpt/environment/stanford_town/stanford_town_env.py index 8721d6cd1..2b6212781 100644 --- a/metagpt/environment/stanford_town_env/stanford_town_env.py +++ b/metagpt/environment/stanford_town/stanford_town_env.py @@ -3,9 +3,7 @@ # @Desc : MG StanfordTown Env from metagpt.environment.base_env import Environment -from metagpt.environment.stanford_town_env.stanford_town_ext_env import ( - StanfordTownExtEnv, -) +from metagpt.environment.stanford_town.stanford_town_ext_env import StanfordTownExtEnv class StanfordTownEnv(Environment, StanfordTownExtEnv): diff --git a/metagpt/environment/stanford_town_env/stanford_town_ext_env.py b/metagpt/environment/stanford_town/stanford_town_ext_env.py similarity index 99% rename from metagpt/environment/stanford_town_env/stanford_town_ext_env.py rename to metagpt/environment/stanford_town/stanford_town_ext_env.py index b41ae375c..30a02d4db 100644 --- a/metagpt/environment/stanford_town_env/stanford_town_ext_env.py +++ b/metagpt/environment/stanford_town/stanford_town_ext_env.py @@ -10,7 +10,7 @@ from typing import Any, Optional from pydantic import ConfigDict, Field, model_validator from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable -from metagpt.environment.stanford_town_env.env_space import ( +from metagpt.environment.stanford_town.env_space import ( EnvAction, EnvActionType, EnvObsParams, diff --git a/metagpt/environment/werewolf_env/__init__.py b/metagpt/environment/werewolf/__init__.py similarity index 100% rename from metagpt/environment/werewolf_env/__init__.py rename to metagpt/environment/werewolf/__init__.py diff --git a/metagpt/environment/werewolf_env/werewolf_env.py b/metagpt/environment/werewolf/werewolf_env.py similarity index 93% rename from metagpt/environment/werewolf_env/werewolf_env.py rename to metagpt/environment/werewolf/werewolf_env.py index d174f322c..e22d9c46f 100644 --- a/metagpt/environment/werewolf_env/werewolf_env.py +++ b/metagpt/environment/werewolf/werewolf_env.py @@ -5,7 +5,7 @@ from pydantic import Field from metagpt.environment.base_env import Environment -from metagpt.environment.werewolf_env.werewolf_ext_env import WerewolfExtEnv +from metagpt.environment.werewolf.werewolf_ext_env import WerewolfExtEnv from metagpt.logs import logger from metagpt.schema import Message diff --git a/metagpt/environment/werewolf_env/werewolf_ext_env.py b/metagpt/environment/werewolf/werewolf_ext_env.py similarity index 100% rename from metagpt/environment/werewolf_env/werewolf_ext_env.py rename to metagpt/environment/werewolf/werewolf_ext_env.py diff --git a/tests/metagpt/environment/android_env/test_android_ext_env.py b/tests/metagpt/environment/android_env/test_android_ext_env.py index c9dfc718b..3fb77ab24 100644 --- a/tests/metagpt/environment/android_env/test_android_ext_env.py +++ b/tests/metagpt/environment/android_env/test_android_ext_env.py @@ -4,8 +4,8 @@ from pathlib import Path -from metagpt.environment.android_env.android_ext_env import AndroidExtEnv -from metagpt.environment.android_env.const import ADB_EXEC_FAIL +from metagpt.environment.android.android_ext_env import AndroidExtEnv +from metagpt.environment.android.const import ADB_EXEC_FAIL def mock_device_shape(self, adb_cmd: str) -> str: @@ -34,9 +34,7 @@ def mock_write_read_operation(self, adb_cmd: str) -> str: def test_android_ext_env(mocker): device_id = "emulator-5554" - mocker.patch( - "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_device_shape - ) + mocker.patch("metagpt.environment.android.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_device_shape) ext_env = AndroidExtEnv(device_id=device_id, screenshot_dir="/data2/", xml_dir="/data2/") assert ext_env.adb_prefix == f"adb -s {device_id} " @@ -46,25 +44,21 @@ def test_android_ext_env(mocker): assert ext_env.device_shape == (720, 1080) mocker.patch( - "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_device_shape_invalid + "metagpt.environment.android.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_device_shape_invalid ) assert ext_env.device_shape == (0, 0) - mocker.patch( - "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_list_devices - ) + mocker.patch("metagpt.environment.android.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_list_devices) assert ext_env.list_devices() == [device_id] - mocker.patch( - "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_get_screenshot - ) + mocker.patch("metagpt.environment.android.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_get_screenshot) assert ext_env.get_screenshot("screenshot_xxxx-xx-xx", "/data/") == Path("/data/screenshot_xxxx-xx-xx.png") - mocker.patch("metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_get_xml) + mocker.patch("metagpt.environment.android.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_get_xml) assert ext_env.get_xml("xml_xxxx-xx-xx", "/data/") == Path("/data/xml_xxxx-xx-xx.xml") mocker.patch( - "metagpt.environment.android_env.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_write_read_operation + "metagpt.environment.android.android_ext_env.AndroidExtEnv.execute_adb_with_cmd", mock_write_read_operation ) res = "OK" assert ext_env.system_back() == res diff --git a/tests/metagpt/environment/minecraft_env/test_minecraft_ext_env.py b/tests/metagpt/environment/minecraft_env/test_minecraft_ext_env.py index ed08b719f..0ebff22eb 100644 --- a/tests/metagpt/environment/minecraft_env/test_minecraft_ext_env.py +++ b/tests/metagpt/environment/minecraft_env/test_minecraft_ext_env.py @@ -3,8 +3,8 @@ # @Desc : the unittest of MinecraftExtEnv -from metagpt.environment.minecraft_env.const import MC_CKPT_DIR -from metagpt.environment.minecraft_env.minecraft_ext_env import MinecraftExtEnv +from metagpt.environment.minecraft.const import MC_CKPT_DIR +from metagpt.environment.minecraft.minecraft_ext_env import MinecraftExtEnv def test_minecraft_ext_env(): diff --git a/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py b/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py index 63e88cf32..81c811d67 100644 --- a/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py +++ b/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py @@ -4,18 +4,18 @@ from pathlib import Path -from metagpt.environment.stanford_town_env.env_space import ( +from metagpt.environment.stanford_town.env_space import ( EnvAction, EnvActionType, EnvObsParams, EnvObsType, ) -from metagpt.environment.stanford_town_env.stanford_town_ext_env import ( - StanfordTownExtEnv, -) +from metagpt.environment.stanford_town.stanford_town_ext_env import StanfordTownExtEnv maze_asset_path = ( - Path(__file__).absolute().parent.joinpath("..", "..", "..", "..", "examples/st_game/static_dirs/assets/the_ville") + Path(__file__) + .absolute() + .parent.joinpath("..", "..", "..", "..", "examples/stanford_town/static_dirs/assets/the_ville") ) diff --git a/tests/metagpt/environment/werewolf_env/test_werewolf_ext_env.py b/tests/metagpt/environment/werewolf_env/test_werewolf_ext_env.py index 0694c5c3d..433f59f2c 100644 --- a/tests/metagpt/environment/werewolf_env/test_werewolf_ext_env.py +++ b/tests/metagpt/environment/werewolf_env/test_werewolf_ext_env.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # @Desc : the unittest of WerewolfExtEnv -from metagpt.environment.werewolf_env.werewolf_ext_env import RoleState, WerewolfExtEnv +from metagpt.environment.werewolf.werewolf_ext_env import RoleState, WerewolfExtEnv from metagpt.roles.role import Role From 84a66bbc20448b6e2185992296083a80cb5fd704 Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 28 Mar 2024 12:03:34 +0800 Subject: [PATCH 075/127] simplify StanfordTown import --- examples/stanford_town/stanford_town.py | 2 +- examples/stanford_town/tests/actions/test_gen_action_details.py | 2 +- examples/stanford_town/tests/plan/test_conversation.py | 2 +- examples/stanford_town/tests/roles/test_st_role.py | 2 +- examples/stanford_town/tests/test_reflect.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/stanford_town/stanford_town.py b/examples/stanford_town/stanford_town.py index 7c39cbf04..17364f7c8 100644 --- a/examples/stanford_town/stanford_town.py +++ b/examples/stanford_town/stanford_town.py @@ -7,7 +7,7 @@ from typing import Any, Optional from examples.stanford_town.roles.st_role import STRole from examples.stanford_town.utils.const import MAZE_ASSET_PATH from metagpt.context import Context -from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv +from metagpt.environment import StanfordTownEnv from metagpt.logs import logger from metagpt.team import Team diff --git a/examples/stanford_town/tests/actions/test_gen_action_details.py b/examples/stanford_town/tests/actions/test_gen_action_details.py index bf6b75c04..af4d94c4d 100644 --- a/examples/stanford_town/tests/actions/test_gen_action_details.py +++ b/examples/stanford_town/tests/actions/test_gen_action_details.py @@ -13,8 +13,8 @@ from examples.stanford_town.actions.gen_action_details import ( ) from examples.stanford_town.roles.st_role import STRole from examples.stanford_town.utils.const import MAZE_ASSET_PATH +from metagpt.environment import StanfordTownEnv from metagpt.environment.api.env_api import EnvAPIAbstract -from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv @pytest.mark.asyncio diff --git a/examples/stanford_town/tests/plan/test_conversation.py b/examples/stanford_town/tests/plan/test_conversation.py index 86c2f0b94..ff8664538 100644 --- a/examples/stanford_town/tests/plan/test_conversation.py +++ b/examples/stanford_town/tests/plan/test_conversation.py @@ -11,7 +11,7 @@ from examples.stanford_town.roles.st_role import STRole from examples.stanford_town.utils.const import MAZE_ASSET_PATH, STORAGE_PATH from examples.stanford_town.utils.mg_ga_transform import get_reverie_meta from examples.stanford_town.utils.utils import copy_folder -from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv +from metagpt.environment import StanfordTownEnv async def init_two_roles(fork_sim_code: str = "base_the_ville_isabella_maria_klaus") -> Tuple["STRole"]: diff --git a/examples/stanford_town/tests/roles/test_st_role.py b/examples/stanford_town/tests/roles/test_st_role.py index b8feecb2a..b2e4c6969 100644 --- a/examples/stanford_town/tests/roles/test_st_role.py +++ b/examples/stanford_town/tests/roles/test_st_role.py @@ -7,7 +7,7 @@ import pytest from examples.stanford_town.memory.agent_memory import BasicMemory from examples.stanford_town.roles.st_role import STRole from examples.stanford_town.utils.const import MAZE_ASSET_PATH -from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv +from metagpt.environment import StanfordTownEnv @pytest.mark.asyncio diff --git a/examples/stanford_town/tests/test_reflect.py b/examples/stanford_town/tests/test_reflect.py index 48422d073..dbcbd974f 100644 --- a/examples/stanford_town/tests/test_reflect.py +++ b/examples/stanford_town/tests/test_reflect.py @@ -11,7 +11,7 @@ from examples.stanford_town.actions.run_reflect_action import ( ) from examples.stanford_town.roles.st_role import STRole from examples.stanford_town.utils.const import MAZE_ASSET_PATH -from metagpt.environment.stanford_town.stanford_town_env import StanfordTownEnv +from metagpt.environment import StanfordTownEnv @pytest.mark.asyncio From d53db1e65fb5bceb536cfc4b180b43f6505bdf44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 28 Mar 2024 15:05:39 +0800 Subject: [PATCH 076/127] feat: + MetaGPT version or branch --- .github/ISSUE_TEMPLATE/show_me_the_bug.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/show_me_the_bug.md b/.github/ISSUE_TEMPLATE/show_me_the_bug.md index 504a2bd12..0c33f0319 100644 --- a/.github/ISSUE_TEMPLATE/show_me_the_bug.md +++ b/.github/ISSUE_TEMPLATE/show_me_the_bug.md @@ -19,6 +19,7 @@ - LLM type and model name: - System version: - Python version: +- MetaGPT version or branch: From da359fdbb156ee442234e344d52c989dabd9374c Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Thu, 28 Mar 2024 15:47:38 +0800 Subject: [PATCH 077/127] lazy import colbert --- metagpt/rag/factories/ranker.py | 7 ++++++- setup.py | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/metagpt/rag/factories/ranker.py b/metagpt/rag/factories/ranker.py index 07cb1b929..476fe8c1a 100644 --- a/metagpt/rag/factories/ranker.py +++ b/metagpt/rag/factories/ranker.py @@ -3,7 +3,6 @@ from llama_index.core.llms import LLM from llama_index.core.postprocessor import LLMRerank from llama_index.core.postprocessor.types import BaseNodePostprocessor -from llama_index.postprocessor.colbert_rerank import ColbertRerank from metagpt.rag.factories.base import ConfigBasedFactory from metagpt.rag.rankers.object_ranker import ObjectSortPostprocessor @@ -38,6 +37,12 @@ class RankerFactory(ConfigBasedFactory): return LLMRerank(**config.model_dump()) def _create_colbert_ranker(self, config: ColbertRerankConfig, **kwargs) -> LLMRerank: + try: + from llama_index.postprocessor.colbert_rerank import ColbertRerank + except ImportError: + raise ImportError( + "`llama-index-postprocessor-colbert-rerank` package not found, please run `pip install llama-index-postprocessor-colbert-rerank`" + ) return ColbertRerank(**config.model_dump()) def _create_object_ranker(self, config: ObjectRankerConfig, **kwargs) -> LLMRerank: diff --git a/setup.py b/setup.py index 4fa5499da..3eab2b6a0 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,6 @@ extras_require = { "llama-index-vector-stores-faiss==0.1.1", "llama-index-vector-stores-elasticsearch==0.1.6", "llama-index-vector-stores-chroma==0.1.6", - "llama-index-postprocessor-colbert-rerank==0.1.1", ], } From 4ee273df4722435c9327a00c49b2fd421e7d2884 Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Thu, 28 Mar 2024 16:06:32 +0800 Subject: [PATCH 078/127] lazy import colbert --- tests/metagpt/rag/factories/test_ranker.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/metagpt/rag/factories/test_ranker.py b/tests/metagpt/rag/factories/test_ranker.py index 3f6b94b47..e40f7f8df 100644 --- a/tests/metagpt/rag/factories/test_ranker.py +++ b/tests/metagpt/rag/factories/test_ranker.py @@ -1,3 +1,5 @@ +import contextlib + import pytest from llama_index.core.llms import MockLLM from llama_index.core.postprocessor import LLMRerank @@ -41,12 +43,13 @@ class TestRankerFactory: assert isinstance(ranker, LLMRerank) def test_create_colbert_ranker(self, mocker, mock_llm): - mocker.patch("metagpt.rag.factories.ranker.ColbertRerank", return_value="colbert") + with contextlib.suppress(ImportError): + mocker.patch("llama_index.postprocessor.colbert_rerank.ColbertRerank", return_value="colbert") - mock_config = ColbertRerankConfig(llm=mock_llm) - ranker = self.ranker_factory._create_colbert_ranker(mock_config) + mock_config = ColbertRerankConfig(llm=mock_llm) + ranker = self.ranker_factory._create_colbert_ranker(mock_config) - assert ranker == "colbert" + assert ranker == "colbert" def test_create_object_ranker(self, mocker, mock_llm): mocker.patch("metagpt.rag.factories.ranker.ObjectSortPostprocessor", return_value="object") From 1c8995211bdd77f3a1fd938bf27a3907ee9e609d Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 28 Mar 2024 16:40:51 +0800 Subject: [PATCH 079/127] move stanford_town folder --- examples/stanford_town/.gitignore | 2 -- examples/stanford_town/run_st_game.py | 10 +++--- examples/stanford_town/storage/.gitignore | 1 + examples/stanford_town/tests/__init__.py | 0 examples/stanford_town/utils/const.py | 13 -------- .../actions => metagpt/ext}/__init__.py | 0 .../ext}/stanford_town/README.md | 0 .../ext}/stanford_town/README_CN.md | 0 metagpt/ext/stanford_town/__init__.py | 3 ++ .../ext/stanford_town/actions}/__init__.py | 0 .../actions/agent_chat_sum_rel.py | 2 +- .../stanford_town/actions/decide_to_talk.py | 2 +- .../stanford_town/actions/dummy_action.py | 0 .../actions/gen_action_details.py | 3 +- .../actions/gen_daily_schedule.py | 3 +- .../actions/gen_hourly_schedule.py | 0 .../actions/gen_iter_chat_utt.py | 4 +-- .../actions/inner_voice_action.py | 6 +++- .../actions/new_decomp_schedule.py | 2 +- .../actions/run_reflect_action.py | 2 +- .../ext}/stanford_town/actions/st_action.py | 2 +- .../stanford_town/actions/summarize_conv.py | 2 +- .../ext}/stanford_town/actions/task_decomp.py | 3 +- .../ext}/stanford_town/actions/wake_up.py | 3 +- .../ext}/stanford_town/memory/__init__.py | 0 .../ext}/stanford_town/memory/agent_memory.py | 0 .../ext}/stanford_town/memory/retrieve.py | 4 +-- .../ext}/stanford_town/memory/scratch.py | 0 .../stanford_town/memory/spatial_memory.py | 0 .../ext/stanford_town/plan}/__init__.py | 0 .../ext}/stanford_town/plan/converse.py | 6 ++-- .../ext}/stanford_town/plan/st_plan.py | 23 +++++++------ metagpt/ext/stanford_town/prompts/__init__.py | 3 ++ .../prompts/action_location_object_vMar11.txt | 0 .../prompts/action_location_sector_v1.txt | 0 .../prompts/action_object_v2.txt | 0 .../prompts/daily_planning_v6.txt | 0 .../prompts/decide_to_talk_v2.txt | 0 .../prompts/generate_event_triple_v1.txt | 0 .../prompts/generate_focal_pt_v1.txt | 0 .../prompts/generate_hourly_schedule_v2.txt | 0 .../prompts/generate_obj_event_v1.txt | 0 .../prompts/generate_pronunciatio_v1.txt | 0 .../prompts/insight_and_evidence_v1.txt | 0 .../prompts/iterative_convo_v1.txt | 0 .../prompts/memo_on_convo_v1.txt | 0 .../prompts/new_decomp_schedule_v1.txt | 0 .../prompts/planning_thought_on_convo_v1.txt | 0 .../prompts/poignancy_action_v1.txt | 0 .../prompts/poignancy_chat_v1.txt | 0 .../prompts/poignancy_event_v1.txt | 0 .../prompts/poignancy_thought_v1.txt | 0 .../summarize_chat_relationship_v2.txt | 0 .../prompts/summarize_conversation_v1.txt | 0 .../stanford_town/prompts/task_decomp_v3.txt | 0 .../stanford_town/prompts/wake_up_hour_v1.txt | 0 .../prompts/whisper_inner_thought_v1.txt | 0 .../ext}/stanford_town/reflect/__init__.py | 0 .../ext}/stanford_town/reflect/reflect.py | 6 ++-- .../ext/stanford_town/roles}/__init__.py | 0 .../ext}/stanford_town/roles/st_role.py | 32 ++++++++++--------- .../ext}/stanford_town/stanford_town.py | 4 +-- .../the_ville/agent_history_init_n25.csv | 0 .../the_ville/agent_history_init_n3.csv | 0 .../the_ville/matrix/maze/arena_maze.csv | 0 .../the_ville/matrix/maze/collision_maze.csv | 0 .../matrix/maze/game_object_maze.csv | 0 .../the_ville/matrix/maze/sector_maze.csv | 0 .../matrix/maze/spawning_location_maze.csv | 0 .../the_ville/matrix/maze_meta_info.json | 0 .../matrix/special_blocks/arena_blocks.csv | 0 .../special_blocks/game_object_blocks.csv | 0 .../matrix/special_blocks/sector_blocks.csv | 0 .../spawning_location_blocks.csv | 0 .../matrix/special_blocks/world_blocks.csv | 0 .../ext/stanford_town/utils}/__init__.py | 0 metagpt/ext/stanford_town/utils/const.py | 15 +++++++++ .../stanford_town/utils/mg_ga_transform.py | 2 +- .../ext}/stanford_town/utils/utils.py | 0 .../roles => tests/metagpt/ext}/__init__.py | 0 .../metagpt/ext/stanford_town}/__init__.py | 0 .../ext/stanford_town/actions}/__init__.py | 2 +- .../actions/test_gen_action_details.py | 10 +++--- .../actions/test_summarize_conv.py | 2 +- .../ext/stanford_town/memory/__init__.py | 3 ++ .../memory}/test_agent_memory.py | 6 ++-- .../memory}/test_basic_memory.py | 2 +- .../memory}/test_spatial_memory.py | 4 +-- .../ext/stanford_town/plan/__init__.py | 3 ++ .../stanford_town}/plan/test_conversation.py | 10 +++--- .../ext/stanford_town}/plan/test_st_plan.py | 4 +-- .../ext/stanford_town/roles/__init__.py | 3 ++ .../ext/stanford_town}/roles/test_st_role.py | 6 ++-- .../ext/stanford_town}/test_reflect.py | 8 ++--- 94 files changed, 119 insertions(+), 102 deletions(-) delete mode 100644 examples/stanford_town/.gitignore delete mode 100644 examples/stanford_town/tests/__init__.py delete mode 100644 examples/stanford_town/utils/const.py rename {examples/stanford_town/actions => metagpt/ext}/__init__.py (100%) rename {examples => metagpt/ext}/stanford_town/README.md (100%) rename {examples => metagpt/ext}/stanford_town/README_CN.md (100%) create mode 100644 metagpt/ext/stanford_town/__init__.py rename {examples/stanford_town/plan => metagpt/ext/stanford_town/actions}/__init__.py (100%) rename {examples => metagpt/ext}/stanford_town/actions/agent_chat_sum_rel.py (95%) rename {examples => metagpt/ext}/stanford_town/actions/decide_to_talk.py (98%) rename {examples => metagpt/ext}/stanford_town/actions/dummy_action.py (100%) rename {examples => metagpt/ext}/stanford_town/actions/gen_action_details.py (99%) rename {examples => metagpt/ext}/stanford_town/actions/gen_daily_schedule.py (97%) rename {examples => metagpt/ext}/stanford_town/actions/gen_hourly_schedule.py (100%) rename {examples => metagpt/ext}/stanford_town/actions/gen_iter_chat_utt.py (97%) rename {examples => metagpt/ext}/stanford_town/actions/inner_voice_action.py (89%) rename {examples => metagpt/ext}/stanford_town/actions/new_decomp_schedule.py (98%) rename {examples => metagpt/ext}/stanford_town/actions/run_reflect_action.py (99%) rename {examples => metagpt/ext}/stanford_town/actions/st_action.py (98%) rename {examples => metagpt/ext}/stanford_town/actions/summarize_conv.py (96%) rename {examples => metagpt/ext}/stanford_town/actions/task_decomp.py (99%) rename {examples => metagpt/ext}/stanford_town/actions/wake_up.py (95%) rename {examples => metagpt/ext}/stanford_town/memory/__init__.py (100%) rename {examples => metagpt/ext}/stanford_town/memory/agent_memory.py (100%) rename {examples => metagpt/ext}/stanford_town/memory/retrieve.py (97%) rename {examples => metagpt/ext}/stanford_town/memory/scratch.py (100%) rename {examples => metagpt/ext}/stanford_town/memory/spatial_memory.py (100%) rename {examples/stanford_town/roles => metagpt/ext/stanford_town/plan}/__init__.py (100%) rename {examples => metagpt/ext}/stanford_town/plan/converse.py (94%) rename {examples => metagpt/ext}/stanford_town/plan/st_plan.py (97%) create mode 100644 metagpt/ext/stanford_town/prompts/__init__.py rename {examples => metagpt/ext}/stanford_town/prompts/action_location_object_vMar11.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/action_location_sector_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/action_object_v2.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/daily_planning_v6.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/decide_to_talk_v2.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/generate_event_triple_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/generate_focal_pt_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/generate_hourly_schedule_v2.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/generate_obj_event_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/generate_pronunciatio_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/insight_and_evidence_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/iterative_convo_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/memo_on_convo_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/new_decomp_schedule_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/planning_thought_on_convo_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/poignancy_action_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/poignancy_chat_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/poignancy_event_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/poignancy_thought_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/summarize_chat_relationship_v2.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/summarize_conversation_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/task_decomp_v3.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/wake_up_hour_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/prompts/whisper_inner_thought_v1.txt (100%) rename {examples => metagpt/ext}/stanford_town/reflect/__init__.py (100%) rename {examples => metagpt/ext}/stanford_town/reflect/reflect.py (97%) rename {examples/stanford_town/tests/actions => metagpt/ext/stanford_town/roles}/__init__.py (100%) rename {examples => metagpt/ext}/stanford_town/roles/st_role.py (96%) rename {examples => metagpt/ext}/stanford_town/stanford_town.py (91%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/agent_history_init_n25.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/agent_history_init_n3.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/maze/arena_maze.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/maze/collision_maze.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/maze/game_object_maze.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/maze/sector_maze.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/maze/spawning_location_maze.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/maze_meta_info.json (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/arena_blocks.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/game_object_blocks.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/sector_blocks.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/spawning_location_blocks.csv (100%) rename {examples => metagpt/ext}/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/world_blocks.csv (100%) rename {examples/stanford_town/tests/plan => metagpt/ext/stanford_town/utils}/__init__.py (100%) create mode 100644 metagpt/ext/stanford_town/utils/const.py rename {examples => metagpt/ext}/stanford_town/utils/mg_ga_transform.py (96%) rename {examples => metagpt/ext}/stanford_town/utils/utils.py (100%) rename {examples/stanford_town/tests/roles => tests/metagpt/ext}/__init__.py (100%) rename {examples/stanford_town/utils => tests/metagpt/ext/stanford_town}/__init__.py (100%) rename {examples/stanford_town/prompts => tests/metagpt/ext/stanford_town/actions}/__init__.py (61%) rename {examples/stanford_town/tests => tests/metagpt/ext/stanford_town}/actions/test_gen_action_details.py (92%) rename {examples/stanford_town/tests => tests/metagpt/ext/stanford_town}/actions/test_summarize_conv.py (82%) create mode 100644 tests/metagpt/ext/stanford_town/memory/__init__.py rename {examples/stanford_town/tests => tests/metagpt/ext/stanford_town/memory}/test_agent_memory.py (94%) rename {examples/stanford_town/tests => tests/metagpt/ext/stanford_town/memory}/test_basic_memory.py (95%) rename {examples/stanford_town/tests => tests/metagpt/ext/stanford_town/memory}/test_spatial_memory.py (77%) create mode 100644 tests/metagpt/ext/stanford_town/plan/__init__.py rename {examples/stanford_town/tests => tests/metagpt/ext/stanford_town}/plan/test_conversation.py (83%) rename {examples/stanford_town/tests => tests/metagpt/ext/stanford_town}/plan/test_st_plan.py (90%) create mode 100644 tests/metagpt/ext/stanford_town/roles/__init__.py rename {examples/stanford_town/tests => tests/metagpt/ext/stanford_town}/roles/test_st_role.py (75%) rename {examples/stanford_town/tests => tests/metagpt/ext/stanford_town}/test_reflect.py (94%) diff --git a/examples/stanford_town/.gitignore b/examples/stanford_town/.gitignore deleted file mode 100644 index fe77153f9..000000000 --- a/examples/stanford_town/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -storage/test* -storage/unittest* \ No newline at end of file diff --git a/examples/stanford_town/run_st_game.py b/examples/stanford_town/run_st_game.py index 4c3b3bb0e..732ae5efc 100644 --- a/examples/stanford_town/run_st_game.py +++ b/examples/stanford_town/run_st_game.py @@ -7,15 +7,15 @@ from typing import Optional import fire -from examples.stanford_town.roles.st_role import STRole -from examples.stanford_town.stanford_town import StanfordTown -from examples.stanford_town.utils.const import STORAGE_PATH -from examples.stanford_town.utils.mg_ga_transform import ( +from metagpt.ext.stanford_town.roles.st_role import STRole +from metagpt.ext.stanford_town.stanford_town import StanfordTown +from metagpt.ext.stanford_town.utils.const import STORAGE_PATH +from metagpt.ext.stanford_town.utils.mg_ga_transform import ( get_reverie_meta, write_curr_sim_code, write_curr_step, ) -from examples.stanford_town.utils.utils import copy_folder +from metagpt.ext.stanford_town.utils.utils import copy_folder from metagpt.logs import logger diff --git a/examples/stanford_town/storage/.gitignore b/examples/stanford_town/storage/.gitignore index 72b125e04..962820861 100644 --- a/examples/stanford_town/storage/.gitignore +++ b/examples/stanford_town/storage/.gitignore @@ -1,3 +1,4 @@ # path to store simulation data test_* +unittest* July* \ No newline at end of file diff --git a/examples/stanford_town/tests/__init__.py b/examples/stanford_town/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/stanford_town/utils/const.py b/examples/stanford_town/utils/const.py deleted file mode 100644 index 38e80f103..000000000 --- a/examples/stanford_town/utils/const.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Desc : - -from pathlib import Path - -ROOT_PATH = Path(__file__).parent.parent -STORAGE_PATH = ROOT_PATH.joinpath("storage") -TEMP_STORAGE_PATH = ROOT_PATH.joinpath("temp_storage") -MAZE_ASSET_PATH = ROOT_PATH.joinpath("static_dirs/assets/the_ville") -PROMPTS_DIR = ROOT_PATH.joinpath("prompts") - -collision_block_id = "32125" diff --git a/examples/stanford_town/actions/__init__.py b/metagpt/ext/__init__.py similarity index 100% rename from examples/stanford_town/actions/__init__.py rename to metagpt/ext/__init__.py diff --git a/examples/stanford_town/README.md b/metagpt/ext/stanford_town/README.md similarity index 100% rename from examples/stanford_town/README.md rename to metagpt/ext/stanford_town/README.md diff --git a/examples/stanford_town/README_CN.md b/metagpt/ext/stanford_town/README_CN.md similarity index 100% rename from examples/stanford_town/README_CN.md rename to metagpt/ext/stanford_town/README_CN.md diff --git a/metagpt/ext/stanford_town/__init__.py b/metagpt/ext/stanford_town/__init__.py new file mode 100644 index 000000000..56ea35c9f --- /dev/null +++ b/metagpt/ext/stanford_town/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : stanford town implement diff --git a/examples/stanford_town/plan/__init__.py b/metagpt/ext/stanford_town/actions/__init__.py similarity index 100% rename from examples/stanford_town/plan/__init__.py rename to metagpt/ext/stanford_town/actions/__init__.py diff --git a/examples/stanford_town/actions/agent_chat_sum_rel.py b/metagpt/ext/stanford_town/actions/agent_chat_sum_rel.py similarity index 95% rename from examples/stanford_town/actions/agent_chat_sum_rel.py rename to metagpt/ext/stanford_town/actions/agent_chat_sum_rel.py index 3e564a60c..98d370bb0 100644 --- a/examples/stanford_town/actions/agent_chat_sum_rel.py +++ b/metagpt/ext/stanford_town/actions/agent_chat_sum_rel.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # @Desc : summarize relationship in a agent chat -from examples.stanford_town.actions.st_action import STAction +from metagpt.ext.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/stanford_town/actions/decide_to_talk.py b/metagpt/ext/stanford_town/actions/decide_to_talk.py similarity index 98% rename from examples/stanford_town/actions/decide_to_talk.py rename to metagpt/ext/stanford_town/actions/decide_to_talk.py index 414ee7e36..a393f31af 100644 --- a/examples/stanford_town/actions/decide_to_talk.py +++ b/metagpt/ext/stanford_town/actions/decide_to_talk.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # @Desc : device to talk to another role, return yes or no -from examples.stanford_town.actions.st_action import STAction +from metagpt.ext.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/stanford_town/actions/dummy_action.py b/metagpt/ext/stanford_town/actions/dummy_action.py similarity index 100% rename from examples/stanford_town/actions/dummy_action.py rename to metagpt/ext/stanford_town/actions/dummy_action.py diff --git a/examples/stanford_town/actions/gen_action_details.py b/metagpt/ext/stanford_town/actions/gen_action_details.py similarity index 99% rename from examples/stanford_town/actions/gen_action_details.py rename to metagpt/ext/stanford_town/actions/gen_action_details.py index 0eb7cb701..3820ddc5b 100644 --- a/examples/stanford_town/actions/gen_action_details.py +++ b/metagpt/ext/stanford_town/actions/gen_action_details.py @@ -5,10 +5,9 @@ import random from metagpt.environment.stanford_town.env_space import EnvObsParams, EnvObsType +from metagpt.ext.stanford_town.actions.st_action import STAction from metagpt.logs import logger -from .st_action import STAction - class GenActionSector(STAction): name: str = "GenActionSector" diff --git a/examples/stanford_town/actions/gen_daily_schedule.py b/metagpt/ext/stanford_town/actions/gen_daily_schedule.py similarity index 97% rename from examples/stanford_town/actions/gen_daily_schedule.py rename to metagpt/ext/stanford_town/actions/gen_daily_schedule.py index 6deb6e6eb..5dffa8995 100644 --- a/examples/stanford_town/actions/gen_daily_schedule.py +++ b/metagpt/ext/stanford_town/actions/gen_daily_schedule.py @@ -3,10 +3,9 @@ # @Desc : gen_daily_schedule +from metagpt.ext.stanford_town.actions.st_action import STAction from metagpt.logs import logger -from .st_action import STAction - class GenDailySchedule(STAction): name: str = "GenDailySchedule" diff --git a/examples/stanford_town/actions/gen_hourly_schedule.py b/metagpt/ext/stanford_town/actions/gen_hourly_schedule.py similarity index 100% rename from examples/stanford_town/actions/gen_hourly_schedule.py rename to metagpt/ext/stanford_town/actions/gen_hourly_schedule.py diff --git a/examples/stanford_town/actions/gen_iter_chat_utt.py b/metagpt/ext/stanford_town/actions/gen_iter_chat_utt.py similarity index 97% rename from examples/stanford_town/actions/gen_iter_chat_utt.py rename to metagpt/ext/stanford_town/actions/gen_iter_chat_utt.py index 82104b6ca..41460afba 100644 --- a/examples/stanford_town/actions/gen_iter_chat_utt.py +++ b/metagpt/ext/stanford_town/actions/gen_iter_chat_utt.py @@ -2,9 +2,9 @@ # -*- coding: utf-8 -*- # @Desc : generate_iterative_chat_utt -from examples.stanford_town.actions.st_action import STAction -from examples.stanford_town.utils.utils import extract_first_json_dict from metagpt.environment.stanford_town.env_space import EnvObsParams, EnvObsType +from metagpt.ext.stanford_town.actions.st_action import STAction +from metagpt.ext.stanford_town.utils.utils import extract_first_json_dict from metagpt.logs import logger diff --git a/examples/stanford_town/actions/inner_voice_action.py b/metagpt/ext/stanford_town/actions/inner_voice_action.py similarity index 89% rename from examples/stanford_town/actions/inner_voice_action.py rename to metagpt/ext/stanford_town/actions/inner_voice_action.py index 121f1dcee..83cfa037b 100644 --- a/examples/stanford_town/actions/inner_voice_action.py +++ b/metagpt/ext/stanford_town/actions/inner_voice_action.py @@ -1,4 +1,8 @@ -from examples.stanford_town.actions.st_action import STAction +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : + +from metagpt.ext.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/stanford_town/actions/new_decomp_schedule.py b/metagpt/ext/stanford_town/actions/new_decomp_schedule.py similarity index 98% rename from examples/stanford_town/actions/new_decomp_schedule.py rename to metagpt/ext/stanford_town/actions/new_decomp_schedule.py index c7078be9c..759ec170f 100644 --- a/examples/stanford_town/actions/new_decomp_schedule.py +++ b/metagpt/ext/stanford_town/actions/new_decomp_schedule.py @@ -4,7 +4,7 @@ import datetime -from examples.stanford_town.actions.st_action import STAction +from metagpt.ext.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/stanford_town/actions/run_reflect_action.py b/metagpt/ext/stanford_town/actions/run_reflect_action.py similarity index 99% rename from examples/stanford_town/actions/run_reflect_action.py rename to metagpt/ext/stanford_town/actions/run_reflect_action.py index 055c03db4..895f6828f 100644 --- a/examples/stanford_town/actions/run_reflect_action.py +++ b/metagpt/ext/stanford_town/actions/run_reflect_action.py @@ -4,7 +4,7 @@ import re -from examples.stanford_town.actions.st_action import STAction +from metagpt.ext.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/stanford_town/actions/st_action.py b/metagpt/ext/stanford_town/actions/st_action.py similarity index 98% rename from examples/stanford_town/actions/st_action.py rename to metagpt/ext/stanford_town/actions/st_action.py index 72ef851e0..321676374 100644 --- a/examples/stanford_town/actions/st_action.py +++ b/metagpt/ext/stanford_town/actions/st_action.py @@ -7,9 +7,9 @@ from abc import abstractmethod from pathlib import Path from typing import Any, Optional, Union -from examples.stanford_town.utils.const import PROMPTS_DIR from metagpt.actions.action import Action from metagpt.config2 import config +from metagpt.ext.stanford_town.utils.const import PROMPTS_DIR from metagpt.logs import logger diff --git a/examples/stanford_town/actions/summarize_conv.py b/metagpt/ext/stanford_town/actions/summarize_conv.py similarity index 96% rename from examples/stanford_town/actions/summarize_conv.py rename to metagpt/ext/stanford_town/actions/summarize_conv.py index f01fb36ad..5be5fcaa4 100644 --- a/examples/stanford_town/actions/summarize_conv.py +++ b/metagpt/ext/stanford_town/actions/summarize_conv.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # @Desc : summarize the content of agents' conversation -from examples.stanford_town.actions.st_action import STAction +from metagpt.ext.stanford_town.actions.st_action import STAction from metagpt.logs import logger diff --git a/examples/stanford_town/actions/task_decomp.py b/metagpt/ext/stanford_town/actions/task_decomp.py similarity index 99% rename from examples/stanford_town/actions/task_decomp.py rename to metagpt/ext/stanford_town/actions/task_decomp.py index d9d5ec9fa..a4e22c24d 100644 --- a/examples/stanford_town/actions/task_decomp.py +++ b/metagpt/ext/stanford_town/actions/task_decomp.py @@ -4,10 +4,9 @@ import datetime +from metagpt.ext.stanford_town.actions.st_action import STAction from metagpt.logs import logger -from ..actions.st_action import STAction - class TaskDecomp(STAction): name: str = "TaskDecomp" diff --git a/examples/stanford_town/actions/wake_up.py b/metagpt/ext/stanford_town/actions/wake_up.py similarity index 95% rename from examples/stanford_town/actions/wake_up.py rename to metagpt/ext/stanford_town/actions/wake_up.py index d39115854..ea44cd3a4 100644 --- a/examples/stanford_town/actions/wake_up.py +++ b/metagpt/ext/stanford_town/actions/wake_up.py @@ -3,10 +3,9 @@ # @Desc : wake_up +from metagpt.ext.stanford_town.actions.st_action import STAction from metagpt.logs import logger -from ..actions.st_action import STAction - class WakeUp(STAction): name: str = "WakeUp" diff --git a/examples/stanford_town/memory/__init__.py b/metagpt/ext/stanford_town/memory/__init__.py similarity index 100% rename from examples/stanford_town/memory/__init__.py rename to metagpt/ext/stanford_town/memory/__init__.py diff --git a/examples/stanford_town/memory/agent_memory.py b/metagpt/ext/stanford_town/memory/agent_memory.py similarity index 100% rename from examples/stanford_town/memory/agent_memory.py rename to metagpt/ext/stanford_town/memory/agent_memory.py diff --git a/examples/stanford_town/memory/retrieve.py b/metagpt/ext/stanford_town/memory/retrieve.py similarity index 97% rename from examples/stanford_town/memory/retrieve.py rename to metagpt/ext/stanford_town/memory/retrieve.py index db3a87bea..c4b32f965 100644 --- a/examples/stanford_town/memory/retrieve.py +++ b/metagpt/ext/stanford_town/memory/retrieve.py @@ -7,8 +7,8 @@ import datetime from numpy import dot from numpy.linalg import norm -from examples.stanford_town.memory.agent_memory import BasicMemory -from examples.stanford_town.utils.utils import get_embedding +from metagpt.ext.stanford_town.memory.agent_memory import BasicMemory +from metagpt.ext.stanford_town.utils.utils import get_embedding def agent_retrieve( diff --git a/examples/stanford_town/memory/scratch.py b/metagpt/ext/stanford_town/memory/scratch.py similarity index 100% rename from examples/stanford_town/memory/scratch.py rename to metagpt/ext/stanford_town/memory/scratch.py diff --git a/examples/stanford_town/memory/spatial_memory.py b/metagpt/ext/stanford_town/memory/spatial_memory.py similarity index 100% rename from examples/stanford_town/memory/spatial_memory.py rename to metagpt/ext/stanford_town/memory/spatial_memory.py diff --git a/examples/stanford_town/roles/__init__.py b/metagpt/ext/stanford_town/plan/__init__.py similarity index 100% rename from examples/stanford_town/roles/__init__.py rename to metagpt/ext/stanford_town/plan/__init__.py diff --git a/examples/stanford_town/plan/converse.py b/metagpt/ext/stanford_town/plan/converse.py similarity index 94% rename from examples/stanford_town/plan/converse.py rename to metagpt/ext/stanford_town/plan/converse.py index b0ff54b68..8eefbc9b4 100644 --- a/examples/stanford_town/plan/converse.py +++ b/metagpt/ext/stanford_town/plan/converse.py @@ -4,9 +4,9 @@ from typing import Tuple -from examples.stanford_town.actions.agent_chat_sum_rel import AgentChatSumRel -from examples.stanford_town.actions.gen_iter_chat_utt import GenIterChatUTT -from examples.stanford_town.memory.retrieve import new_agent_retrieve +from metagpt.ext.stanford_town.actions.agent_chat_sum_rel import AgentChatSumRel +from metagpt.ext.stanford_town.actions.gen_iter_chat_utt import GenIterChatUTT +from metagpt.ext.stanford_town.memory.retrieve import new_agent_retrieve from metagpt.logs import logger diff --git a/examples/stanford_town/plan/st_plan.py b/metagpt/ext/stanford_town/plan/st_plan.py similarity index 97% rename from examples/stanford_town/plan/st_plan.py rename to metagpt/ext/stanford_town/plan/st_plan.py index 080d87de1..6b6323b7a 100644 --- a/examples/stanford_town/plan/st_plan.py +++ b/metagpt/ext/stanford_town/plan/st_plan.py @@ -7,21 +7,20 @@ import math import random from typing import Tuple, Union +from metagpt.ext.stanford_town.actions.decide_to_talk import DecideToTalk +from metagpt.ext.stanford_town.actions.gen_action_details import GenActionDetails +from metagpt.ext.stanford_town.actions.gen_daily_schedule import GenDailySchedule +from metagpt.ext.stanford_town.actions.gen_hourly_schedule import GenHourlySchedule +from metagpt.ext.stanford_town.actions.new_decomp_schedule import NewDecompSchedule +from metagpt.ext.stanford_town.actions.summarize_conv import SummarizeConv +from metagpt.ext.stanford_town.actions.task_decomp import TaskDecomp +from metagpt.ext.stanford_town.actions.wake_up import WakeUp +from metagpt.ext.stanford_town.memory.retrieve import new_agent_retrieve +from metagpt.ext.stanford_town.plan.converse import agent_conversation +from metagpt.ext.stanford_town.utils.utils import get_embedding from metagpt.llm import LLM from metagpt.logs import logger -from ..actions.decide_to_talk import DecideToTalk -from ..actions.gen_action_details import GenActionDetails -from ..actions.gen_daily_schedule import GenDailySchedule -from ..actions.gen_hourly_schedule import GenHourlySchedule -from ..actions.new_decomp_schedule import NewDecompSchedule -from ..actions.summarize_conv import SummarizeConv -from ..actions.task_decomp import TaskDecomp -from ..actions.wake_up import WakeUp -from ..memory.retrieve import new_agent_retrieve -from ..plan.converse import agent_conversation -from ..utils.utils import get_embedding - async def plan(role: "STRole", roles: dict["STRole"], new_day: bool, retrieved: dict) -> str: # PART 1: Generate the hourly schedule. diff --git a/metagpt/ext/stanford_town/prompts/__init__.py b/metagpt/ext/stanford_town/prompts/__init__.py new file mode 100644 index 000000000..2af51b8dc --- /dev/null +++ b/metagpt/ext/stanford_town/prompts/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : stanford town prompt templates diff --git a/examples/stanford_town/prompts/action_location_object_vMar11.txt b/metagpt/ext/stanford_town/prompts/action_location_object_vMar11.txt similarity index 100% rename from examples/stanford_town/prompts/action_location_object_vMar11.txt rename to metagpt/ext/stanford_town/prompts/action_location_object_vMar11.txt diff --git a/examples/stanford_town/prompts/action_location_sector_v1.txt b/metagpt/ext/stanford_town/prompts/action_location_sector_v1.txt similarity index 100% rename from examples/stanford_town/prompts/action_location_sector_v1.txt rename to metagpt/ext/stanford_town/prompts/action_location_sector_v1.txt diff --git a/examples/stanford_town/prompts/action_object_v2.txt b/metagpt/ext/stanford_town/prompts/action_object_v2.txt similarity index 100% rename from examples/stanford_town/prompts/action_object_v2.txt rename to metagpt/ext/stanford_town/prompts/action_object_v2.txt diff --git a/examples/stanford_town/prompts/daily_planning_v6.txt b/metagpt/ext/stanford_town/prompts/daily_planning_v6.txt similarity index 100% rename from examples/stanford_town/prompts/daily_planning_v6.txt rename to metagpt/ext/stanford_town/prompts/daily_planning_v6.txt diff --git a/examples/stanford_town/prompts/decide_to_talk_v2.txt b/metagpt/ext/stanford_town/prompts/decide_to_talk_v2.txt similarity index 100% rename from examples/stanford_town/prompts/decide_to_talk_v2.txt rename to metagpt/ext/stanford_town/prompts/decide_to_talk_v2.txt diff --git a/examples/stanford_town/prompts/generate_event_triple_v1.txt b/metagpt/ext/stanford_town/prompts/generate_event_triple_v1.txt similarity index 100% rename from examples/stanford_town/prompts/generate_event_triple_v1.txt rename to metagpt/ext/stanford_town/prompts/generate_event_triple_v1.txt diff --git a/examples/stanford_town/prompts/generate_focal_pt_v1.txt b/metagpt/ext/stanford_town/prompts/generate_focal_pt_v1.txt similarity index 100% rename from examples/stanford_town/prompts/generate_focal_pt_v1.txt rename to metagpt/ext/stanford_town/prompts/generate_focal_pt_v1.txt diff --git a/examples/stanford_town/prompts/generate_hourly_schedule_v2.txt b/metagpt/ext/stanford_town/prompts/generate_hourly_schedule_v2.txt similarity index 100% rename from examples/stanford_town/prompts/generate_hourly_schedule_v2.txt rename to metagpt/ext/stanford_town/prompts/generate_hourly_schedule_v2.txt diff --git a/examples/stanford_town/prompts/generate_obj_event_v1.txt b/metagpt/ext/stanford_town/prompts/generate_obj_event_v1.txt similarity index 100% rename from examples/stanford_town/prompts/generate_obj_event_v1.txt rename to metagpt/ext/stanford_town/prompts/generate_obj_event_v1.txt diff --git a/examples/stanford_town/prompts/generate_pronunciatio_v1.txt b/metagpt/ext/stanford_town/prompts/generate_pronunciatio_v1.txt similarity index 100% rename from examples/stanford_town/prompts/generate_pronunciatio_v1.txt rename to metagpt/ext/stanford_town/prompts/generate_pronunciatio_v1.txt diff --git a/examples/stanford_town/prompts/insight_and_evidence_v1.txt b/metagpt/ext/stanford_town/prompts/insight_and_evidence_v1.txt similarity index 100% rename from examples/stanford_town/prompts/insight_and_evidence_v1.txt rename to metagpt/ext/stanford_town/prompts/insight_and_evidence_v1.txt diff --git a/examples/stanford_town/prompts/iterative_convo_v1.txt b/metagpt/ext/stanford_town/prompts/iterative_convo_v1.txt similarity index 100% rename from examples/stanford_town/prompts/iterative_convo_v1.txt rename to metagpt/ext/stanford_town/prompts/iterative_convo_v1.txt diff --git a/examples/stanford_town/prompts/memo_on_convo_v1.txt b/metagpt/ext/stanford_town/prompts/memo_on_convo_v1.txt similarity index 100% rename from examples/stanford_town/prompts/memo_on_convo_v1.txt rename to metagpt/ext/stanford_town/prompts/memo_on_convo_v1.txt diff --git a/examples/stanford_town/prompts/new_decomp_schedule_v1.txt b/metagpt/ext/stanford_town/prompts/new_decomp_schedule_v1.txt similarity index 100% rename from examples/stanford_town/prompts/new_decomp_schedule_v1.txt rename to metagpt/ext/stanford_town/prompts/new_decomp_schedule_v1.txt diff --git a/examples/stanford_town/prompts/planning_thought_on_convo_v1.txt b/metagpt/ext/stanford_town/prompts/planning_thought_on_convo_v1.txt similarity index 100% rename from examples/stanford_town/prompts/planning_thought_on_convo_v1.txt rename to metagpt/ext/stanford_town/prompts/planning_thought_on_convo_v1.txt diff --git a/examples/stanford_town/prompts/poignancy_action_v1.txt b/metagpt/ext/stanford_town/prompts/poignancy_action_v1.txt similarity index 100% rename from examples/stanford_town/prompts/poignancy_action_v1.txt rename to metagpt/ext/stanford_town/prompts/poignancy_action_v1.txt diff --git a/examples/stanford_town/prompts/poignancy_chat_v1.txt b/metagpt/ext/stanford_town/prompts/poignancy_chat_v1.txt similarity index 100% rename from examples/stanford_town/prompts/poignancy_chat_v1.txt rename to metagpt/ext/stanford_town/prompts/poignancy_chat_v1.txt diff --git a/examples/stanford_town/prompts/poignancy_event_v1.txt b/metagpt/ext/stanford_town/prompts/poignancy_event_v1.txt similarity index 100% rename from examples/stanford_town/prompts/poignancy_event_v1.txt rename to metagpt/ext/stanford_town/prompts/poignancy_event_v1.txt diff --git a/examples/stanford_town/prompts/poignancy_thought_v1.txt b/metagpt/ext/stanford_town/prompts/poignancy_thought_v1.txt similarity index 100% rename from examples/stanford_town/prompts/poignancy_thought_v1.txt rename to metagpt/ext/stanford_town/prompts/poignancy_thought_v1.txt diff --git a/examples/stanford_town/prompts/summarize_chat_relationship_v2.txt b/metagpt/ext/stanford_town/prompts/summarize_chat_relationship_v2.txt similarity index 100% rename from examples/stanford_town/prompts/summarize_chat_relationship_v2.txt rename to metagpt/ext/stanford_town/prompts/summarize_chat_relationship_v2.txt diff --git a/examples/stanford_town/prompts/summarize_conversation_v1.txt b/metagpt/ext/stanford_town/prompts/summarize_conversation_v1.txt similarity index 100% rename from examples/stanford_town/prompts/summarize_conversation_v1.txt rename to metagpt/ext/stanford_town/prompts/summarize_conversation_v1.txt diff --git a/examples/stanford_town/prompts/task_decomp_v3.txt b/metagpt/ext/stanford_town/prompts/task_decomp_v3.txt similarity index 100% rename from examples/stanford_town/prompts/task_decomp_v3.txt rename to metagpt/ext/stanford_town/prompts/task_decomp_v3.txt diff --git a/examples/stanford_town/prompts/wake_up_hour_v1.txt b/metagpt/ext/stanford_town/prompts/wake_up_hour_v1.txt similarity index 100% rename from examples/stanford_town/prompts/wake_up_hour_v1.txt rename to metagpt/ext/stanford_town/prompts/wake_up_hour_v1.txt diff --git a/examples/stanford_town/prompts/whisper_inner_thought_v1.txt b/metagpt/ext/stanford_town/prompts/whisper_inner_thought_v1.txt similarity index 100% rename from examples/stanford_town/prompts/whisper_inner_thought_v1.txt rename to metagpt/ext/stanford_town/prompts/whisper_inner_thought_v1.txt diff --git a/examples/stanford_town/reflect/__init__.py b/metagpt/ext/stanford_town/reflect/__init__.py similarity index 100% rename from examples/stanford_town/reflect/__init__.py rename to metagpt/ext/stanford_town/reflect/__init__.py diff --git a/examples/stanford_town/reflect/reflect.py b/metagpt/ext/stanford_town/reflect/reflect.py similarity index 97% rename from examples/stanford_town/reflect/reflect.py rename to metagpt/ext/stanford_town/reflect/reflect.py index beec53dc2..954cbc1ac 100644 --- a/examples/stanford_town/reflect/reflect.py +++ b/metagpt/ext/stanford_town/reflect/reflect.py @@ -5,7 +5,7 @@ import datetime import time -from examples.stanford_town.actions.run_reflect_action import ( +from metagpt.ext.stanford_town.actions.run_reflect_action import ( AgentChatPoignancy, AgentEventPoignancy, AgentEventTriple, @@ -14,8 +14,8 @@ from examples.stanford_town.actions.run_reflect_action import ( AgentMemoryOnConvo, AgentPlanThoughtOnConvo, ) -from examples.stanford_town.memory.retrieve import new_agent_retrieve -from examples.stanford_town.utils.utils import get_embedding +from metagpt.ext.stanford_town.memory.retrieve import new_agent_retrieve +from metagpt.ext.stanford_town.utils.utils import get_embedding from metagpt.logs import logger diff --git a/examples/stanford_town/tests/actions/__init__.py b/metagpt/ext/stanford_town/roles/__init__.py similarity index 100% rename from examples/stanford_town/tests/actions/__init__.py rename to metagpt/ext/stanford_town/roles/__init__.py diff --git a/examples/stanford_town/roles/st_role.py b/metagpt/ext/stanford_town/roles/st_role.py similarity index 96% rename from examples/stanford_town/roles/st_role.py rename to metagpt/ext/stanford_town/roles/st_role.py index 414fd40b8..79f58b07d 100644 --- a/examples/stanford_town/roles/st_role.py +++ b/metagpt/ext/stanford_town/roles/st_role.py @@ -20,21 +20,6 @@ from typing import TYPE_CHECKING, Optional from pydantic import ConfigDict, Field, field_validator, model_validator -from examples.stanford_town.actions.dummy_action import DummyAction, DummyMessage -from examples.stanford_town.actions.inner_voice_action import AgentWhisperThoughtAction -from examples.stanford_town.actions.run_reflect_action import AgentEventTriple -from examples.stanford_town.memory.agent_memory import AgentMemory, BasicMemory -from examples.stanford_town.memory.scratch import Scratch -from examples.stanford_town.memory.spatial_memory import MemoryTree -from examples.stanford_town.plan.st_plan import plan -from examples.stanford_town.reflect.reflect import generate_poig_score, role_reflect -from examples.stanford_town.utils.const import STORAGE_PATH, collision_block_id -from examples.stanford_town.utils.mg_ga_transform import ( - get_role_environment, - save_environment, - save_movement, -) -from examples.stanford_town.utils.utils import get_embedding, path_finder from metagpt.actions.add_requirement import UserRequirement from metagpt.environment.stanford_town.env_space import ( EnvAction, @@ -42,6 +27,23 @@ from metagpt.environment.stanford_town.env_space import ( EnvObsParams, EnvObsType, ) +from metagpt.ext.stanford_town.actions.dummy_action import DummyAction, DummyMessage +from metagpt.ext.stanford_town.actions.inner_voice_action import ( + AgentWhisperThoughtAction, +) +from metagpt.ext.stanford_town.actions.run_reflect_action import AgentEventTriple +from metagpt.ext.stanford_town.memory.agent_memory import AgentMemory, BasicMemory +from metagpt.ext.stanford_town.memory.scratch import Scratch +from metagpt.ext.stanford_town.memory.spatial_memory import MemoryTree +from metagpt.ext.stanford_town.plan.st_plan import plan +from metagpt.ext.stanford_town.reflect.reflect import generate_poig_score, role_reflect +from metagpt.ext.stanford_town.utils.const import STORAGE_PATH, collision_block_id +from metagpt.ext.stanford_town.utils.mg_ga_transform import ( + get_role_environment, + save_environment, + save_movement, +) +from metagpt.ext.stanford_town.utils.utils import get_embedding, path_finder from metagpt.logs import logger from metagpt.roles.role import Role, RoleContext from metagpt.schema import Message diff --git a/examples/stanford_town/stanford_town.py b/metagpt/ext/stanford_town/stanford_town.py similarity index 91% rename from examples/stanford_town/stanford_town.py rename to metagpt/ext/stanford_town/stanford_town.py index 17364f7c8..96f6a4b4e 100644 --- a/examples/stanford_town/stanford_town.py +++ b/metagpt/ext/stanford_town/stanford_town.py @@ -4,10 +4,10 @@ from typing import Any, Optional -from examples.stanford_town.roles.st_role import STRole -from examples.stanford_town.utils.const import MAZE_ASSET_PATH from metagpt.context import Context from metagpt.environment import StanfordTownEnv +from metagpt.ext.stanford_town.roles.st_role import STRole +from metagpt.ext.stanford_town.utils.const import MAZE_ASSET_PATH from metagpt.logs import logger from metagpt.team import Team diff --git a/examples/stanford_town/static_dirs/assets/the_ville/agent_history_init_n25.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/agent_history_init_n25.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/agent_history_init_n25.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/agent_history_init_n25.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/agent_history_init_n3.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/agent_history_init_n3.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/agent_history_init_n3.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/agent_history_init_n3.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/arena_maze.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze/arena_maze.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/arena_maze.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze/arena_maze.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/collision_maze.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze/collision_maze.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/collision_maze.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze/collision_maze.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/game_object_maze.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze/game_object_maze.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/game_object_maze.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze/game_object_maze.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/sector_maze.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze/sector_maze.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/sector_maze.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze/sector_maze.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/spawning_location_maze.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze/spawning_location_maze.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/maze/spawning_location_maze.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze/spawning_location_maze.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/maze_meta_info.json b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze_meta_info.json similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/maze_meta_info.json rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/maze_meta_info.json diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/arena_blocks.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/arena_blocks.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/arena_blocks.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/arena_blocks.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/game_object_blocks.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/game_object_blocks.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/game_object_blocks.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/game_object_blocks.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/sector_blocks.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/sector_blocks.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/sector_blocks.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/sector_blocks.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/spawning_location_blocks.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/spawning_location_blocks.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/spawning_location_blocks.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/spawning_location_blocks.csv diff --git a/examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/world_blocks.csv b/metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/world_blocks.csv similarity index 100% rename from examples/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/world_blocks.csv rename to metagpt/ext/stanford_town/static_dirs/assets/the_ville/matrix/special_blocks/world_blocks.csv diff --git a/examples/stanford_town/tests/plan/__init__.py b/metagpt/ext/stanford_town/utils/__init__.py similarity index 100% rename from examples/stanford_town/tests/plan/__init__.py rename to metagpt/ext/stanford_town/utils/__init__.py diff --git a/metagpt/ext/stanford_town/utils/const.py b/metagpt/ext/stanford_town/utils/const.py new file mode 100644 index 000000000..7a5c6fd24 --- /dev/null +++ b/metagpt/ext/stanford_town/utils/const.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : + +from pathlib import Path + +from metagpt.const import EXAMPLE_PATH + +ST_ROOT_PATH = Path(__file__).parent.parent +STORAGE_PATH = EXAMPLE_PATH.joinpath("stanford_town/storage") +TEMP_STORAGE_PATH = EXAMPLE_PATH.joinpath("stanford_town/temp_storage") +MAZE_ASSET_PATH = ST_ROOT_PATH.joinpath("static_dirs/assets/the_ville") +PROMPTS_DIR = ST_ROOT_PATH.joinpath("prompts") + +collision_block_id = "32125" diff --git a/examples/stanford_town/utils/mg_ga_transform.py b/metagpt/ext/stanford_town/utils/mg_ga_transform.py similarity index 96% rename from examples/stanford_town/utils/mg_ga_transform.py rename to metagpt/ext/stanford_town/utils/mg_ga_transform.py index 521199a05..e0de76a6b 100644 --- a/examples/stanford_town/utils/mg_ga_transform.py +++ b/metagpt/ext/stanford_town/utils/mg_ga_transform.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Optional -from examples.stanford_town.utils.const import STORAGE_PATH, TEMP_STORAGE_PATH +from metagpt.ext.stanford_town.utils.const import STORAGE_PATH, TEMP_STORAGE_PATH from metagpt.logs import logger from metagpt.utils.common import read_json_file, write_json_file diff --git a/examples/stanford_town/utils/utils.py b/metagpt/ext/stanford_town/utils/utils.py similarity index 100% rename from examples/stanford_town/utils/utils.py rename to metagpt/ext/stanford_town/utils/utils.py diff --git a/examples/stanford_town/tests/roles/__init__.py b/tests/metagpt/ext/__init__.py similarity index 100% rename from examples/stanford_town/tests/roles/__init__.py rename to tests/metagpt/ext/__init__.py diff --git a/examples/stanford_town/utils/__init__.py b/tests/metagpt/ext/stanford_town/__init__.py similarity index 100% rename from examples/stanford_town/utils/__init__.py rename to tests/metagpt/ext/stanford_town/__init__.py diff --git a/examples/stanford_town/prompts/__init__.py b/tests/metagpt/ext/stanford_town/actions/__init__.py similarity index 61% rename from examples/stanford_town/prompts/__init__.py rename to tests/metagpt/ext/stanford_town/actions/__init__.py index cb4131c76..2bcf8efd0 100644 --- a/examples/stanford_town/prompts/__init__.py +++ b/tests/metagpt/ext/stanford_town/actions/__init__.py @@ -1,3 +1,3 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : prompt templates +# @Desc : diff --git a/examples/stanford_town/tests/actions/test_gen_action_details.py b/tests/metagpt/ext/stanford_town/actions/test_gen_action_details.py similarity index 92% rename from examples/stanford_town/tests/actions/test_gen_action_details.py rename to tests/metagpt/ext/stanford_town/actions/test_gen_action_details.py index af4d94c4d..5dc5cbbe3 100644 --- a/examples/stanford_town/tests/actions/test_gen_action_details.py +++ b/tests/metagpt/ext/stanford_town/actions/test_gen_action_details.py @@ -4,17 +4,17 @@ import pytest -from examples.stanford_town.actions.gen_action_details import ( +from metagpt.environment import StanfordTownEnv +from metagpt.environment.api.env_api import EnvAPIAbstract +from metagpt.ext.stanford_town.actions.gen_action_details import ( GenActionArena, GenActionDetails, GenActionObject, GenActionSector, GenActObjDescription, ) -from examples.stanford_town.roles.st_role import STRole -from examples.stanford_town.utils.const import MAZE_ASSET_PATH -from metagpt.environment import StanfordTownEnv -from metagpt.environment.api.env_api import EnvAPIAbstract +from metagpt.ext.stanford_town.roles.st_role import STRole +from metagpt.ext.stanford_town.utils.const import MAZE_ASSET_PATH @pytest.mark.asyncio diff --git a/examples/stanford_town/tests/actions/test_summarize_conv.py b/tests/metagpt/ext/stanford_town/actions/test_summarize_conv.py similarity index 82% rename from examples/stanford_town/tests/actions/test_summarize_conv.py rename to tests/metagpt/ext/stanford_town/actions/test_summarize_conv.py index fa0de2031..5dfabcab9 100644 --- a/examples/stanford_town/tests/actions/test_summarize_conv.py +++ b/tests/metagpt/ext/stanford_town/actions/test_summarize_conv.py @@ -4,7 +4,7 @@ import pytest -from examples.stanford_town.actions.summarize_conv import SummarizeConv +from metagpt.ext.stanford_town.actions.summarize_conv import SummarizeConv @pytest.mark.asyncio diff --git a/tests/metagpt/ext/stanford_town/memory/__init__.py b/tests/metagpt/ext/stanford_town/memory/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/tests/metagpt/ext/stanford_town/memory/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/examples/stanford_town/tests/test_agent_memory.py b/tests/metagpt/ext/stanford_town/memory/test_agent_memory.py similarity index 94% rename from examples/stanford_town/tests/test_agent_memory.py rename to tests/metagpt/ext/stanford_town/memory/test_agent_memory.py index e7ad3fc43..d17014898 100644 --- a/examples/stanford_town/tests/test_agent_memory.py +++ b/tests/metagpt/ext/stanford_town/memory/test_agent_memory.py @@ -6,9 +6,9 @@ from datetime import datetime, timedelta import pytest -from examples.stanford_town.memory.agent_memory import AgentMemory -from examples.stanford_town.memory.retrieve import agent_retrieve -from examples.stanford_town.utils.const import STORAGE_PATH +from metagpt.ext.stanford_town.memory import AgentMemory +from metagpt.ext.stanford_town.memory.retrieve import agent_retrieve +from metagpt.ext.stanford_town.utils.const import STORAGE_PATH from metagpt.logs import logger """ diff --git a/examples/stanford_town/tests/test_basic_memory.py b/tests/metagpt/ext/stanford_town/memory/test_basic_memory.py similarity index 95% rename from examples/stanford_town/tests/test_basic_memory.py rename to tests/metagpt/ext/stanford_town/memory/test_basic_memory.py index fa2a5c487..36a9b2f99 100644 --- a/examples/stanford_town/tests/test_basic_memory.py +++ b/tests/metagpt/ext/stanford_town/memory/test_basic_memory.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta import pytest -from examples.stanford_town.memory.agent_memory import BasicMemory +from metagpt.ext.stanford_town.memory.agent_memory import BasicMemory from metagpt.logs import logger """ diff --git a/examples/stanford_town/tests/test_spatial_memory.py b/tests/metagpt/ext/stanford_town/memory/test_spatial_memory.py similarity index 77% rename from examples/stanford_town/tests/test_spatial_memory.py rename to tests/metagpt/ext/stanford_town/memory/test_spatial_memory.py index 822db753e..e05b273fd 100644 --- a/examples/stanford_town/tests/test_spatial_memory.py +++ b/tests/metagpt/ext/stanford_town/memory/test_spatial_memory.py @@ -2,8 +2,8 @@ # -*- coding: utf-8 -*- # @Desc : the unittest of MemoryTree -from ..memory.spatial_memory import MemoryTree -from ..utils.const import STORAGE_PATH +from metagpt.ext.stanford_town.memory.spatial_memory import MemoryTree +from metagpt.ext.stanford_town.utils.const import STORAGE_PATH def test_spatial_memory(): diff --git a/tests/metagpt/ext/stanford_town/plan/__init__.py b/tests/metagpt/ext/stanford_town/plan/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/tests/metagpt/ext/stanford_town/plan/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/examples/stanford_town/tests/plan/test_conversation.py b/tests/metagpt/ext/stanford_town/plan/test_conversation.py similarity index 83% rename from examples/stanford_town/tests/plan/test_conversation.py rename to tests/metagpt/ext/stanford_town/plan/test_conversation.py index ff8664538..35dd216f9 100644 --- a/examples/stanford_town/tests/plan/test_conversation.py +++ b/tests/metagpt/ext/stanford_town/plan/test_conversation.py @@ -6,12 +6,12 @@ from typing import Tuple import pytest -from examples.stanford_town.plan.converse import agent_conversation -from examples.stanford_town.roles.st_role import STRole -from examples.stanford_town.utils.const import MAZE_ASSET_PATH, STORAGE_PATH -from examples.stanford_town.utils.mg_ga_transform import get_reverie_meta -from examples.stanford_town.utils.utils import copy_folder from metagpt.environment import StanfordTownEnv +from metagpt.ext.stanford_town.plan.converse import agent_conversation +from metagpt.ext.stanford_town.roles.st_role import STRole +from metagpt.ext.stanford_town.utils.const import MAZE_ASSET_PATH, STORAGE_PATH +from metagpt.ext.stanford_town.utils.mg_ga_transform import get_reverie_meta +from metagpt.ext.stanford_town.utils.utils import copy_folder async def init_two_roles(fork_sim_code: str = "base_the_ville_isabella_maria_klaus") -> Tuple["STRole"]: diff --git a/examples/stanford_town/tests/plan/test_st_plan.py b/tests/metagpt/ext/stanford_town/plan/test_st_plan.py similarity index 90% rename from examples/stanford_town/tests/plan/test_st_plan.py rename to tests/metagpt/ext/stanford_town/plan/test_st_plan.py index ad20d3068..18663a066 100644 --- a/examples/stanford_town/tests/plan/test_st_plan.py +++ b/tests/metagpt/ext/stanford_town/plan/test_st_plan.py @@ -4,12 +4,12 @@ import pytest -from examples.stanford_town.plan.st_plan import ( +from metagpt.ext.stanford_town.plan.st_plan import ( _choose_retrieved, _should_react, _wait_react, ) -from examples.stanford_town.tests.plan.test_converse import init_two_roles +from tests.metagpt.ext.stanford_town.plan.test_conversation import init_two_roles def test_should_react(): diff --git a/tests/metagpt/ext/stanford_town/roles/__init__.py b/tests/metagpt/ext/stanford_town/roles/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/tests/metagpt/ext/stanford_town/roles/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/examples/stanford_town/tests/roles/test_st_role.py b/tests/metagpt/ext/stanford_town/roles/test_st_role.py similarity index 75% rename from examples/stanford_town/tests/roles/test_st_role.py rename to tests/metagpt/ext/stanford_town/roles/test_st_role.py index b2e4c6969..affa6e87f 100644 --- a/examples/stanford_town/tests/roles/test_st_role.py +++ b/tests/metagpt/ext/stanford_town/roles/test_st_role.py @@ -4,10 +4,10 @@ import pytest -from examples.stanford_town.memory.agent_memory import BasicMemory -from examples.stanford_town.roles.st_role import STRole -from examples.stanford_town.utils.const import MAZE_ASSET_PATH from metagpt.environment import StanfordTownEnv +from metagpt.ext.stanford_town.memory.agent_memory import BasicMemory +from metagpt.ext.stanford_town.roles.st_role import STRole +from metagpt.ext.stanford_town.utils.const import MAZE_ASSET_PATH @pytest.mark.asyncio diff --git a/examples/stanford_town/tests/test_reflect.py b/tests/metagpt/ext/stanford_town/test_reflect.py similarity index 94% rename from examples/stanford_town/tests/test_reflect.py rename to tests/metagpt/ext/stanford_town/test_reflect.py index dbcbd974f..0be23166c 100644 --- a/examples/stanford_town/tests/test_reflect.py +++ b/tests/metagpt/ext/stanford_town/test_reflect.py @@ -4,14 +4,14 @@ import pytest -from examples.stanford_town.actions.run_reflect_action import ( +from metagpt.environment import StanfordTownEnv +from metagpt.ext.stanford_town.actions.run_reflect_action import ( AgentEventTriple, AgentFocusPt, AgentInsightAndGuidance, ) -from examples.stanford_town.roles.st_role import STRole -from examples.stanford_town.utils.const import MAZE_ASSET_PATH -from metagpt.environment import StanfordTownEnv +from metagpt.ext.stanford_town.roles.st_role import STRole +from metagpt.ext.stanford_town.utils.const import MAZE_ASSET_PATH @pytest.mark.asyncio From d80f65b53856f47373bbb7bb7931b19fd92c12ae Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 28 Mar 2024 18:23:43 +0800 Subject: [PATCH 080/127] update ut --- .../stanford_town/stanford_town_env.py | 2 +- .../actions/gen_action_details.py | 1 - .../actions/gen_iter_chat_utt.py | 2 +- .../ext/stanford_town/actions/task_decomp.py | 9 +++-- .../stanford_town/memory/spatial_memory.py | 5 +-- metagpt/ext/stanford_town/plan/st_plan.py | 30 +++++------------ tests/data/rsp_cache.json | 20 ++++++++++- .../test_stanford_town_ext_env.py | 2 +- .../actions/test_gen_action_details.py | 1 - .../stanford_town/memory/test_agent_memory.py | 2 +- .../ext/stanford_town/plan/test_st_plan.py | 33 +++++-------------- 11 files changed, 47 insertions(+), 60 deletions(-) diff --git a/metagpt/environment/stanford_town/stanford_town_env.py b/metagpt/environment/stanford_town/stanford_town_env.py index 2b6212781..af8a882b2 100644 --- a/metagpt/environment/stanford_town/stanford_town_env.py +++ b/metagpt/environment/stanford_town/stanford_town_env.py @@ -6,5 +6,5 @@ from metagpt.environment.base_env import Environment from metagpt.environment.stanford_town.stanford_town_ext_env import StanfordTownExtEnv -class StanfordTownEnv(Environment, StanfordTownExtEnv): +class StanfordTownEnv(StanfordTownExtEnv, Environment): pass diff --git a/metagpt/ext/stanford_town/actions/gen_action_details.py b/metagpt/ext/stanford_town/actions/gen_action_details.py index 3820ddc5b..8e268a723 100644 --- a/metagpt/ext/stanford_town/actions/gen_action_details.py +++ b/metagpt/ext/stanford_town/actions/gen_action_details.py @@ -153,7 +153,6 @@ class GenActionArena(STAction): prompt_input = create_prompt_input(role, act_desp, act_world, act_sector) prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) self.fail_default_resp = self._func_fail_default_resp() - print("prompt ", prompt) output = await self._run_gpt35_max_tokens(prompt, max_tokens=15) logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") return output diff --git a/metagpt/ext/stanford_town/actions/gen_iter_chat_utt.py b/metagpt/ext/stanford_town/actions/gen_iter_chat_utt.py index 41460afba..40f6d3af0 100644 --- a/metagpt/ext/stanford_town/actions/gen_iter_chat_utt.py +++ b/metagpt/ext/stanford_town/actions/gen_iter_chat_utt.py @@ -77,7 +77,7 @@ class GenIterChatUTT(STAction): if role.rc.memory.chat_list: if int((scratch.curr_time - role.rc.memory.chat_list[-1].created).total_seconds() / 60) > 480: prev_convo_insert = "" - print(prev_convo_insert) + logger.info(f"prev_convo_insert: {prev_convo_insert}") curr_sector = f"{access_tile['sector']}" curr_arena = f"{access_tile['arena']}" diff --git a/metagpt/ext/stanford_town/actions/task_decomp.py b/metagpt/ext/stanford_town/actions/task_decomp.py index a4e22c24d..3a23a7345 100644 --- a/metagpt/ext/stanford_town/actions/task_decomp.py +++ b/metagpt/ext/stanford_town/actions/task_decomp.py @@ -100,14 +100,14 @@ class TaskDecomp(STAction): curr_time_range = "" - print("DEBUG") - print(role.scratch.f_daily_schedule_hourly_org) - print(all_indices) + logger.debug("DEBUG") + logger.debug(role.scratch.f_daily_schedule_hourly_org) + logger.debug(all_indices) summ_str = f'Today is {role.scratch.curr_time.strftime("%B %d, %Y")}. ' summ_str += "From " for index in all_indices: - print("index", index) + logger.debug(f"index {index}") if index < len(role.scratch.f_daily_schedule_hourly_org): start_min = 0 for i in range(index): @@ -161,7 +161,6 @@ class TaskDecomp(STAction): for fi_task, fi_duration in fin_output: ftime_sum += fi_duration - # print ("for debugging... line 365", fin_output) fin_output[-1][1] += truncated_act_dur - ftime_sum output = fin_output diff --git a/metagpt/ext/stanford_town/memory/spatial_memory.py b/metagpt/ext/stanford_town/memory/spatial_memory.py index c1030cdbb..71b856907 100644 --- a/metagpt/ext/stanford_town/memory/spatial_memory.py +++ b/metagpt/ext/stanford_town/memory/spatial_memory.py @@ -9,6 +9,7 @@ from pathlib import Path from pydantic import BaseModel, Field +from metagpt.logs import logger from metagpt.utils.common import read_json_file, write_json_file @@ -23,12 +24,12 @@ class MemoryTree(BaseModel): dash = " >" * depth if isinstance(tree, list): if tree: - print(dash, tree) + logger.info(f"{dash} {tree}") return for key, val in tree.items(): if key: - print(dash, key) + logger.info(f"{dash} {tree}") _print_tree(val, depth + 1) _print_tree(self.tree, 0) diff --git a/metagpt/ext/stanford_town/plan/st_plan.py b/metagpt/ext/stanford_town/plan/st_plan.py index 6b6323b7a..f63052fc5 100644 --- a/metagpt/ext/stanford_town/plan/st_plan.py +++ b/metagpt/ext/stanford_town/plan/st_plan.py @@ -448,7 +448,7 @@ async def generate_new_decomp_schedule( count = 0 # enumerate count truncated_fin = False - print("DEBUG::: ", scratch.name) + logger.debug(f"DEBUG::: {scratch.name}") for act, dur in scratch.f_daily_schedule: if (dur_sum >= start_hour * 60) and (dur_sum < end_hour * 60): main_act_dur += [[act, dur]] @@ -463,7 +463,7 @@ async def generate_new_decomp_schedule( ) # DEC 7 DEBUG;.. is the +1 the right thing to do??? # DEC 7 DEBUG;.. is the +1 the right thing to do??? # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass + 1) - print("DEBUG::: ", truncated_act_dur) + logger.debug(f"DEBUG::: {truncated_act_dur}") # DEC 7 DEBUG;.. is the +1 the right thing to do??? # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass) @@ -550,10 +550,6 @@ async def _long_term_planning(role: "STRole", new_day: bool): created, expiration, s, p, o, thought, keywords, thought_poignancy, thought_embedding_pair, None ) - # print("Sleeping for 20 seconds...") - # time.sleep(10) - # print("Done sleeping!") - async def _determine_action(role: "STRole"): """ @@ -636,22 +632,20 @@ async def _determine_action(role: "STRole"): # Generate an instance from the action description and duration. By # this point, we assume that all the relevant actions are decomposed and # ready in f_daily_schedule. - print("DEBUG LJSDLFSKJF") + logger.debug("DEBUG LJSDLFSKJF") for i in role.scratch.f_daily_schedule: - print(i) - print(curr_index) - print(len(role.scratch.f_daily_schedule)) - print(role.scratch.name) - print("------") + logger.debug(i) + logger.debug(curr_index) + logger.debug(len(role.scratch.f_daily_schedule)) + logger.debug(role.scratch.name) # 1440 x_emergency = 0 for i in role.scratch.f_daily_schedule: x_emergency += i[1] - # print ("x_emergency", x_emergency) if 1440 - x_emergency > 0: - print("x_emergency__AAA", x_emergency) + logger.info(f"x_emergency__AAA: {x_emergency}") role.scratch.f_daily_schedule += [["sleeping", 1440 - x_emergency]] act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] @@ -675,14 +669,12 @@ def revise_identity(role: "STRole"): for i in val: statements += f"{i.created.strftime('%A %B %d -- %H:%M %p')}: {i.embedding_key}\n" - # print (";adjhfno;asdjao;idfjo;af", p_name) plan_prompt = statements + "\n" plan_prompt += f"Given the statements above, is there anything that {p_name} should remember as they plan for" plan_prompt += f" *{role.scratch.curr_time.strftime('%A %B %d')}*? " plan_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement)\n\n" plan_prompt += f"Write the response from {p_name}'s perspective." plan_note = LLM().ask(plan_prompt) - # print (plan_note) thought_prompt = statements + "\n" thought_prompt += ( @@ -690,7 +682,6 @@ def revise_identity(role: "STRole"): ) thought_prompt += f"Write the response from {p_name}'s perspective." thought_note = LLM().ask(thought_prompt) - # print (thought_note) currently_prompt = ( f"{p_name}'s status from {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" @@ -701,11 +692,7 @@ def revise_identity(role: "STRole"): currently_prompt += f"It is now {role.scratch.curr_time.strftime('%A %B %d')}. Given the above, write {p_name}'s status for {role.scratch.curr_time.strftime('%A %B %d')} that reflects {p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}. Write this in third-person talking about {p_name}." currently_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement).\n\n" currently_prompt += "Follow this format below:\nStatus: " - # print ("DEBUG ;adjhfno;asdjao;asdfsidfjo;af", p_name) - # print (currently_prompt) new_currently = LLM().ask(currently_prompt) - # print (new_currently) - # print (new_currently[10:]) role.scratch.currently = new_currently @@ -716,5 +703,4 @@ def revise_identity(role: "STRole"): new_daily_req = LLM().ask(daily_req_prompt) new_daily_req = new_daily_req.replace("\n", " ") - print("WE ARE HERE!!!", new_daily_req) role.scratch.daily_plan_req = new_daily_req diff --git a/tests/data/rsp_cache.json b/tests/data/rsp_cache.json index 565241779..772b4d5ed 100644 --- a/tests/data/rsp_cache.json +++ b/tests/data/rsp_cache.json @@ -427,5 +427,23 @@ "As a data scientist, you need to help user to achieve their goal step by step in a continuous Jupyter notebook. Since it is a notebook environment, don't use asyncio.run. Instead, use await if you need to call an async function.#SYSTEM_MSG_END#\n# User Requirement\nRun data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.\n\n# Plan Status\n\n## Finished Tasks\n### code\n```python\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.datasets import load_wine\nwine = load_wine()\nwine_df = pd.DataFrame(data=np.c_[wine['data'], wine['target']],\n columns=wine['feature_names'] + ['target'])\nnumerical_summary = wine_df.describe()\ncategorical_summary = wine_df.select_dtypes(include=['object', 'category']).describe()\ncorrelation_matrix = wine_df.corr()\nwine_df.hist(bins=15, figsize=(15, 10), layout=(4, 4))\nplt.tight_layout()\nplt.show()\nprint(\"Numerical Summary:\\n\", numerical_summary)\nprint(\"\\nCategorical Summary:\\n\", categorical_summary)\nprint(\"\\nCorrelation Matrix:\\n\", correlation_matrix)\n\nfrom sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val = train_test_split(\n wine_df.drop('target', axis=1),\n wine_df['target'],\n test_size=0.2,\n random_state=42\n)\nprint(f\"Training set shape: {X_train.shape}\")\nprint(f\"Validation set shape: {X_val.shape}\")\nprint(f\"Training target shape: {y_train.shape}\")\nprint(f\"Validation target shape: {y_val.shape}\")\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score\nmodel = RandomForestClassifier(n_estimators=100, random_state=42)\nmodel.fit(X_train, y_train)\ny_val_pred = model.predict(X_val)\nval_accuracy = accuracy_score(y_val, y_val_pred)\nprint(f\"Validation Accuracy: {val_accuracy:.4f}\")\n```\n\n### execution result\na successful run\n\na successful run\n\na successful run\n\n## Current Task\nEvaluate the model on the validation set and show the validation accuracy.\n\n## Task Guidance\nWrite complete code for 'Current Task'. And avoid duplicating code from 'Finished Tasks', such as repeated import of packages, reading data, etc.\nSpecifically, \nThe current task is about evaluating a model, please note the following:\n- Ensure that the evaluated data is same processed as the training data. If not, remember use object in 'Done Tasks' to transform the data.\n- Use trained model from previous task result directly, do not mock or reload model yourself.\n\n\n\n# Tool Info\n\n\n# Constraints\n- Take on Current Task if it is in Plan Status, otherwise, tackle User Requirement directly.\n- Ensure the output new code is executable in the same Jupyter notebook as the previous executed code.\n- Always prioritize using pre-defined tools for the same functionality.\n\n# Output\nWhile some concise thoughts are helpful, code is absolutely required. Always output one and only one code block in your response. Output code in the following format:\n```python\nyour code\n```\n": "```python\n# Evaluate the model on the validation set and show the validation accuracy\nval_accuracy = accuracy_score(y_val, y_val_pred)\nprint(f\"Validation Accuracy: {val_accuracy:.4f}\")\n```", "user: \n## User Requirement\nRun data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.\n## Context\n\n## Current Plan\n[\n {\n \"task_id\": \"1\",\n \"dependent_task_ids\": [],\n \"instruction\": \"Perform exploratory data analysis on the sklearn Wine recognition dataset including summary statistics and a plot.\",\n \"task_type\": \"eda\",\n \"code\": \"import numpy as np\\nimport pandas as pd\\nimport matplotlib.pyplot as plt\\nfrom sklearn.datasets import load_wine\\n\\n# Load the wine dataset\\nwine = load_wine()\\nwine_df = pd.DataFrame(data=np.c_[wine['data'], wine['target']],\\n columns=wine['feature_names'] + ['target'])\\n\\n# Summary statistics for numerical features\\nnumerical_summary = wine_df.describe()\\n\\n# Summary statistics for categorical features\\ncategorical_summary = wine_df.select_dtypes(include=['object', 'category']).describe()\\n\\n# Correlation matrix for numerical features\\ncorrelation_matrix = wine_df.corr()\\n\\n# Plotting a histogram for each numerical feature\\nwine_df.hist(bins=15, figsize=(15, 10), layout=(4, 4))\\nplt.tight_layout()\\nplt.show()\\n\\n# Displaying the summary statistics\\nprint(\\\"Numerical Summary:\\\\n\\\", numerical_summary)\\nprint(\\\"\\\\nCategorical Summary:\\\\n\\\", categorical_summary)\\nprint(\\\"\\\\nCorrelation Matrix:\\\\n\\\", correlation_matrix)\\n\",\n \"result\": \"a successful run\",\n \"is_success\": true,\n \"is_finished\": true\n },\n {\n \"task_id\": \"2\",\n \"dependent_task_ids\": [\n \"1\"\n ],\n \"instruction\": \"Preprocess the dataset by splitting it into training and validation sets with a 80-20 split.\",\n \"task_type\": \"data preprocessing\",\n \"code\": \"\",\n \"result\": \"\",\n \"is_success\": false,\n \"is_finished\": false\n },\n {\n \"task_id\": \"3\",\n \"dependent_task_ids\": [\n \"2\"\n ],\n \"instruction\": \"Train a model using the training set to predict wine class.\",\n \"task_type\": \"model train\",\n \"code\": \"\",\n \"result\": \"\",\n \"is_success\": false,\n \"is_finished\": false\n },\n {\n \"task_id\": \"4\",\n \"dependent_task_ids\": [\n \"3\"\n ],\n \"instruction\": \"Evaluate the model on the validation set and show the validation accuracy.\",\n \"task_type\": \"model evaluate\",\n \"code\": \"\",\n \"result\": \"\",\n \"is_success\": false,\n \"is_finished\": false\n }\n]\n## Current Task\n{\"task_id\":\"2\",\"dependent_task_ids\":[\"1\"],\"instruction\":\"Preprocess the dataset by splitting it into training and validation sets with a 80-20 split.\",\"task_type\":\"data preprocessing\",\"code\":\"\",\"result\":\"\",\"is_success\":false,\"is_finished\":false}\n\nuser: \n# Latest Data Info\nLatest data info after previous tasks:\na successful run\n\nassistant: from sklearn.model_selection import train_test_split\n\n# Split the data into training and validation sets (80-20 split)\nX_train, X_val, y_train, y_val = train_test_split(\n wine_df.drop('target', axis=1), # features\n wine_df['target'], # target variable\n test_size=0.2, # 20% for validation\n random_state=42 # seed for reproducibility\n)\n\n# Output the shapes of the resulting data splits\nprint(f\"Training set shape: {X_train.shape}\")\nprint(f\"Validation set shape: {X_val.shape}\")\nprint(f\"Training target shape: {y_train.shape}\")\nprint(f\"Validation target shape: {y_val.shape}\")\n\nuser: a successful runThis is a review. Please review output from metagpt.actions.di.execute_nb_code.ExecuteNbCode\nIf you want to change, add, delete a task or merge tasks in the plan, say 'change task task_id or current task, ... (things to change)' If you confirm the output from the current task and wish to continue, type: confirmIf you think user requirement has been fulfilled completedly, you can finish the process by typing: finish\nIf you want to terminate the process, type: exit\nPlease type your review below:\n": "confirm", "user: \n## User Requirement\nRun data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.\n## Context\n\n## Current Plan\n[\n {\n \"task_id\": \"1\",\n \"dependent_task_ids\": [],\n \"instruction\": \"Perform exploratory data analysis on the sklearn Wine recognition dataset including summary statistics and a plot.\",\n \"task_type\": \"eda\",\n \"code\": \"import numpy as np\\nimport pandas as pd\\nimport matplotlib.pyplot as plt\\nfrom sklearn.datasets import load_wine\\n\\n# Load the wine dataset\\nwine = load_wine()\\nwine_df = pd.DataFrame(data=np.c_[wine['data'], wine['target']],\\n columns=wine['feature_names'] + ['target'])\\n\\n# Summary statistics for numerical features\\nnumerical_summary = wine_df.describe()\\n\\n# Summary statistics for categorical features\\ncategorical_summary = wine_df.select_dtypes(include=['object', 'category']).describe()\\n\\n# Correlation matrix for numerical features\\ncorrelation_matrix = wine_df.corr()\\n\\n# Plotting a histogram for each numerical feature\\nwine_df.hist(bins=15, figsize=(15, 10), layout=(4, 4))\\nplt.tight_layout()\\nplt.show()\\n\\n# Displaying the summary statistics\\nprint(\\\"Numerical Summary:\\\\n\\\", numerical_summary)\\nprint(\\\"\\\\nCategorical Summary:\\\\n\\\", categorical_summary)\\nprint(\\\"\\\\nCorrelation Matrix:\\\\n\\\", correlation_matrix)\\n\",\n \"result\": \"a successful run\",\n \"is_success\": true,\n \"is_finished\": true\n },\n {\n \"task_id\": \"2\",\n \"dependent_task_ids\": [\n \"1\"\n ],\n \"instruction\": \"Preprocess the dataset by splitting it into training and validation sets with a 80-20 split.\",\n \"task_type\": \"data preprocessing\",\n \"code\": \"from sklearn.model_selection import train_test_split\\n\\n# Split the data into training and validation sets (80-20 split)\\nX_train, X_val, y_train, y_val = train_test_split(\\n wine_df.drop('target', axis=1), # features\\n wine_df['target'], # target variable\\n test_size=0.2, # 20% for validation\\n random_state=42 # seed for reproducibility\\n)\\n\\n# Output the shapes of the resulting data splits\\nprint(f\\\"Training set shape: {X_train.shape}\\\")\\nprint(f\\\"Validation set shape: {X_val.shape}\\\")\\nprint(f\\\"Training target shape: {y_train.shape}\\\")\\nprint(f\\\"Validation target shape: {y_val.shape}\\\")\\n\",\n \"result\": \"a successful run\",\n \"is_success\": true,\n \"is_finished\": true\n },\n {\n \"task_id\": \"3\",\n \"dependent_task_ids\": [\n \"2\"\n ],\n \"instruction\": \"Train a model using the training set to predict wine class.\",\n \"task_type\": \"model train\",\n \"code\": \"\",\n \"result\": \"\",\n \"is_success\": false,\n \"is_finished\": false\n },\n {\n \"task_id\": \"4\",\n \"dependent_task_ids\": [\n \"3\"\n ],\n \"instruction\": \"Evaluate the model on the validation set and show the validation accuracy.\",\n \"task_type\": \"model evaluate\",\n \"code\": \"\",\n \"result\": \"\",\n \"is_success\": false,\n \"is_finished\": false\n }\n]\n## Current Task\n{\"task_id\":\"3\",\"dependent_task_ids\":[\"2\"],\"instruction\":\"Train a model using the training set to predict wine class.\",\"task_type\":\"model train\",\"code\":\"\",\"result\":\"\",\"is_success\":false,\"is_finished\":false}\n\nuser: \n# Latest Data Info\nLatest data info after previous tasks:\na successful run\n\nassistant: from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score\n\n# Initialize the RandomForestClassifier\nmodel = RandomForestClassifier(n_estimators=100, random_state=42)\n\n# Train the model on the training data\nmodel.fit(X_train, y_train)\n\n# Predict the wine classes on the validation set\ny_val_pred = model.predict(X_val)\n\n# Calculate the accuracy on the validation set\nval_accuracy = accuracy_score(y_val, y_val_pred)\n\nprint(f\"Validation Accuracy: {val_accuracy:.4f}\")\n\nuser: a successful runThis is a review. Please review output from metagpt.actions.di.execute_nb_code.ExecuteNbCode\nIf you want to change, add, delete a task or merge tasks in the plan, say 'change task task_id or current task, ... (things to change)' If you confirm the output from the current task and wish to continue, type: confirmIf you think user requirement has been fulfilled completedly, you can finish the process by typing: finish\nIf you want to terminate the process, type: exit\nPlease type your review below:\n": "confirm", - "user: \n## User Requirement\nRun data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.\n## Context\n\n## Current Plan\n[\n {\n \"task_id\": \"1\",\n \"dependent_task_ids\": [],\n \"instruction\": \"Perform exploratory data analysis on the sklearn Wine recognition dataset including summary statistics and a plot.\",\n \"task_type\": \"eda\",\n \"code\": \"import numpy as np\\nimport pandas as pd\\nimport matplotlib.pyplot as plt\\nfrom sklearn.datasets import load_wine\\n\\n# Load the wine dataset\\nwine = load_wine()\\nwine_df = pd.DataFrame(data=np.c_[wine['data'], wine['target']],\\n columns=wine['feature_names'] + ['target'])\\n\\n# Summary statistics for numerical features\\nnumerical_summary = wine_df.describe()\\n\\n# Summary statistics for categorical features\\ncategorical_summary = wine_df.select_dtypes(include=['object', 'category']).describe()\\n\\n# Correlation matrix for numerical features\\ncorrelation_matrix = wine_df.corr()\\n\\n# Plotting a histogram for each numerical feature\\nwine_df.hist(bins=15, figsize=(15, 10), layout=(4, 4))\\nplt.tight_layout()\\nplt.show()\\n\\n# Displaying the summary statistics\\nprint(\\\"Numerical Summary:\\\\n\\\", numerical_summary)\\nprint(\\\"\\\\nCategorical Summary:\\\\n\\\", categorical_summary)\\nprint(\\\"\\\\nCorrelation Matrix:\\\\n\\\", correlation_matrix)\\n\",\n \"result\": \"a successful run\",\n \"is_success\": true,\n \"is_finished\": true\n },\n {\n \"task_id\": \"2\",\n \"dependent_task_ids\": [\n \"1\"\n ],\n \"instruction\": \"Preprocess the dataset by splitting it into training and validation sets with a 80-20 split.\",\n \"task_type\": \"data preprocessing\",\n \"code\": \"from sklearn.model_selection import train_test_split\\n\\n# Split the data into training and validation sets (80-20 split)\\nX_train, X_val, y_train, y_val = train_test_split(\\n wine_df.drop('target', axis=1), # features\\n wine_df['target'], # target variable\\n test_size=0.2, # 20% for validation\\n random_state=42 # seed for reproducibility\\n)\\n\\n# Output the shapes of the resulting data splits\\nprint(f\\\"Training set shape: {X_train.shape}\\\")\\nprint(f\\\"Validation set shape: {X_val.shape}\\\")\\nprint(f\\\"Training target shape: {y_train.shape}\\\")\\nprint(f\\\"Validation target shape: {y_val.shape}\\\")\\n\",\n \"result\": \"a successful run\",\n \"is_success\": true,\n \"is_finished\": true\n },\n {\n \"task_id\": \"3\",\n \"dependent_task_ids\": [\n \"2\"\n ],\n \"instruction\": \"Train a model using the training set to predict wine class.\",\n \"task_type\": \"model train\",\n \"code\": \"from sklearn.ensemble import RandomForestClassifier\\nfrom sklearn.metrics import accuracy_score\\n\\n# Initialize the RandomForestClassifier\\nmodel = RandomForestClassifier(n_estimators=100, random_state=42)\\n\\n# Train the model on the training data\\nmodel.fit(X_train, y_train)\\n\\n# Predict the wine classes on the validation set\\ny_val_pred = model.predict(X_val)\\n\\n# Calculate the accuracy on the validation set\\nval_accuracy = accuracy_score(y_val, y_val_pred)\\n\\nprint(f\\\"Validation Accuracy: {val_accuracy:.4f}\\\")\\n\",\n \"result\": \"a successful run\",\n \"is_success\": true,\n \"is_finished\": true\n },\n {\n \"task_id\": \"4\",\n \"dependent_task_ids\": [\n \"3\"\n ],\n \"instruction\": \"Evaluate the model on the validation set and show the validation accuracy.\",\n \"task_type\": \"model evaluate\",\n \"code\": \"\",\n \"result\": \"\",\n \"is_success\": false,\n \"is_finished\": false\n }\n]\n## Current Task\n{\"task_id\":\"4\",\"dependent_task_ids\":[\"3\"],\"instruction\":\"Evaluate the model on the validation set and show the validation accuracy.\",\"task_type\":\"model evaluate\",\"code\":\"\",\"result\":\"\",\"is_success\":false,\"is_finished\":false}\n\nassistant: # Evaluate the model on the validation set and show the validation accuracy\nval_accuracy = accuracy_score(y_val, y_val_pred)\nprint(f\"Validation Accuracy: {val_accuracy:.4f}\")\n\nuser: a successful runThis is a review. Please review output from metagpt.actions.di.execute_nb_code.ExecuteNbCode\nIf you want to change, add, delete a task or merge tasks in the plan, say 'change task task_id or current task, ... (things to change)' If you confirm the output from the current task and wish to continue, type: confirmIf you think user requirement has been fulfilled completedly, you can finish the process by typing: finish\nIf you want to terminate the process, type: exit\nPlease type your review below:\n": "confirm" + "user: \n## User Requirement\nRun data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.\n## Context\n\n## Current Plan\n[\n {\n \"task_id\": \"1\",\n \"dependent_task_ids\": [],\n \"instruction\": \"Perform exploratory data analysis on the sklearn Wine recognition dataset including summary statistics and a plot.\",\n \"task_type\": \"eda\",\n \"code\": \"import numpy as np\\nimport pandas as pd\\nimport matplotlib.pyplot as plt\\nfrom sklearn.datasets import load_wine\\n\\n# Load the wine dataset\\nwine = load_wine()\\nwine_df = pd.DataFrame(data=np.c_[wine['data'], wine['target']],\\n columns=wine['feature_names'] + ['target'])\\n\\n# Summary statistics for numerical features\\nnumerical_summary = wine_df.describe()\\n\\n# Summary statistics for categorical features\\ncategorical_summary = wine_df.select_dtypes(include=['object', 'category']).describe()\\n\\n# Correlation matrix for numerical features\\ncorrelation_matrix = wine_df.corr()\\n\\n# Plotting a histogram for each numerical feature\\nwine_df.hist(bins=15, figsize=(15, 10), layout=(4, 4))\\nplt.tight_layout()\\nplt.show()\\n\\n# Displaying the summary statistics\\nprint(\\\"Numerical Summary:\\\\n\\\", numerical_summary)\\nprint(\\\"\\\\nCategorical Summary:\\\\n\\\", categorical_summary)\\nprint(\\\"\\\\nCorrelation Matrix:\\\\n\\\", correlation_matrix)\\n\",\n \"result\": \"a successful run\",\n \"is_success\": true,\n \"is_finished\": true\n },\n {\n \"task_id\": \"2\",\n \"dependent_task_ids\": [\n \"1\"\n ],\n \"instruction\": \"Preprocess the dataset by splitting it into training and validation sets with a 80-20 split.\",\n \"task_type\": \"data preprocessing\",\n \"code\": \"from sklearn.model_selection import train_test_split\\n\\n# Split the data into training and validation sets (80-20 split)\\nX_train, X_val, y_train, y_val = train_test_split(\\n wine_df.drop('target', axis=1), # features\\n wine_df['target'], # target variable\\n test_size=0.2, # 20% for validation\\n random_state=42 # seed for reproducibility\\n)\\n\\n# Output the shapes of the resulting data splits\\nprint(f\\\"Training set shape: {X_train.shape}\\\")\\nprint(f\\\"Validation set shape: {X_val.shape}\\\")\\nprint(f\\\"Training target shape: {y_train.shape}\\\")\\nprint(f\\\"Validation target shape: {y_val.shape}\\\")\\n\",\n \"result\": \"a successful run\",\n \"is_success\": true,\n \"is_finished\": true\n },\n {\n \"task_id\": \"3\",\n \"dependent_task_ids\": [\n \"2\"\n ],\n \"instruction\": \"Train a model using the training set to predict wine class.\",\n \"task_type\": \"model train\",\n \"code\": \"from sklearn.ensemble import RandomForestClassifier\\nfrom sklearn.metrics import accuracy_score\\n\\n# Initialize the RandomForestClassifier\\nmodel = RandomForestClassifier(n_estimators=100, random_state=42)\\n\\n# Train the model on the training data\\nmodel.fit(X_train, y_train)\\n\\n# Predict the wine classes on the validation set\\ny_val_pred = model.predict(X_val)\\n\\n# Calculate the accuracy on the validation set\\nval_accuracy = accuracy_score(y_val, y_val_pred)\\n\\nprint(f\\\"Validation Accuracy: {val_accuracy:.4f}\\\")\\n\",\n \"result\": \"a successful run\",\n \"is_success\": true,\n \"is_finished\": true\n },\n {\n \"task_id\": \"4\",\n \"dependent_task_ids\": [\n \"3\"\n ],\n \"instruction\": \"Evaluate the model on the validation set and show the validation accuracy.\",\n \"task_type\": \"model evaluate\",\n \"code\": \"\",\n \"result\": \"\",\n \"is_success\": false,\n \"is_finished\": false\n }\n]\n## Current Task\n{\"task_id\":\"4\",\"dependent_task_ids\":[\"3\"],\"instruction\":\"Evaluate the model on the validation set and show the validation accuracy.\",\"task_type\":\"model evaluate\",\"code\":\"\",\"result\":\"\",\"is_success\":false,\"is_finished\":false}\n\nassistant: # Evaluate the model on the validation set and show the validation accuracy\nval_accuracy = accuracy_score(y_val, y_val_pred)\nprint(f\"Validation Accuracy: {val_accuracy:.4f}\")\n\nuser: a successful runThis is a review. Please review output from metagpt.actions.di.execute_nb_code.ExecuteNbCode\nIf you want to change, add, delete a task or merge tasks in the plan, say 'change task task_id or current task, ... (things to change)' If you confirm the output from the current task and wish to continue, type: confirmIf you think user requirement has been fulfilled completedly, you can finish the process by typing: finish\nIf you want to terminate the process, type: exit\nPlease type your review below:\n": "confirm", + "\"\"\"\nGiven only the information above, what are 3 most salient high-level questions we can answer about the subjects grounded in the statements?\n1)\n\"\"\"\nOutput the response to the prompt above in json. Output must be a list of str.\nExample output json:\n{\"output\": \"[\"What should Jane do for lunch\", \"Does Jane like strawberry\", \"Who is Jane\"]\"}": "{\n \"output\": [\n \"What is the significance of the information provided?\",\n \"What are the implications of Jane's preferences?\",\n \"How does the information relate to the subject's identity?\"\n ]\n}", + "Input:\n[user: Klaus Mueller has a close relationship with Maria Lopez, user:s Mueller and Maria Lopez have a close relationship, user: Klaus Mueller has a close relationship with Maria Lopez, user: Klaus Mueller has a close relationship with Maria Lopez, user: Klaus Mueller and Maria Lopez have a strong relationship, user: Klaus Mueller is a dormmate of Maria Lopez., user: Klaus Mueller and Maria Lopez have a strong bond, user: Klaus Mueller has a crush on Maria Lopez, user: Klaus Mueller and Maria Lopez have been friends for more than 2 years., user: Klaus Mueller has a close relationship with Maria Lopez, user: Klaus Mueller Maria Lopez is heading off to college., user: Klaus Mueller and Maria Lopez have a close relationship, user: Klaus Mueller is actively researching a topic, user: Klaus Mueller is close friends and classmates with Maria Lopez., user: Klaus Mueller is socially active, user: Klaus Mueller has a crush on Maria Lopez., user: Klaus Mueller and Maria Lopez have been friends for a long time, user: Klaus Mueller is academically inclined, user: For Klaus Mueller's planning: should remember to ask Maria Lopez about her research paper, as she found it interesting that he mentioned it., user: Klaus Mueller is acquainted with Isabella Rodriguez, user: Klaus Mueller is organized and prepared, user: Maria Lopez is conversing about conversing about Maria's research paper mentioned by Klaus, user: Klaus Mueller is conversing about conversing about Maria's research paper mentioned by Klaus, user: Klaus Mueller is a student, user: Klaus Mueller is a student, user: Klaus Mueller is conversing about two friends named Klaus Mueller and Maria Lopez discussing their morning plans and progress on a research paper before Maria heads off to college., user: Klaus Mueller is socially active, user: Klaus Mueller is socially active, user: Klaus Mueller is socially active and acquainted with Isabella Rodriguez, user: Klaus Mueller has a crush on Maria Lopez]\n\nWhat 5 high-level insights can you infer from the above statements? Please ensure it includes 'because of' and generates according to the example format.(example format: insight (because of 1, 5, 3)) .\n1.": "It seems that Klaus Mueller has a strong and close relationship with Maria Lopez because they have been friends for more than 2 years, are dormmates, and have a strong bond.\n2. Klaus Mueller is academically inclined and actively researching a topic because he is conversing about Maria's research paper and is organized and prepared.\n3. Klaus Mueller has a crush on Maria Lopez, which is evident from his repeated mention of their close relationship and his interest in her research paper.\n4. Klaus Mueller is socially active and acquainted with Isabella Rodriguez, as he is conversing about two friends discussing their morning plans and progress on a research paper before Maria heads off to college.\n5. Maria Lopez and Klaus Mueller are conversing about their morning plans and", + "Task: Turn the input into (subject, predicate, object). \n\nInput: Sam Johnson is eating breakfast. \nOutput: (Dolores Murphy, eat, breakfast) \n--- \nInput: Joon Park is brewing coffee.\nOutput: (Joon Park, brew, coffee)\n---\nInput: Jane Cook is sleeping. \nOutput: (Jane Cook, is, sleep)\n---\nInput: Michael Bernstein is writing email on a computer. \nOutput: (Michael Bernstein, write, email)\n---\nInput: Percy Liang is teaching students in a classroom. \nOutput: (Percy Liang, teach, students)\n---\nInput: Merrie Morris is running on a treadmill. \nOutput: (Merrie Morris, run, treadmill)\n---\nInput: Klaus Mueller is Klaus Mueller is academically inclined. \nOutput: (Klaus Mueller,": "I'm sorry, but the input \"Klaus Mueller is Klaus Mueller is academically inclined.\" seems to be incomplete or incorrect. Could you please", + "\"\"\"\nConversation: \nRole_A: \"what's the weather today?\"\nRole_B: \"It looks pretty good, and I will take a walk then.\"\n\n\nSummarize the conversation above in one sentence:\nThis is a conversation about\n\"\"\"\nOutput the response to the prompt above in json. The output must continue the sentence above by filling in the tag. Don't start with 'this is a conversation about...' Just finish the sentence but do not miss any important details (including who are chatting).\nExample output json:\n{\"output\": \"conversing about what to eat for lunch\"}": "{\"output\": \"the weather and plans for a walk\"}", + "\"\"\"\n[Statements]\n\n\nBased on the statements above, summarize Isabella Rodriguez and Klaus Mueller's relationship. What do they feel or know about each other?\n\n\n\"\n\"\"\"\nOutput the response to the prompt above in json. The output should be a string that responds to the question.\nExample output json:\n{\"output\": \"Jane Doe is working on a project\"}": "{\"output\": \"Isabella Rodriguez and Klaus Mueller have a close working relationship and seem to have a good understanding of each other's strengths and weaknesses. They trust each other and are comfortable sharing their thoughts and ideas.\"}", + "Context for the task: \n\nPART 1. \nHere is Here is a brief description of Isabella Rodriguez.\nName: Isabella Rodriguez\nAge: 34\nInnate traits: friendly, outgoing, hospitable\nLearned traits: Isabella Rodriguez is a cafe owner of Hobbs Cafe who loves to make people feel welcome. She is always looking for ways to make the cafe a place where people can come to relax and enjoy themselves.\nCurrently: Isabella Rodriguez is planning on having a Valentine's Day party at Hobbs Cafe with her customers on February 14th, 2023 at 5pm. She is gathering party material, and is telling everyone to join the party at Hobbs Cafe on February 14th, 2023, from 5pm to 7pm.\nLifestyle: Isabella Rodriguez goes to bed around 11pm, awakes up around 6am.\nDaily plan requirement: Isabella Rodriguez opens Hobbs Cafe at 8am everyday, and works at the counter until 8pm, at which point she closes the cafe.\nCurrent Date: \n\n\nHere is the memory that is in Isabella Rodriguez's head: \n\n\nPART 2. \nPast Context: \n\n\nCurrent Location: main room in Isabella Rodriguez's apartment\n\nCurrent Context: \nIsabella Rodriguez was None when Isabella Rodriguez saw Klaus Mueller in the middle of None.\nIsabella Rodriguez is initiating a conversation with Klaus Mueller.\n\nIsabella Rodriguez and Klaus Mueller are chatting. Here is their conversation so far: \n[The conversation has not started yet -- start it!]\n\n---\nTask: Given the above, what should Isabella Rodriguez say to Klaus Mueller next in the conversation? And did it end the conversation?\n\nOutput format: Output a json of the following format: \n{\n\"Isabella Rodriguez\": \"\",\n\"Did the conversation end with Isabella Rodriguez's utterance?\": \"\"\n}": "{\n \"Isabella Rodriguez\": \"Hey Klaus, I'm so glad you stopped by! I'm getting everything ready for our Valentine's Day party at Hobbs Cafe. I hope you can make it on February 14th at 5pm. It's going to be a great time with music, food, and fun activities. I'd love to see you there!\",\n \"Did the conversation end with Isabella Rodriguez's utterance?\": \"false\"\n}", + "\"\"\"\n[Statements]\n\n\nBased on the statements above, summarize Klaus Mueller and Isabella Rodriguez's relationship. What do they feel or know about each other?\n\n\n\"\n\"\"\"\nOutput the response to the prompt above in json. The output should be a string that responds to the question.\nExample output json:\n{\"output\": \"Jane Doe is working on a project\"}": "{\"output\": \"Klaus Mueller and Isabella Rodriguez have a professional relationship, and they seem to have mutual respect for each other's work. They have collaborated on several projects and have a good understanding of each other's expertise.\"}", + "Context for the task: \n\nPART 1. \nHere is Here is a brief description of Klaus Mueller.\nName: Klaus Mueller\nAge: 20\nInnate traits: kind, inquisitive, passionate\nLearned traits: Klaus Mueller is a student at Oak Hill College studying sociology. He is passionate about social justice and loves to explore different perspectives.\nCurrently: Klaus Mueller is writing a research paper on the effects of gentrification in low-income communities.\nLifestyle: Klaus Mueller goes to bed around 11pm, awakes up around 7am, eats dinner around 5pm.\nDaily plan requirement: Klaus Mueller goes to the library at Oak Hill College early in the morning, spends his days writing, and eats at Hobbs Cafe.\nCurrent Date: \n\n\nHere is the memory that is in Klaus Mueller's head: \n\n\nPART 2. \nPast Context: \n\n\nCurrent Location: Klaus Mueller's room in Dorm for Oak Hill College\n\nCurrent Context: \nKlaus Mueller was None when Klaus Mueller saw Isabella Rodriguez in the middle of None.\nKlaus Mueller is initiating a conversation with Isabella Rodriguez.\n\nKlaus Mueller and Isabella Rodriguez are chatting. Here is their conversation so far: \nIsabella Rodriguez: Hey Klaus, I'm so glad you stopped by! I'm getting everything ready for our Valentine's Day party at Hobbs Cafe. I hope you can make it on February 14th at 5pm. It's going to be a great time with music, food, and fun activities. I'd love to see you there!\n\n\n---\nTask: Given the above, what should Klaus Mueller say to Isabella Rodriguez next in the conversation? And did it end the conversation?\n\nOutput format: Output a json of the following format: \n{\n\"Klaus Mueller\": \"\",\n\"Did the conversation end with Klaus Mueller's utterance?\": \"\"\n}": "{\n \"Klaus Mueller\": \"Hey Isabella, that sounds like a fantastic event! I'll definitely be there on February 14th at 5pm. Thanks for inviting me!\",\n \"Did the conversation end with Klaus Mueller's utterance?\": \"false\"\n}", + "Context for the task: \n\nPART 1. \nHere is Here is a brief description of Isabella Rodriguez.\nName: Isabella Rodriguez\nAge: 34\nInnate traits: friendly, outgoing, hospitable\nLearned traits: Isabella Rodriguez is a cafe owner of Hobbs Cafe who loves to make people feel welcome. She is always looking for ways to make the cafe a place where people can come to relax and enjoy themselves.\nCurrently: Isabella Rodriguez is planning on having a Valentine's Day party at Hobbs Cafe with her customers on February 14th, 2023 at 5pm. She is gathering party material, and is telling everyone to join the party at Hobbs Cafe on February 14th, 2023, from 5pm to 7pm.\nLifestyle: Isabella Rodriguez goes to bed around 11pm, awakes up around 6am.\nDaily plan requirement: Isabella Rodriguez opens Hobbs Cafe at 8am everyday, and works at the counter until 8pm, at which point she closes the cafe.\nCurrent Date: \n\n\nHere is the memory that is in Isabella Rodriguez's head: \n\n\nPART 2. \nPast Context: \n\n\nCurrent Location: main room in Isabella Rodriguez's apartment\n\nCurrent Context: \nIsabella Rodriguez was None when Isabella Rodriguez saw Klaus Mueller in the middle of None.\nIsabella Rodriguez is initiating a conversation with Klaus Mueller.\n\nIsabella Rodriguez and Klaus Mueller are chatting. Here is their conversation so far: \nIsabella Rodriguez: Hey Klaus, I'm so glad you stopped by! I'm getting everything ready for our Valentine's Day party at Hobbs Cafe. I hope you can make it on February 14th at 5pm. It's going to be a great time with music, food, and fun activities. I'd love to see you there!\nKlaus Mueller: Hey Isabella, that sounds like a fantastic event! I'll definitely be there on February 14th at 5pm. Thanks for inviting me!\n\n\n---\nTask: Given the above, what should Isabella Rodriguez say to Klaus Mueller next in the conversation? And did it end the conversation?\n\nOutput format: Output a json of the following format: \n{\n\"Isabella Rodriguez\": \"\",\n\"Did the conversation end with Isabella Rodriguez's utterance?\": \"\"\n}": "{\n \"Isabella Rodriguez\": \"Great! I'm looking forward to seeing you there. I'll make sure to save you a seat. Thanks for stopping by, Klaus!\",\n \"Did the conversation end with Isabella Rodriguez's utterance?\": \"false\"\n}", + "Context for the task: \n\nPART 1. \nHere is Here is a brief description of Klaus Mueller.\nName: Klaus Mueller\nAge: 20\nInnate traits: kind, inquisitive, passionate\nLearned traits: Klaus Mueller is a student at Oak Hill College studying sociology. He is passionate about social justice and loves to explore different perspectives.\nCurrently: Klaus Mueller is writing a research paper on the effects of gentrification in low-income communities.\nLifestyle: Klaus Mueller goes to bed around 11pm, awakes up around 7am, eats dinner around 5pm.\nDaily plan requirement: Klaus Mueller goes to the library at Oak Hill College early in the morning, spends his days writing, and eats at Hobbs Cafe.\nCurrent Date: \n\n\nHere is the memory that is in Klaus Mueller's head: \n\n\nPART 2. \nPast Context: \n\n\nCurrent Location: Klaus Mueller's room in Dorm for Oak Hill College\n\nCurrent Context: \nKlaus Mueller was None when Klaus Mueller saw Isabella Rodriguez in the middle of None.\nKlaus Mueller is initiating a conversation with Isabella Rodriguez.\n\nKlaus Mueller and Isabella Rodriguez are chatting. Here is their conversation so far: \nIsabella Rodriguez: Hey Klaus, I'm so glad you stopped by! I'm getting everything ready for our Valentine's Day party at Hobbs Cafe. I hope you can make it on February 14th at 5pm. It's going to be a great time with music, food, and fun activities. I'd love to see you there!\nKlaus Mueller: Hey Isabella, that sounds like a fantastic event! I'll definitely be there on February 14th at 5pm. Thanks for inviting me!\nIsabella Rodriguez: Great! I'm looking forward to seeing you there. I'll make sure to save you a seat. Thanks for stopping by, Klaus!\n\n\n---\nTask: Given the above, what should Klaus Mueller say to Isabella Rodriguez next in the conversation? And did it end the conversation?\n\nOutput format: Output a json of the following format: \n{\n\"Klaus Mueller\": \"\",\n\"Did the conversation end with Klaus Mueller's utterance?\": \"\"\n}": "{\n \"Klaus Mueller\": \"Isabella, I'm really interested in your work with the community. I'd love to hear more about it sometime.\",\n \"Did the conversation end with Klaus Mueller's utterance?\": \"false\"\n}", + "Task -- choose an appropriate area from the area options for a task at hand. \n\nSam Kim lives in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen.\nSam Kim is currently in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. \nArea options: {Sam Kim's house, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}.\n* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place.\n* Must be one of the \"Area options,\" verbatim.\nFor taking a walk, Sam Kim should go to the following area: {Johnson Park}\n---\nJane Anderson lives in {Oak Hill College Student Dormatory} that has Jane Anderson's room.\nJane Anderson is currently in {Oak Hill College} that has a classroom, library\nArea options: {Oak Hill College Student Dormatory, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. \n* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place.\n* Must be one of the \"Area options,\" verbatim.\nFor eating dinner, Jane Anderson should go to the following area: {Hobbs Cafe}\n---\nHobbs Cafe lives in {Klaus Mueller} that has sleeping.\nsleeping is currently in {Klaus Mueller} that has !!. !!\nArea options: {!!}. \n* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place.\n* Must be one of the \"Area options,\" verbatim.\n!! is !!. For !!, !! should go to the following area: {": "For sleeping, Hobbs Cafe should go to the following area: {K", + "Jane Anderson is in kitchen in Jane Anderson's house.\nJane Anderson is going to Jane Anderson's house that has the following areas: {kitchen, bedroom, bathroom}\nStay in the current area if the activity can be done there. Never go into other people's rooms unless necessary.\nFor cooking, Jane Anderson should go to the following area in Jane Anderson's house:\nAnswer: {kitchen}\n---\nTom Watson is in common room in Tom Watson's apartment. \nTom Watson is going to Hobbs Cafe that has the following areas: {cafe}\nStay in the current area if the activity can be done there. Never go into other people's rooms unless necessary.\nFor getting coffee, Tom Watson should go to the following area in Hobbs Cafe:\nAnswer: {cafe}\n---\n\nKlaus Mueller is going to Dorm for Oak Hill College that has the following areas: {garden, Klaus Mueller's room, woman's bathroom, common room, man's bathroom}\n* Stay in the current area if the activity can be done there. \n* NEVER go into other people's rooms unless necessary.\nKlaus Mueller is sleeping. For sleeping, Klaus Mueller should go to the following area in Dorm for Oak Hill College (MUST pick one of {garden, Klaus Mueller's room, woman's bathroom, common room, man's bathroom}):\nAnswer: {": "Klaus Mueller's room}", + "Current activity: sleep in bed\nObjects available: {bed, easel, closet, painting}\nPick ONE most relevant object from the objects available: bed\n---\nCurrent activity: painting\nObjects available: {easel, closet, sink, microwave}\nPick ONE most relevant object from the objects available: easel\n---\nCurrent activity: cooking\nObjects available: {stove, sink, fridge, counter}\nPick ONE most relevant object from the objects available: stove\n---\nCurrent activity: watch TV\nObjects available: {couch, TV, remote, coffee table}\nPick ONE most relevant object from the objects available: TV\n---\nCurrent activity: study\nObjects available: {desk, computer, chair, bookshelf}\nPick ONE most relevant object from the objects available: desk\n---\nCurrent activity: talk on the phone\nObjects available: {phone, charger, bed, nightstand}\nPick ONE most relevant object from the objects available: phone\n---\nCurrent activity: sleeping\nObjects available: {bed, game console, closet, desk}\nPick ONE most relevant object from the objects available:": "bed", + "\"\"\"\nTask: We want to understand the state of an object that is being used by someone. \n\nLet's think step by step. \nWe want to know about bed's state. \nStep 1. Klaus Mueller is at/using the sleeping.\nStep 2. Describe the bed's state: bed is\n\"\"\"\nOutput the response to the prompt above in json. The output should ONLY contain the phrase that should go in .\nExample output json:\n{\"output\": \"being fixed\"}": "{\n \"output\": \"being used\"\n}", + "\"\"\"\nConvert an action description to an emoji (important: use two or less emojis).\n\nAction description: sleeping\nEmoji:\n\"\"\"\nOutput the response to the prompt above in json. The value for the output must ONLY contain the emojis.\nExample output json:\n{\"output\": \"🛁🧖‍♀️\"}": "{\n \"output\": \"😴\"\n}", + "Task: Turn the input into (subject, predicate, object). \n\nInput: Sam Johnson is eating breakfast. \nOutput: (Dolores Murphy, eat, breakfast) \n--- \nInput: Joon Park is brewing coffee.\nOutput: (Joon Park, brew, coffee)\n---\nInput: Jane Cook is sleeping. \nOutput: (Jane Cook, is, sleep)\n---\nInput: Michael Bernstein is writing email on a computer. \nOutput: (Michael Bernstein, write, email)\n---\nInput: Percy Liang is teaching students in a classroom. \nOutput: (Percy Liang, teach, students)\n---\nInput: Merrie Morris is running on a treadmill. \nOutput: (Merrie Morris, run, treadmill)\n---\nInput: Klaus Mueller is sleeping. \nOutput: (Klaus Mueller,": "I'm sorry, but it seems like the input was cut off. Could you please provide the complete input so I can help you with the output?", + "\"\"\"\nConvert an action description to an emoji (important: use two or less emojis).\n\nAction description: being used\nEmoji:\n\"\"\"\nOutput the response to the prompt above in json. The value for the output must ONLY contain the emojis.\nExample output json:\n{\"output\": \"🛁🧖‍♀️\"}": "{\n \"output\": \"🔨\"\n}", + "Task: Turn the input into (subject, predicate, object). \n\nInput: Sam Johnson is eating breakfast. \nOutput: (Dolores Murphy, eat, breakfast) \n--- \nInput: Joon Park is brewing coffee.\nOutput: (Joon Park, brew, coffee)\n---\nInput: Jane Cook is sleeping. \nOutput: (Jane Cook, is, sleep)\n---\nInput: Michael Bernstein is writing email on a computer. \nOutput: (Michael Bernstein, write, email)\n---\nInput: Percy Liang is teaching students in a classroom. \nOutput: (Percy Liang, teach, students)\n---\nInput: Merrie Morris is running on a treadmill. \nOutput: (Merrie Morris, run, treadmill)\n---\nInput: bed is being used. \nOutput: (bed,": "I'm sorry, but the input \"bed is being used\" does not follow the format of the previous examples where a subject, predicate, and object" } \ No newline at end of file diff --git a/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py b/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py index 81c811d67..282a45dfa 100644 --- a/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py +++ b/tests/metagpt/environment/stanford_town_env/test_stanford_town_ext_env.py @@ -15,7 +15,7 @@ from metagpt.environment.stanford_town.stanford_town_ext_env import StanfordTown maze_asset_path = ( Path(__file__) .absolute() - .parent.joinpath("..", "..", "..", "..", "examples/stanford_town/static_dirs/assets/the_ville") + .parent.joinpath("..", "..", "..", "..", "metagpt/ext/stanford_town/static_dirs/assets/the_ville") ) diff --git a/tests/metagpt/ext/stanford_town/actions/test_gen_action_details.py b/tests/metagpt/ext/stanford_town/actions/test_gen_action_details.py index 5dc5cbbe3..616c03f33 100644 --- a/tests/metagpt/ext/stanford_town/actions/test_gen_action_details.py +++ b/tests/metagpt/ext/stanford_town/actions/test_gen_action_details.py @@ -40,7 +40,6 @@ async def test_gen_action_details(): sector = await GenActionSector().run(role, access_tile, act_desp) arena = await GenActionArena().run(role, act_desp, act_world, sector) temp_address = f"{act_world}:{sector}:{arena}" - print(temp_address) obj = await GenActionObject().run(role, act_desp, temp_address) act_obj_desp = await GenActObjDescription().run(role, obj, act_desp) diff --git a/tests/metagpt/ext/stanford_town/memory/test_agent_memory.py b/tests/metagpt/ext/stanford_town/memory/test_agent_memory.py index d17014898..db7ca3212 100644 --- a/tests/metagpt/ext/stanford_town/memory/test_agent_memory.py +++ b/tests/metagpt/ext/stanford_town/memory/test_agent_memory.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta import pytest -from metagpt.ext.stanford_town.memory import AgentMemory +from metagpt.ext.stanford_town.memory.agent_memory import AgentMemory from metagpt.ext.stanford_town.memory.retrieve import agent_retrieve from metagpt.ext.stanford_town.utils.const import STORAGE_PATH from metagpt.logs import logger diff --git a/tests/metagpt/ext/stanford_town/plan/test_st_plan.py b/tests/metagpt/ext/stanford_town/plan/test_st_plan.py index 18663a066..f7f395040 100644 --- a/tests/metagpt/ext/stanford_town/plan/test_st_plan.py +++ b/tests/metagpt/ext/stanford_town/plan/test_st_plan.py @@ -2,39 +2,24 @@ # -*- coding: utf-8 -*- # @Desc : unittest of st_plan + import pytest -from metagpt.ext.stanford_town.plan.st_plan import ( - _choose_retrieved, - _should_react, - _wait_react, -) +from metagpt.ext.stanford_town.plan.st_plan import _choose_retrieved, _should_react from tests.metagpt.ext.stanford_town.plan.test_conversation import init_two_roles -def test_should_react(): - role_ir, role_km = init_two_roles() +@pytest.mark.asyncio +async def test_should_react(): + role_ir, role_km = await init_two_roles() roles = {role_ir.name: role_ir, role_km.name: role_km} + role_ir.scratch.act_address = "mock data" - observed = role_ir.observe() + observed = await role_ir.observe() retrieved = role_ir.retrieve(observed) focused_event = _choose_retrieved(role_ir.name, retrieved) if focused_event: - reaction_mode = _should_react(role_ir, focused_event, roles) # chat with Isabella Rodriguez - assert "chat with" in reaction_mode - - -@pytest.mark.asyncio -async def test_wait_react(): - role_ir, role_km = init_two_roles("base_the_ville_isabella_maria_klaus") - reaction_mode = "wait: February 13, 2023, 00:01:30" - f_daily_schedule = role_ir.scratch.f_daily_schedule - # [['sleeping', 360], ['waking up and completing her morning routine (getting out of bed)', 5], ['sleeping', 180]] - - await _wait_react(role_ir, reaction_mode) - new_f_daily_schedule = role_ir.scratch.f_daily_schedule - # [['sleeping', 360], ['waking up and completing her morning routine (getting out of bed)', 5], - # ['waking up and completing her morning routine (brushing her teeth)', 5], ['sleeping', 180]] - assert len(f_daily_schedule) == len(new_f_daily_schedule) + reaction_mode = await _should_react(role_ir, focused_event, roles) # chat with Isabella Rodriguez + assert not reaction_mode From 6c40b5ccae42d5dda09f2503f3211ef3db0744d2 Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 28 Mar 2024 18:24:14 +0800 Subject: [PATCH 081/127] pypi with non-py files --- MANIFEST.in | 3 +++ setup.py | 1 + 2 files changed, 4 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..292433f80 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +recursive-include metagpt/ext/stanford_town/prompts *.txt +recursive-include metagpt/ext/stanford_town/static_dirs *.csv +recursive-include metagpt/ext/stanford_town/static_dirs *.json \ No newline at end of file diff --git a/setup.py b/setup.py index f834b4c44..06274ee6d 100644 --- a/setup.py +++ b/setup.py @@ -87,4 +87,5 @@ setup( "metagpt=metagpt.software_company:app", ], }, + include_package_data=True, ) From bd174489ac67267ac839afaffdf48d4e3296fbdc Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 28 Mar 2024 18:29:59 +0800 Subject: [PATCH 082/127] update readme --- metagpt/ext/stanford_town/README.md | 6 +++--- metagpt/ext/stanford_town/README_CN.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metagpt/ext/stanford_town/README.md b/metagpt/ext/stanford_town/README.md index 915c61356..88ed4b782 100644 --- a/metagpt/ext/stanford_town/README.md +++ b/metagpt/ext/stanford_town/README.md @@ -1,15 +1,15 @@ ## Stanford Town Game ### Pre-Description -The path configured in `examples/stanford_town/utils/const.py` is the storage path of the current project. In order to facilitate GA( [generative_agents](https://github.com/joonspk-research/generative_agents) )'s frontend docking data (to avoid changing its code), you can set the value `temp_storage_path` to `temp_storage` of `generative_agents` when start `run_st_game.py`. like +In order to facilitate GA( [generative_agents](https://github.com/joonspk-research/generative_agents) )'s frontend docking data (to avoid changing its code), you can set the value `temp_storage_path` to `temp_storage` of `generative_agents` when start `run_st_game.py`. like `python3 run_st_game.py --temp_storage_path path/to/ga/temp_storage xxx` Or change the path under `const.py` like beflow ``` -STORAGE_PATH = ROOT_PATH.joinpath("storage") -TEMP_STORAGE_PATH = ROOT_PATH.joinpath("temp_storage") +STORAGE_PATH = EXAMPLE_PATH.joinpath("storage") +TEMP_STORAGE_PATH = EXAMPLE_PATH.joinpath("temp_storage") # updated STORAGE_PATH = Path("{path/to/ga/storage}") TEMP_STORAGE_PATH = Path("{path/to/ga/temp_storage}") diff --git a/metagpt/ext/stanford_town/README_CN.md b/metagpt/ext/stanford_town/README_CN.md index 445a5b1b3..c55a742fc 100644 --- a/metagpt/ext/stanford_town/README_CN.md +++ b/metagpt/ext/stanford_town/README_CN.md @@ -1,15 +1,15 @@ ## Stanford Town Game ### 前置 -`examples/stanford_town/utils/const.py`配置的路径为当前项目的存储路径，为了方便GA（ [generative_agents](https://github.com/joonspk-research/generative_agents) ）的前端对接数据（避免改动它那块的代码），可在启动`run_st_game.py`加上`temp_storage_path`指向`generative_agents`对应的`temp_storage`路径。比如 +为了方便GA（ [generative_agents](https://github.com/joonspk-research/generative_agents) ）的前端对接数据（避免改动它那块的代码），可在启动`run_st_game.py`加上`temp_storage_path`指向`generative_agents`对应的`temp_storage`路径。比如 `python3 run_st_game.py --temp_storage_path path/to/ga/temp_storage xxx` 或将`const.py`下的 ``` -STORAGE_PATH = ROOT_PATH.joinpath("storage") -TEMP_STORAGE_PATH = ROOT_PATH.joinpath("temp_storage") +STORAGE_PATH = EXAMPLE_PATH.joinpath("storage") +TEMP_STORAGE_PATH = EXAMPLE_PATH.joinpath("temp_storage") # 更新为 STORAGE_PATH = Path("{path/to/ga/storage}") TEMP_STORAGE_PATH = Path("{path/to/ga/temp_storage}") From 325550a3dc344e4236e34bfeec52164fe084eadc Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 26 Feb 2024 20:02:16 +0800 Subject: [PATCH 083/127] update st_game using MG v0.7 --- examples/st_game/.gitignore | 2 + examples/st_game/README.md | 26 + examples/st_game/README_CN.md | 25 + examples/st_game/__init__.py | 3 + examples/st_game/actions/__init__.py | 3 + .../st_game/actions/agent_chat_sum_rel.py | 39 + examples/st_game/actions/decide_to_talk.py | 97 +++ examples/st_game/actions/dummy_action.py | 23 + .../st_game/actions/gen_action_details.py | 403 ++++++++++ .../st_game/actions/gen_daily_schedule.py | 61 ++ .../st_game/actions/gen_hourly_schedule.py | 181 +++++ examples/st_game/actions/gen_iter_chat_utt.py | 125 +++ .../st_game/actions/inner_voice_action.py | 31 + .../st_game/actions/new_decomp_schedule.py | 154 ++++ .../st_game/actions/run_reflect_action.py | 277 +++++++ examples/st_game/actions/st_action.py | 119 +++ examples/st_game/actions/summarize_conv.py | 47 ++ examples/st_game/actions/task_decomp.py | 175 +++++ examples/st_game/actions/wake_up.py | 43 ++ examples/st_game/memory/__init__.py | 0 examples/st_game/memory/agent_memory.py | 374 +++++++++ examples/st_game/memory/retrieve.py | 180 +++++ examples/st_game/memory/scratch.py | 397 ++++++++++ examples/st_game/memory/spatial_memory.py | 115 +++ examples/st_game/plan/__init__.py | 3 + examples/st_game/plan/converse.py | 93 +++ examples/st_game/plan/st_plan.py | 719 ++++++++++++++++++ examples/st_game/prompts/__init__.py | 3 + .../prompts/action_location_object_vMar11.txt | 30 + .../prompts/action_location_sector_v1.txt | 34 + examples/st_game/prompts/action_object_v2.txt | 32 + .../st_game/prompts/daily_planning_v6.txt | 14 + .../st_game/prompts/decide_to_talk_v2.txt | 18 + .../prompts/generate_event_triple_v1.txt | 30 + .../st_game/prompts/generate_focal_pt_v1.txt | 11 + .../prompts/generate_hourly_schedule_v2.txt | 18 + .../st_game/prompts/generate_obj_event_v1.txt | 16 + .../prompts/generate_pronunciatio_v1.txt | 10 + .../prompts/insight_and_evidence_v1.txt | 12 + .../st_game/prompts/iterative_convo_v1.txt | 46 ++ examples/st_game/prompts/memo_on_convo_v1.txt | 15 + .../prompts/new_decomp_schedule_v1.txt | 24 + .../prompts/planning_thought_on_convo_v1.txt | 15 + .../st_game/prompts/poignancy_action_v1.txt | 15 + .../st_game/prompts/poignancy_chat_v1.txt | 17 + .../st_game/prompts/poignancy_event_v1.txt | 15 + .../st_game/prompts/poignancy_thought_v1.txt | 15 + .../summarize_chat_relationship_v2.txt | 15 + .../prompts/summarize_conversation_v1.txt | 11 + examples/st_game/prompts/task_decomp_v3.txt | 39 + examples/st_game/prompts/wake_up_hour_v1.txt | 12 + .../prompts/whisper_inner_thought_v1.txt | 11 + examples/st_game/reflect/__init__.py | 3 + examples/st_game/reflect/reflect.py | 245 ++++++ examples/st_game/requirements.txt | 0 examples/st_game/roles/__init__.py | 3 + examples/st_game/roles/st_role.py | 627 +++++++++++++++ examples/st_game/run_st_game.py | 75 ++ examples/st_game/stanford_town.py | 48 ++ .../the_ville/agent_history_init_n25.csv | 26 + .../the_ville/agent_history_init_n3.csv | 4 + .../the_ville/matrix/maze/arena_maze.csv | 0 .../the_ville/matrix/maze/collision_maze.csv | 0 .../matrix/maze/game_object_maze.csv | 0 .../the_ville/matrix/maze/sector_maze.csv | 0 .../matrix/maze/spawning_location_maze.csv | 0 .../the_ville/matrix/maze_meta_info.json | 0 .../matrix/special_blocks/arena_blocks.csv | 0 .../special_blocks/game_object_blocks.csv | 0 .../matrix/special_blocks/sector_blocks.csv | 0 .../spawning_location_blocks.csv | 0 .../matrix/special_blocks/world_blocks.csv | 0 examples/st_game/storage/.gitignore | 2 + .../environment/0.json | 26 + .../associative_memory/embeddings.json | 1 + .../associative_memory/kw_strength.json | 2 + .../associative_memory/nodes.json | 1 + .../bootstrap_memory/scratch.json | 51 ++ .../bootstrap_memory/spatial_memory.json | 66 ++ .../test_memory/embeddings.json | 1 + .../test_memory/kw_strength.json | 4 + .../bootstrap_memory/test_memory/nodes.json | 1 + .../associative_memory/embeddings.json | 1 + .../associative_memory/kw_strength.json | 2 + .../associative_memory/nodes.json | 1 + .../bootstrap_memory/scratch.json | 51 ++ .../bootstrap_memory/spatial_memory.json | 86 +++ .../associative_memory/embeddings.json | 1 + .../associative_memory/kw_strength.json | 2 + .../associative_memory/nodes.json | 1 + .../Maria Lopez/bootstrap_memory/scratch.json | 51 ++ .../bootstrap_memory/spatial_memory.json | 87 +++ .../reverie/meta.json | 13 + examples/st_game/tests/__init__.py | 0 examples/st_game/tests/actions/__init__.py | 3 + .../tests/actions/test_gen_action_details.py | 78 ++ .../tests/actions/test_summarize_conv.py | 15 + examples/st_game/tests/plan/__init__.py | 3 + .../st_game/tests/plan/test_conversation.py | 66 ++ examples/st_game/tests/plan/test_st_plan.py | 36 + examples/st_game/tests/roles/__init__.py | 3 + examples/st_game/tests/roles/test_st_role.py | 25 + examples/st_game/tests/test_agent_memory.py | 89 +++ examples/st_game/tests/test_basic_memory.py | 76 ++ examples/st_game/tests/test_reflect.py | 47 ++ examples/st_game/tests/test_spatial_memory.py | 17 + examples/st_game/utils/__init__.py | 3 + examples/st_game/utils/const.py | 13 + examples/st_game/utils/mg_ga_transform.py | 61 ++ examples/st_game/utils/utils.py | 225 ++++++ metagpt/environment/README.md | 2 +- .../test_stanford_town_ext_env.py | 2 +- 112 files changed, 6437 insertions(+), 2 deletions(-) create mode 100644 examples/st_game/.gitignore create mode 100644 examples/st_game/README.md create mode 100644 examples/st_game/README_CN.md create mode 100644 examples/st_game/__init__.py create mode 100644 examples/st_game/actions/__init__.py create mode 100644 examples/st_game/actions/agent_chat_sum_rel.py create mode 100644 examples/st_game/actions/decide_to_talk.py create mode 100644 examples/st_game/actions/dummy_action.py create mode 100644 examples/st_game/actions/gen_action_details.py create mode 100644 examples/st_game/actions/gen_daily_schedule.py create mode 100644 examples/st_game/actions/gen_hourly_schedule.py create mode 100644 examples/st_game/actions/gen_iter_chat_utt.py create mode 100644 examples/st_game/actions/inner_voice_action.py create mode 100644 examples/st_game/actions/new_decomp_schedule.py create mode 100644 examples/st_game/actions/run_reflect_action.py create mode 100644 examples/st_game/actions/st_action.py create mode 100644 examples/st_game/actions/summarize_conv.py create mode 100644 examples/st_game/actions/task_decomp.py create mode 100644 examples/st_game/actions/wake_up.py create mode 100644 examples/st_game/memory/__init__.py create mode 100644 examples/st_game/memory/agent_memory.py create mode 100644 examples/st_game/memory/retrieve.py create mode 100644 examples/st_game/memory/scratch.py create mode 100644 examples/st_game/memory/spatial_memory.py create mode 100644 examples/st_game/plan/__init__.py create mode 100644 examples/st_game/plan/converse.py create mode 100644 examples/st_game/plan/st_plan.py create mode 100644 examples/st_game/prompts/__init__.py create mode 100644 examples/st_game/prompts/action_location_object_vMar11.txt create mode 100644 examples/st_game/prompts/action_location_sector_v1.txt create mode 100644 examples/st_game/prompts/action_object_v2.txt create mode 100644 examples/st_game/prompts/daily_planning_v6.txt create mode 100644 examples/st_game/prompts/decide_to_talk_v2.txt create mode 100644 examples/st_game/prompts/generate_event_triple_v1.txt create mode 100644 examples/st_game/prompts/generate_focal_pt_v1.txt create mode 100644 examples/st_game/prompts/generate_hourly_schedule_v2.txt create mode 100644 examples/st_game/prompts/generate_obj_event_v1.txt create mode 100644 examples/st_game/prompts/generate_pronunciatio_v1.txt create mode 100644 examples/st_game/prompts/insight_and_evidence_v1.txt create mode 100644 examples/st_game/prompts/iterative_convo_v1.txt create mode 100644 examples/st_game/prompts/memo_on_convo_v1.txt create mode 100644 examples/st_game/prompts/new_decomp_schedule_v1.txt create mode 100644 examples/st_game/prompts/planning_thought_on_convo_v1.txt create mode 100644 examples/st_game/prompts/poignancy_action_v1.txt create mode 100644 examples/st_game/prompts/poignancy_chat_v1.txt create mode 100644 examples/st_game/prompts/poignancy_event_v1.txt create mode 100644 examples/st_game/prompts/poignancy_thought_v1.txt create mode 100644 examples/st_game/prompts/summarize_chat_relationship_v2.txt create mode 100644 examples/st_game/prompts/summarize_conversation_v1.txt create mode 100644 examples/st_game/prompts/task_decomp_v3.txt create mode 100644 examples/st_game/prompts/wake_up_hour_v1.txt create mode 100644 examples/st_game/prompts/whisper_inner_thought_v1.txt create mode 100644 examples/st_game/reflect/__init__.py create mode 100644 examples/st_game/reflect/reflect.py create mode 100644 examples/st_game/requirements.txt create mode 100644 examples/st_game/roles/__init__.py create mode 100644 examples/st_game/roles/st_role.py create mode 100644 examples/st_game/run_st_game.py create mode 100644 examples/st_game/stanford_town.py create mode 100644 examples/st_game/static_dirs/assets/the_ville/agent_history_init_n25.csv create mode 100644 examples/st_game/static_dirs/assets/the_ville/agent_history_init_n3.csv rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze/arena_maze.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze/collision_maze.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze/game_object_maze.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze/sector_maze.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze/spawning_location_maze.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/maze_meta_info.json (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/special_blocks/arena_blocks.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/special_blocks/game_object_blocks.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/special_blocks/sector_blocks.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/special_blocks/spawning_location_blocks.csv (100%) rename {tests/data/environment/stanford_town => examples/st_game/static_dirs/assets}/the_ville/matrix/special_blocks/world_blocks.csv (100%) create mode 100644 examples/st_game/storage/.gitignore create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/environment/0.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/embeddings.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/kw_strength.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/associative_memory/nodes.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/scratch.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/spatial_memory.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/embeddings.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/kw_strength.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Isabella Rodriguez/bootstrap_memory/test_memory/nodes.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/embeddings.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/kw_strength.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/associative_memory/nodes.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/scratch.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Klaus Mueller/bootstrap_memory/spatial_memory.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/embeddings.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/kw_strength.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/associative_memory/nodes.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/scratch.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/personas/Maria Lopez/bootstrap_memory/spatial_memory.json create mode 100644 examples/st_game/storage/base_the_ville_isabella_maria_klaus/reverie/meta.json create mode 100644 examples/st_game/tests/__init__.py create mode 100644 examples/st_game/tests/actions/__init__.py create mode 100644 examples/st_game/tests/actions/test_gen_action_details.py create mode 100644 examples/st_game/tests/actions/test_summarize_conv.py create mode 100644 examples/st_game/tests/plan/__init__.py create mode 100644 examples/st_game/tests/plan/test_conversation.py create mode 100644 examples/st_game/tests/plan/test_st_plan.py create mode 100644 examples/st_game/tests/roles/__init__.py create mode 100644 examples/st_game/tests/roles/test_st_role.py create mode 100644 examples/st_game/tests/test_agent_memory.py create mode 100644 examples/st_game/tests/test_basic_memory.py create mode 100644 examples/st_game/tests/test_reflect.py create mode 100644 examples/st_game/tests/test_spatial_memory.py create mode 100644 examples/st_game/utils/__init__.py create mode 100644 examples/st_game/utils/const.py create mode 100644 examples/st_game/utils/mg_ga_transform.py create mode 100644 examples/st_game/utils/utils.py diff --git a/examples/st_game/.gitignore b/examples/st_game/.gitignore new file mode 100644 index 000000000..fe77153f9 --- /dev/null +++ b/examples/st_game/.gitignore @@ -0,0 +1,2 @@ +storage/test* +storage/unittest* \ No newline at end of file diff --git a/examples/st_game/README.md b/examples/st_game/README.md new file mode 100644 index 000000000..cc4ada230 --- /dev/null +++ b/examples/st_game/README.md @@ -0,0 +1,26 @@ +## Stanford Town Game + +### Pre-Description +The path configured in `examples/st_game/utils/const.py` is the storage path of the current project. In order to facilitate GA(generative_agents)'s frontend docking data (to avoid changing its code), you can change the path under `const.py` like beflow + +``` +STORAGE_PATH = ROOT_PATH.joinpath("storage") +TEMP_STORAGE_PATH = ROOT_PATH.joinpath("temp_storage") +# updated +STORAGE_PATH = Path("{path/to/ga/storage}") +TEMP_STORAGE_PATH = Path("{path/to/ga/temp_storage}") +``` + +This can be used to achieve docking of simulation data without changing the GA code. Otherwise, the GA code must be modified to adapt to the MG output path. + +### Backend service startup +The execution entry is `python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` + +`idea` is the user's voice to the first Agent, and it is disseminated through this voice to see whether the final multi-agents achieve the goal of hosting or participating in the event. + +### Frontend service startup +Enter `generative_agents/environment/frontend_server` and use `python manage.py runserver` to start the front-end service. +Visit `http://localhost:8000/simulator_home` to enter the current simulation interface. + +## Appreciation +The reproduction work has referred the `https://github.com/joonspk-research/generative_agents`, let's make a general statement here. diff --git a/examples/st_game/README_CN.md b/examples/st_game/README_CN.md new file mode 100644 index 000000000..9780b3ed8 --- /dev/null +++ b/examples/st_game/README_CN.md @@ -0,0 +1,25 @@ +## Stanford Town Game + +### 前置 +`examples/st_game/utils/const.py`配置的路径为当前项目的存储路径，为了方便GA（generative_agents）的前端对接数据（避免改动它那块的代码），可将`const.py`下的 + +``` +STORAGE_PATH = ROOT_PATH.joinpath("storage") +TEMP_STORAGE_PATH = ROOT_PATH.joinpath("temp_storage") +# 更新为 +STORAGE_PATH = Path("{path/to/ga/storage}") +TEMP_STORAGE_PATH = Path("{path/to/ga/temp_storage}") +``` +这样可用实现不改变GA代码情况下，实现仿真数据的对接。不然得修改GA的代码来适配MG的输出路径。 + +### 后端服务启动 +执行入口为：`python3 run_st_game.py "Host a open lunch party at 13:00 pm" "base_the_ville_isabella_maria_klaus" "test_sim" 10` + +`idea`为用户给第一个Agent的用户心声，并通过这个心声进行传播，看最后多智能体是否达到举办、参加活动的目标。 + +### 前端服务启动 +进入`generative_agents/environment/frontend_server`，使用`python manage.py runserver`启动前端服务。 +访问`http://localhost:8000/simulator_home` 进入当前的仿真界面。 + +## Appreciation +The reproduction work has referred the `https://github.com/joonspk-research/generative_agents`, let's make a general statement here. diff --git a/examples/st_game/__init__.py b/examples/st_game/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/examples/st_game/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/examples/st_game/actions/__init__.py b/examples/st_game/actions/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/examples/st_game/actions/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/examples/st_game/actions/agent_chat_sum_rel.py b/examples/st_game/actions/agent_chat_sum_rel.py new file mode 100644 index 000000000..00f15e82f --- /dev/null +++ b/examples/st_game/actions/agent_chat_sum_rel.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : summarize relationship in a agent chat + +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +class AgentChatSumRel(STAction): + name: str = "AgentChatSumRel" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + resp = False + try: + _ = llm_resp.split('"')[0].strip() + resp = True + except Exception: + pass + return resp + + def _func_cleanup(self, llm_resp: str, prompt: str) -> str: + return llm_resp.split('"')[0].strip() + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, init_role: "STRole", target_role: "STRole", statements: str) -> str: + def create_prompt_input(init_role: "STRole", target_role: "STRole", statements: str) -> str: + prompt_input = [statements, init_role.name, target_role.name] + return prompt_input + + prompt_input = create_prompt_input(init_role, target_role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "summarize_chat_relationship_v2.txt") + + example_output = "Jane Doe is working on a project" + special_instruction = "The output should be a string that responds to the question." + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {init_role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/decide_to_talk.py b/examples/st_game/actions/decide_to_talk.py new file mode 100644 index 000000000..cabb5ceff --- /dev/null +++ b/examples/st_game/actions/decide_to_talk.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : device to talk to another role, return yes or no + +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +class DecideToTalk(STAction): + name: str = "DecideToTalk" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + resp = False + try: + if llm_resp.split("Answer in yes or no:")[-1].strip().lower() in ["yes", "no"]: + resp = True + except ValueError: + pass + return resp + + def _func_cleanup(self, llm_resp: str, prompt: str) -> str: + return llm_resp.split("Answer in yes or no:")[-1].strip().lower() + + def _func_fail_default_resp(self) -> str: + return "yes" + + async def run(self, init_role: "STRole", target_role: "STRole", retrieved: dict, *args, **kwargs) -> bool: + """Run action""" + + def create_prompt_input(init_role: "STRole", target_role: "STRole", retrieved: dict) -> str: + scratch = init_role.rc.scratch + target_scratch = target_role.rc.scratch + last_chat = init_role.rc.memory.get_last_chat(target_role.name) + last_chatted_time = "" + last_chat_about = "" + if last_chat: + last_chatted_time = last_chat.created.strftime("%B %d, %Y, %H:%M:%S") + last_chat_about = last_chat.description + + context = "" + for c_node in retrieved["events"]: + curr_desc = c_node.description.split(" ") + curr_desc[2:3] = ["was"] + curr_desc = " ".join(curr_desc) + context += f"{curr_desc}. " + context += "\n" + for c_node in retrieved["thoughts"]: + context += f"{c_node.description}. " + + curr_time = scratch.curr_time.strftime("%B %d, %Y, %H:%M:%S %p") + init_act_desc = scratch.act_description + if "(" in init_act_desc: + init_act_desc = init_act_desc.split("(")[-1][:-1] + + if len(scratch.planned_path) == 0 and "waiting" not in init_act_desc: + init_p_desc = f"{init_role.name} is already {init_act_desc}" + elif "waiting" in init_act_desc: + init_p_desc = f"{init_role.name} is {init_act_desc}" + else: + init_p_desc = f"{init_role.name} is on the way to {init_act_desc}" + + target_act_desc = scratch.act_description + if "(" in target_act_desc: + target_act_desc = target_act_desc.split("(")[-1][:-1] + + if len(target_scratch.planned_path) == 0 and "waiting" not in init_act_desc: + target_p_desc = f"{target_role.name} is already {target_act_desc}" + elif "waiting" in init_act_desc: + target_p_desc = f"{init_role.name} is {init_act_desc}" + else: + target_p_desc = f"{target_role.name} is on the way to {target_act_desc}" + + prompt_input = [] + prompt_input += [context] + + prompt_input += [curr_time] + + prompt_input += [init_role.name] + prompt_input += [target_role.name] + prompt_input += [last_chatted_time] + prompt_input += [last_chat_about] + + prompt_input += [init_p_desc] + prompt_input += [target_p_desc] + prompt_input += [init_role.name] + prompt_input += [target_role.name] + return prompt_input + + prompt_input = create_prompt_input(init_role, target_role, retrieved) + prompt = self.generate_prompt_with_tmpl_filename( + prompt_input=prompt_input, tmpl_filename="decide_to_talk_v2.txt" + ) + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=20) # yes or no + result = True if output == "yes" else False + logger.info(f"Role: {init_role.name} Action: {self.cls_name} output: {result}") + return result diff --git a/examples/st_game/actions/dummy_action.py b/examples/st_game/actions/dummy_action.py new file mode 100644 index 000000000..ba87d8516 --- /dev/null +++ b/examples/st_game/actions/dummy_action.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : dummy action to make every STRole can deal DummyMessage which is caused by DummyAction + +from dataclasses import dataclass + +from metagpt.actions import Action +from metagpt.schema import Message + + +class DummyAction(Action): + async def run(self, *args, **kwargs): + raise NotImplementedError + + +@dataclass +class DummyMessage(Message): + """ + dummy message to pass to role and make them to have a execution every round + """ + + content: str = "dummy" + cause_by: str = "DummyAction" diff --git a/examples/st_game/actions/gen_action_details.py b/examples/st_game/actions/gen_action_details.py new file mode 100644 index 000000000..aacc6852a --- /dev/null +++ b/examples/st_game/actions/gen_action_details.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : gen_action_details + +import random + +from metagpt.environment.api.env_api import EnvAPIAbstract +from metagpt.logs import logger + +from .st_action import STAction + + +class GenActionSector(STAction): + name: str = "GenActionSector" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cleaned_response = llm_resp.split("}")[0] + return cleaned_response + + def _func_validate(self, llm_resp: str, prompt: str): + if len(llm_resp.strip()) < 1: + return False + if "}" not in llm_resp: + return False + if "," in llm_resp: + return False + return True + + def _func_fail_default_resp(self): + fs = "kitchen" + return fs + + async def run(self, role: "STRole", access_tile: dict[str, str], act_desp: str): + def create_prompt_input(role, access_tile: dict[str, str], act_desp): + act_world = f"{access_tile['world']}" + + prompt_input = [] + + prompt_input += [role.scratch.get_str_name()] + prompt_input += [role.scratch.living_area.split(":")[1]] + x = f"{act_world}:{role.scratch.living_area.split(':')[1]}" + prompt_input += [role.s_mem.get_str_accessible_sector_arenas(x)] + + prompt_input += [role.scratch.get_str_name()] + prompt_input += [f"{access_tile['sector']}"] + x = f"{act_world}:{access_tile['sector']}" + prompt_input += [role.s_mem.get_str_accessible_sector_arenas(x)] + + if role.scratch.get_str_daily_plan_req() != "": + prompt_input += [f"\n{role.scratch.get_str_daily_plan_req()}"] + else: + prompt_input += [""] + + # MAR 11 TEMP + prompt_input = [] + act_world = access_tile["world"] + accessible_sector_str = role.s_mem.get_str_accessible_sectors(act_world) + curr = accessible_sector_str.split(", ") + fin_accessible_sectors = [] + for i in curr: + if "'s house" in i: + if role.scratch.last_name in i: + fin_accessible_sectors += [i] + else: + fin_accessible_sectors += [i] + accessible_sector_str = ", ".join(fin_accessible_sectors) + # END MAR 11 TEMP + + prompt_input += [accessible_sector_str] + + act_desp_1 = act_desp + act_desp_2 = act_desp + if "(" in act_desp: + act_desp_1 = act_desp.split("(")[0].strip() + act_desp_2 = act_desp.split("(")[-1][:-1] + prompt_input += [role.scratch.get_str_name()] + prompt_input += [act_desp_1] + + prompt_input += [act_desp_2] + prompt_input += [role.scratch.get_str_name()] + return prompt_input + + prompt_template = "action_location_sector_v1.txt" + prompt_input = create_prompt_input(role, access_tile, act_desp) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=15) + y = f"{access_tile['world']}" + x = [i.strip() for i in role.s_mem.get_str_accessible_sectors(y).split(",")] + if output not in x: + # output = random.choice(x) + output = role.scratch.living_area.split(":")[1] + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenActionArena(STAction): + name: str = "GenActionArena" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cleaned_response = llm_resp.split("}")[0] + return cleaned_response + + def _func_validate(self, llm_resp: str, prompt: str): + if len(llm_resp.strip()) < 1: + return False + if "}" not in llm_resp: + return False + if "," in llm_resp: + return False + return True + + def _func_fail_default_resp(self): + fs = "kitchen" + return fs + + async def run(self, role: "STRole", act_desp: str, act_world: str, act_sector: str): + def create_prompt_input(role, act_desp, act_world, act_sector): + prompt_input = [] + prompt_input += [role.scratch.get_str_name()] + x = f"{act_world}:{act_sector}" + prompt_input += [act_sector] + + # MAR 11 TEMP + accessible_arena_str = role.s_mem.get_str_accessible_sector_arenas(x) + curr = accessible_arena_str.split(", ") + fin_accessible_arenas = [] + for i in curr: + if "'s room" in i: + if role.scratch.last_name in i: + fin_accessible_arenas += [i] + else: + fin_accessible_arenas += [i] + accessible_arena_str = ", ".join(fin_accessible_arenas) + # END MAR 11 TEMP + prompt_input += [accessible_arena_str] + act_desp_1 = act_desp + act_desp_2 = act_desp + if "(" in act_desp: + act_desp_1 = act_desp.split("(")[0].strip() + act_desp_2 = act_desp.split("(")[-1][:-1] + prompt_input += [role.scratch.get_str_name()] + prompt_input += [act_desp_1] + + prompt_input += [act_desp_2] + prompt_input += [role.scratch.get_str_name()] + + prompt_input += [act_sector] + prompt_input += [accessible_arena_str] + return prompt_input + + prompt_template = "action_location_object_vMar11.txt" + prompt_input = create_prompt_input(role, act_desp, act_world, act_sector) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp() + print('prompt ', prompt) + output = await self._run_gpt35_max_tokens(prompt, max_tokens=15) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenActionObject(STAction): + name: str = "GenActionObject" + + def _func_validate(self, llm_resp: str, prompt: str): + if len(llm_resp.strip()) < 1: + return False + return True + + def _func_cleanup(self, llm_resp: str, prompt: str): + cleaned_response = llm_resp.strip() + return cleaned_response + + def _func_fail_default_resp(self): + fs = "bed" + return fs + + async def run(self, role: "STRole", act_desp: str, temp_address: str): + def create_prompt_input(role, act_desp, temp_address): + prompt_input = [] + if "(" in act_desp: + act_desp = act_desp.split("(")[-1][:-1] + + prompt_input += [act_desp] + prompt_input += [role.s_mem.get_str_accessible_arena_game_objects(temp_address)] + return prompt_input + + prompt_template = "action_object_v2.txt" + prompt_input = create_prompt_input(role, act_desp, temp_address) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=15) + x = [i.strip() for i in role.s_mem.get_str_accessible_arena_game_objects(temp_address).split(",")] + if output not in x: + output = random.choice(x) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenPronunciatio(STAction): + name: str = "GenPronunciatio" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cr = llm_resp.strip() + if len(cr) > 3: + cr = cr[:3] + return cr + + def _func_validate(self, llm_resp: str, prompt: str): + try: + self._func_cleanup(llm_resp, prompt="") + if len(llm_resp) == 0: + return False + except Exception: + return False + return True + + def _func_fail_default_resp(self): + fs = "😋" + return fs + + async def run(self, role: "STRole", act_desp: str): + def create_prompt_input(act_desp): + if "(" in act_desp: + act_desp = act_desp.split("(")[-1].split(")")[0] + prompt_input = [act_desp] + return prompt_input + + prompt_template = "generate_pronunciatio_v1.txt" + prompt_input = create_prompt_input(act_desp) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + example_output = "🛁🧖‍♀️" + special_instruction = "The value for the output must ONLY contain the emojis." + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenEventTriple(STAction): + name: str = "GenEventTriple" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cr = llm_resp.strip() + cr = [i.strip() for i in cr.split(")")[0].split(",")] + return cr + + def _func_validate(self, llm_resp: str, prompt: str): + try: + llm_resp = self._func_cleanup(llm_resp, prompt="") + if len(llm_resp) != 2: + return False + except Exception: + return False + return True + + def _func_fail_default_resp(self, role): + fs = (role.name, "is", "idle") + return fs + + async def run(self, role: "STRole", act_desp: str): + def create_prompt_input(role, act_desp): + if "(" in act_desp: + act_desp = act_desp.split("(")[-1].split(")")[0] + prompt_input = [role.name, act_desp, role.name] + return prompt_input + + prompt_template = "generate_event_triple_v1.txt" + prompt_input = create_prompt_input(role, act_desp) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp(role) + output = await self._run_gpt35_max_tokens(prompt, max_tokens=30) + output = (role.name, output[0], output[1]) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenActObjDescription(STAction): + name: str = "GenActObjDescription" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cr = llm_resp.strip() + if cr[-1] == ".": + cr = cr[:-1] + return cr + + def _func_validate(self, llm_resp: str, prompt: str): + try: + llm_resp = self._func_cleanup(llm_resp, prompt="") + except Exception: + return False + return True + + def _func_fail_default_resp(self, act_game_object): + fs = f"{act_game_object} is idle" + return fs + + async def run(self, role: "STRole", act_game_object: str, act_desp: str): + def create_prompt_input(act_game_object, act_desp, role): + prompt_input = [act_game_object, role.name, act_desp, act_game_object, act_game_object] + return prompt_input + + prompt_template = "generate_obj_event_v1.txt" + prompt_input = create_prompt_input(act_game_object, act_desp, role) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + example_output = "being fixed" + special_instruction = "The output should ONLY contain the phrase that should go in ." + self.fail_default_resp = self._func_fail_default_resp(act_game_object) + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenObjEventTriple(STAction): + name: str = "GenObjEventTriple" + + def _func_cleanup(self, llm_resp: str, prompt: str): + cr = llm_resp.strip() + cr = [i.strip() for i in cr.split(")")[0].split(",")] + return cr + + def _func_validate(self, llm_resp: str, prompt: str): + try: + llm_resp = self._func_cleanup(llm_resp, prompt="") + if len(llm_resp) != 2: + return False + except Exception: + return False + return True + + def _func_fail_default_resp(self, act_game_object: str): + fs = (act_game_object, "is", "idle") + return fs + + async def run(self, role: "STRole", act_game_object, act_obj_desp): + def create_prompt_input(act_game_object, act_obj_desp): + prompt_input = [act_game_object, act_obj_desp, act_game_object] + return prompt_input + + prompt_template = "generate_event_triple_v1.txt" + prompt_input = create_prompt_input(act_game_object, act_obj_desp) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp(act_game_object) + output = await self._run_gpt35_max_tokens(prompt, max_tokens=30) + output = (act_game_object, output[0], output[1]) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +class GenActionDetails(STAction): + name: str = "GenActionDetails" + + def _func_cleanup(self, llm_resp: str, prompt: str) -> list: + pass + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + # TODO -- this sometimes generates error + try: + self._func_cleanup(llm_resp) + except Exception: + return False + return True + + def _func_fail_default_resp(self): + fs = {} + return fs + + async def run(self, role: "STRole", act_desp: str, act_dura): + access_tile = await role.rc.env.observe( + EnvAPIAbstract(api_name="access_tile", kwargs={"tile": role.scratch.curr_tile}) + ) + act_world = access_tile["world"] + act_sector = await GenActionSector().run(role, access_tile, act_desp) + act_arena = await GenActionArena().run(role, act_desp, act_world, act_sector) + act_address = f"{act_world}:{act_sector}:{act_arena}" + if not role.s_mem.get_str_accessible_arena_game_objects(act_address): + act_game_object = "" + else: + act_game_object = await GenActionObject().run(role, act_desp, act_address) + new_address = f"{act_world}:{act_sector}:{act_arena}:{act_game_object}" + act_pron = await GenPronunciatio().run(role, act_desp) + act_event = await GenEventTriple().run(role, act_desp) + # Persona's actions also influence the object states. We set those up here. + act_obj_desp = await GenActObjDescription().run(role, act_game_object, act_desp) + act_obj_pron = await GenPronunciatio().run(role, act_obj_desp) + act_obj_event = await GenObjEventTriple().run(role, act_game_object, act_obj_desp) + result_dict = { + "action_address": new_address, + "action_duration": int(act_dura), + "action_description": act_desp, + "action_pronunciatio": act_pron, + "action_event": act_event, + "chatting_with": None, + "chat": None, + "chatting_with_buffer": None, + "chatting_end_time": None, + "act_obj_description": act_obj_desp, + "act_obj_pronunciatio": act_obj_pron, + "act_obj_event": act_obj_event, + } + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {result_dict}") + return result_dict diff --git a/examples/st_game/actions/gen_daily_schedule.py b/examples/st_game/actions/gen_daily_schedule.py new file mode 100644 index 000000000..6deb6e6eb --- /dev/null +++ b/examples/st_game/actions/gen_daily_schedule.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : gen_daily_schedule + + +from metagpt.logs import logger + +from .st_action import STAction + + +class GenDailySchedule(STAction): + name: str = "GenDailySchedule" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt="") + except Exception: + return False + return True + + def _func_cleanup(self, llm_resp: str, prompt: str) -> list: + cr = [] + _cr = llm_resp.split(")") + for i in _cr: + if i[-1].isdigit(): + i = i[:-1].strip() + if i[-1] == "." or i[-1] == ",": + cr += [i[:-1].strip()] + return cr + + def _func_fail_default_resp(self) -> int: + fs = [ + "wake up and complete the morning routine at 6:00 am", + "eat breakfast at 7:00 am", + "read a book from 8:00 am to 12:00 pm", + "have lunch at 12:00 pm", + "take a nap from 1:00 pm to 4:00 pm", + "relax and watch TV from 7:00 pm to 8:00 pm", + "go to bed at 11:00 pm", + ] + return fs + + async def run(self, role: "STRole", wake_up_hour: str): + def create_prompt_input(role, wake_up_hour): + prompt_input = [] + prompt_input += [role.scratch.get_str_iss()] + prompt_input += [role.scratch.get_str_lifestyle()] + prompt_input += [role.scratch.get_str_curr_date_str()] + prompt_input += [role.scratch.get_str_firstname()] + prompt_input += [f"{str(wake_up_hour)}:00 am"] + return prompt_input + + wake_up_hour = int(wake_up_hour) + prompt_template = "daily_planning_v6.txt" + prompt_input = create_prompt_input(role, wake_up_hour) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=500) + output = [f"wake up and complete the morning routine at {wake_up_hour}:00 am"] + output + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/gen_hourly_schedule.py b/examples/st_game/actions/gen_hourly_schedule.py new file mode 100644 index 000000000..5d59f96dd --- /dev/null +++ b/examples/st_game/actions/gen_hourly_schedule.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : gen_hourly_schedule + +import random +import string + +from metagpt.logs import logger + +from .st_action import STAction + + +def get_random_alphanumeric(i=6, j=6): + """ + Returns a random alpha numeric strength that has the length of somewhere + between i and j. + + INPUT: + i: min_range for the length + j: max_range for the length + OUTPUT: + an alpha numeric str with the length of somewhere between i and j. + """ + k = random.randint(i, j) + x = "".join(random.choices(string.ascii_letters + string.digits, k=k)) + return x + + +class GenHourlySchedule(STAction): + name: str = "GenHourlySchedule" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt="") + except Exception: + return False + return True + + def _func_cleanup(self, llm_resp: str, prompt: str) -> list: + cr = llm_resp.strip() + if cr[-1] == ".": + cr = cr[:-1] + # to only use the first line of output + cr = cr.split("\n")[0] + return cr + + def _func_fail_default_resp(self) -> int: + fs = "asleep" + return fs + + async def _generate_schedule_for_given_hour( + self, role: "STRole", curr_hour_str, p_f_ds_hourly_org, hour_str, intermission2=None + ): + def create_prompt_input(persona, curr_hour_str, p_f_ds_hourly_org, hour_str, intermission2=None): + schedule_format = "" + for i in hour_str: + schedule_format += f"[{persona.scratch.get_str_curr_date_str()} -- {i}]" + schedule_format += " Activity: [Fill in]\n" + schedule_format = schedule_format[:-1] + + intermission_str = "Here the originally intended hourly breakdown of" + intermission_str += f" {persona.scratch.get_str_firstname()}'s schedule today: " + for count, i in enumerate(persona.scratch.daily_req): + intermission_str += f"{str(count + 1)}) {i}, " + intermission_str = intermission_str[:-2] + + prior_schedule = "" + if p_f_ds_hourly_org: + prior_schedule = "\n" + for count, i in enumerate(p_f_ds_hourly_org): + prior_schedule += f"[(ID:{get_random_alphanumeric()})" + prior_schedule += f" {persona.scratch.get_str_curr_date_str()} --" + prior_schedule += f" {hour_str[count]}] Activity:" + prior_schedule += f" {persona.scratch.get_str_firstname()}" + prior_schedule += f" is {i}\n" + + prompt_ending = f"[(ID:{get_random_alphanumeric()})" + prompt_ending += f" {persona.scratch.get_str_curr_date_str()}" + prompt_ending += f" -- {curr_hour_str}] Activity:" + prompt_ending += f" {persona.scratch.get_str_firstname()} is" + + if intermission2: + intermission2 = f"\n{intermission2}" + + prompt_input = [] + prompt_input += [schedule_format] + prompt_input += [persona.scratch.get_str_iss()] + + prompt_input += [prior_schedule + "\n"] + prompt_input += [intermission_str] + if intermission2: + prompt_input += [intermission2] + else: + prompt_input += [""] + prompt_input += [prompt_ending] + + return prompt_input + + prompt_template = "generate_hourly_schedule_v2.txt" + prompt_input = create_prompt_input(role, curr_hour_str, p_f_ds_hourly_org, hour_str, intermission2) + prompt_input_str = "\n".join(prompt_input) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, prompt_template) + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=50) + logger.info( + f"Role: {role.name} _generate_schedule_for_given_hour prompt_input: {prompt_input_str}, " + f"output: {output}" + ) + return output + + async def run(self, role: "STRole", wake_up_hour: int): + hour_str = [ + "00:00 AM", + "01:00 AM", + "02:00 AM", + "03:00 AM", + "04:00 AM", + "05:00 AM", + "06:00 AM", + "07:00 AM", + "08:00 AM", + "09:00 AM", + "10:00 AM", + "11:00 AM", + "12:00 PM", + "01:00 PM", + "02:00 PM", + "03:00 PM", + "04:00 PM", + "05:00 PM", + "06:00 PM", + "07:00 PM", + "08:00 PM", + "09:00 PM", + "10:00 PM", + "11:00 PM", + ] + n_m1_activity = [] + diversity_repeat_count = 1 # TODO mg 1->3 + for i in range(diversity_repeat_count): + logger.info(f"diversity_repeat_count idx: {i}") + n_m1_activity_set = set(n_m1_activity) + if len(n_m1_activity_set) < 5: + n_m1_activity = [] + for count, curr_hour_str in enumerate(hour_str): + if wake_up_hour > 0: + n_m1_activity += ["sleeping"] + wake_up_hour -= 1 + else: + logger.info(f"_generate_schedule_for_given_hour idx: {count}, n_m1_activity: {n_m1_activity}") + n_m1_activity += [ + await self._generate_schedule_for_given_hour(role, curr_hour_str, n_m1_activity, hour_str) + ] + + # Step 1. Compressing the hourly schedule to the following format: + # The integer indicates the number of hours. They should add up to 24. + # [['sleeping', 6], ['waking up and starting her morning routine', 1], + # ['eating breakfast', 1], ['getting ready for the day', 1], + # ['working on her painting', 2], ['taking a break', 1], + # ['having lunch', 1], ['working on her painting', 3], + # ['taking a break', 2], ['working on her painting', 2], + # ['relaxing and watching TV', 1], ['going to bed', 1], ['sleeping', 2]] + _n_m1_hourly_compressed = [] + prev = None + prev_count = 0 + for i in n_m1_activity: + if i != prev: + prev_count = 1 + _n_m1_hourly_compressed += [[i, prev_count]] + prev = i + elif _n_m1_hourly_compressed: + _n_m1_hourly_compressed[-1][1] += 1 + + # Step 2. Expand to min scale (from hour scale) + # [['sleeping', 360], ['waking up and starting her morning routine', 60], + # ['eating breakfast', 60],.. + n_m1_hourly_compressed = [] + for task, duration in _n_m1_hourly_compressed: + n_m1_hourly_compressed += [[task, duration * 60]] + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {n_m1_hourly_compressed}") + return n_m1_hourly_compressed diff --git a/examples/st_game/actions/gen_iter_chat_utt.py b/examples/st_game/actions/gen_iter_chat_utt.py new file mode 100644 index 000000000..2b0d46f4e --- /dev/null +++ b/examples/st_game/actions/gen_iter_chat_utt.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : generate_iterative_chat_utt + +from examples.st_game.actions.st_action import STAction +from examples.st_game.utils.utils import extract_first_json_dict +from metagpt.environment.api.env_api import EnvAPIAbstract +from metagpt.logs import logger + + +class GenIterChatUTT(STAction): + name: str = "GenIterChatUTT" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + resp = False + try: + _ = extract_first_json_dict(llm_resp) + resp = True + except Exception: + pass + return resp + + def _func_cleanup(self, llm_resp: str, prompt: str) -> dict: + gpt_response = extract_first_json_dict(llm_resp) + + cleaned_dict = dict() + cleaned = [] + for key, val in gpt_response.items(): + cleaned += [val] + cleaned_dict["utterance"] = cleaned[0] + cleaned_dict["end"] = True + if "f" in str(cleaned[1]) or "F" in str(cleaned[1]): + cleaned_dict["end"] = False + + return cleaned_dict + + def _func_fail_default_resp(self) -> dict: + cleaned_dict = dict() + cleaned_dict["utterance"] = "..." + cleaned_dict["end"] = False + return cleaned_dict + + async def run( + self, + init_role: "STRole", + target_role: "STRole", + retrieved: dict, + curr_context: str, + curr_chat: list[str], + *args, + **kwargs, + ) -> dict: + def create_prompt_input( + access_tile: dict[str, str], + init_role: "STRole", + target_role: "STRole", + retrieved: dict, + curr_context: str, + curr_chat: list[str], + ): + role = init_role + scratch = role.rc.scratch + target_scratch = target_role.rc.scratch + prev_convo_insert = "\n" + if role.rc.memory.chat_list: + for i in role.rc.memory.chat_list: + if i.object == target_role.name: + v1 = int((scratch.curr_time - i.created).total_seconds() / 60) + prev_convo_insert += ( + f"{str(v1)} minutes ago, {scratch.name} and " + f"{target_scratch.name} were already {i.description} " + f"This context takes place after that conversation." + ) + break + if prev_convo_insert == "\n": + prev_convo_insert = "" + if role.rc.memory.chat_list: + if int((scratch.curr_time - role.rc.memory.chat_list[-1].created).total_seconds() / 60) > 480: + prev_convo_insert = "" + print(prev_convo_insert) + + curr_sector = f"{access_tile['sector']}" + curr_arena = f"{access_tile['arena']}" + curr_location = f"{curr_arena} in {curr_sector}" + + retrieved_str = "" + for key, vals in retrieved.items(): + for v in vals: + retrieved_str += f"- {v.description}\n" + + convo_str = "" + for i in curr_chat: + convo_str += ": ".join(i) + "\n" + if convo_str == "": + convo_str = "[The conversation has not started yet -- start it!]" + + init_iss = f"Here is Here is a brief description of {scratch.name}.\n{scratch.get_str_iss()}" + prompt_input = [ + init_iss, + scratch.name, + retrieved_str, + prev_convo_insert, + curr_location, + curr_context, + scratch.name, + target_scratch.name, + convo_str, + scratch.name, + target_scratch.name, + scratch.name, + scratch.name, + scratch.name, + ] + return prompt_input + + access_tile = await init_role.rc.env.observe( + EnvAPIAbstract(api_name="access_tile", kwargs={"tile": init_role.scratch.curr_tile}) + ) + prompt_input = create_prompt_input(access_tile, init_role, target_role, retrieved, curr_context, curr_chat) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "iterative_convo_v1.txt") + # original using `ChatGPT_safe_generate_response_OLD` + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_wo_extra_prompt(prompt) + logger.info(f"Role: {init_role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/inner_voice_action.py b/examples/st_game/actions/inner_voice_action.py new file mode 100644 index 000000000..cfbdf0303 --- /dev/null +++ b/examples/st_game/actions/inner_voice_action.py @@ -0,0 +1,31 @@ +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +class AgentWhisperThoughtAction(STAction): + name: str = "AgentWhisperThoughtAction" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> list: + return llm_resp.split('"')[0].strip() + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, test_input=None, verbose=False) -> str: + def create_prompt_input(role: "STRole", statements, test_input=None): + prompt_input = [role.scratch.name, statements] + return prompt_input + + prompt_input = create_prompt_input(role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "whisper_inner_thought_v1.txt") + + output = await self._run_gpt35_max_tokens(prompt, max_tokens=50) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/new_decomp_schedule.py b/examples/st_game/actions/new_decomp_schedule.py new file mode 100644 index 000000000..8e5c79401 --- /dev/null +++ b/examples/st_game/actions/new_decomp_schedule.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : new_decomp_schedule + +import datetime + +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +class NewDecompSchedule(STAction): + name: str = "NewDecompSchedule" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + resp = False + try: + llm_resp = self._func_cleanup(llm_resp, prompt) + dur_sum = 0 + for act, dur in llm_resp: + dur_sum += dur + if isinstance(act, str): + return False + if isinstance(dur, int): + return False + x = prompt.split("\n")[0].split("originally planned schedule from")[-1].strip()[:-1] + x = [datetime.datetime.strptime(i.strip(), "%H:%M %p") for i in x.split(" to ")] + delta_min = int((x[1] - x[0]).total_seconds() / 60) + + if int(dur_sum) != int(delta_min): + return False + except Exception: + pass + return resp + + def _func_cleanup(self, llm_resp: str, prompt: str) -> list: + new_schedule = prompt + " " + llm_resp.strip() + new_schedule = new_schedule.split("The revised schedule:")[-1].strip() + new_schedule = new_schedule.split("\n") + + ret_temp = [] + for i in new_schedule: + ret_temp += [i.split(" -- ")] + + ret = [] + for time_str, action in ret_temp: + start_time = time_str.split(" ~ ")[0].strip() + end_time = time_str.split(" ~ ")[1].strip() + delta = datetime.datetime.strptime(end_time, "%H:%M") - datetime.datetime.strptime(start_time, "%H:%M") + delta_min = int(delta.total_seconds() / 60) + if delta_min < 0: + delta_min = 0 + ret += [[action, delta_min]] + + return ret + + def _func_fail_default_resp(self, main_act_dur: int, truncated_act_dur: int) -> int: + dur_sum = 0 + for act, dur in main_act_dur: + dur_sum += dur + + ret = truncated_act_dur[:] + ret += main_act_dur[len(ret) - 1 :] + + # If there are access, we need to trim... + ret_dur_sum = 0 + count = 0 + over = None + for act, dur in ret: + ret_dur_sum += dur + if ret_dur_sum == dur_sum: + break + if ret_dur_sum > dur_sum: + over = ret_dur_sum - dur_sum + break + count += 1 + + if over: + ret = ret[: count + 1] + ret[-1][1] -= over + + return ret + + async def run( + self, + role: "STRole", + main_act_dur: int, + truncated_act_dur: int, + start_time_hour: datetime, + end_time_hour: datetime, + inserted_act: str, + inserted_act_dur: int, + *args, + **kwargs, + ): + def create_prompt_input( + role: "STRole", + main_act_dur: int, + truncated_act_dur: int, + start_time_hour: datetime, + end_time_hour: datetime, + inserted_act: str, + inserted_act_dur: int, + ): + persona_name = role.name + start_hour_str = start_time_hour.strftime("%H:%M %p") + end_hour_str = end_time_hour.strftime("%H:%M %p") + + original_plan = "" + for_time = start_time_hour + for i in main_act_dur: + original_plan += ( + f'{for_time.strftime("%H:%M")} ~ ' + f'{(for_time + datetime.timedelta(minutes=int(i[1]))).strftime("%H:%M")} -- ' + i[0] + ) + original_plan += "\n" + for_time += datetime.timedelta(minutes=int(i[1])) + + new_plan_init = "" + for_time = start_time_hour + for count, i in enumerate(truncated_act_dur): + new_plan_init += ( + f'{for_time.strftime("%H:%M")} ~ ' + f'{(for_time + datetime.timedelta(minutes=int(i[1]))).strftime("%H:%M")} -- ' + i[0] + ) + new_plan_init += "\n" + if count < len(truncated_act_dur) - 1: + for_time += datetime.timedelta(minutes=int(i[1])) + + new_plan_init += (for_time + datetime.timedelta(minutes=int(i[1]))).strftime("%H:%M") + " ~" + + prompt_input = [ + persona_name, + start_hour_str, + end_hour_str, + original_plan, + persona_name, + inserted_act, + inserted_act_dur, + persona_name, + start_hour_str, + end_hour_str, + end_hour_str, + new_plan_init, + ] + return prompt_input + + prompt_input = create_prompt_input( + role, main_act_dur, truncated_act_dur, start_time_hour, end_time_hour, inserted_act, inserted_act_dur + ) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "new_decomp_schedule_v1.txt") + self.fail_default_resp = self._func_fail_default_resp(main_act_dur, truncated_act_dur) + output = await self._run_gpt35_max_tokens(prompt, max_tokens=1000) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/run_reflect_action.py b/examples/st_game/actions/run_reflect_action.py new file mode 100644 index 000000000..d3bf008cb --- /dev/null +++ b/examples/st_game/actions/run_reflect_action.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : Integration Reflect Action + +import re + +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +# Run GPT Prompt Focal Point method +class AgentFocusPt(STAction): + name: str = "AgentFocusPt" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> str: + try: + """ + Cleanup handling has been completed for run_v2 + """ + return llm_resp + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, n: int, test_input=None) -> str: + def create_prompt_input(role: "STRole", statements, n, test_input=None): + prompt_input = [statements, str(n)] + return prompt_input + + prompt_input = create_prompt_input(role, statements, n) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "generate_focal_pt_v1.txt") + + example_output = '["What should Jane do for lunch", "Does Jane like strawberry", "Who is Jane"]' + special_instruction = "Output must be a list of str." + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Insight and Guidance +class AgentInsightAndGuidance(STAction): + name: str = "AgentInsightAndGuidance" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> dict: + try: + llm_resp = "1. " + llm_resp.strip() + ret = dict() + for i in llm_resp.split("\n"): + row = " ".join(i.split(". ")[1:]) + if "(because of " not in row: + continue + thought = row.split("(because of ")[0].strip() + if ")" not in row.split("(because of ")[1]: + continue + evi_raw = row.split("(because of ")[1].split(")")[0].strip() + evi_raw = re.findall(r"\d+", evi_raw) + evi_raw = [int(i.strip()) for i in evi_raw] + ret[thought] = evi_raw + return ret + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self, n: int) -> str: + return ["I am hungry"] * n + + async def run(self, role: "STRole", statements: str, n: int, test_input=None) -> dict: + def create_prompt_input(role, statements, n, test_input=None): + prompt_input = [statements, str(n)] + return prompt_input + + prompt_input = create_prompt_input(role, statements, n) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "insight_and_evidence_v1.txt") + + self.fail_default_resp = self._func_fail_default_resp(n) + output = await self._run_gpt35_max_tokens(prompt, max_tokens=150) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Event Triple +class AgentEventTriple(STAction): + name: str = "AgentEventTriple" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + llm_resp = self._func_cleanup(llm_resp, prompt="") + if len(llm_resp) != 2: + return False + except Exception: + return False + return True + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> list: + try: + cr = llm_resp.strip() + cr = [i.strip() for i in cr.split(")")[0].split(",")] + if len(cr) != 2: + return cr[-2:] + return cr + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, statements: str, role: "STRole", verbose=False) -> tuple: + def create_prompt_input(statements, role): + if "(" in statements: + statements = statements.split("(")[-1].split(")")[0] + prompt_input = [role.scratch.name, statements, role.scratch.name] + return prompt_input + + prompt_input = create_prompt_input(statements, role) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "generate_event_triple_v1.txt") + + output = await self._run_gpt35_max_tokens(prompt, max_tokens=30) + output = (role.scratch.name, output[0], output[1]) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Event Poignancy +class AgentEventPoignancy(STAction): + name: str = "AgentEventPoignancy" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> int: + try: + llm_resp = int(llm_resp.strip()) + return llm_resp + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, test_input=None, verbose=False) -> str: + def create_prompt_input(role: "STRole", statements: str, test_input=None): + prompt_input = [role.scratch.name, role.scratch.get_str_iss(), role.scratch.name, statements] + return prompt_input + + prompt_input = create_prompt_input(role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "poignancy_event_v1.txt") + + example_output = "5" # ######## + special_instruction = "The output should ONLY contain ONE integer value on the scale of 1 to 10." + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Chat Poignancy +class AgentChatPoignancy(STAction): + name: str = "AgentChatPoignancy" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> int: + try: + llm_resp = int(llm_resp.strip()) + return llm_resp + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, test_input=None, verbose=False) -> str: + def create_prompt_input(role: "STRole", statements, test_input=None): + prompt_input = [role.scratch.name, role.scratch.get_str_iss(), role.scratch.name, statements] + return prompt_input + + prompt_input = create_prompt_input(role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "poignancy_chat_v1.txt") + + example_output = "5" # ######## + special_instruction = "The output should ONLY contain ONE integer value on the scale of 1 to 10." + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Planning Thought on Convo +class AgentPlanThoughtOnConvo(STAction): + name: str = "AgentPlanThoughtOnConvo" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> str: + try: + return llm_resp.split('"')[0].strip() + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, test_input=None, verbose=False) -> str: + def create_prompt_input(role, statements, test_input=None): + prompt_input = [statements, role.scratch.name, role.scratch.name, role.scratch.name] + return prompt_input + + prompt_input = create_prompt_input(role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "planning_thought_on_convo_v1.txt") + + output = await self._run_gpt35_max_tokens(prompt, max_tokens=50) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output + + +# Run GPT Prompt Memory on Convo +class AgentMemoryOnConvo(STAction): + name: str = "AgentMemoryOnConvo" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + try: + self._func_cleanup(llm_resp, prompt) + return True + except Exception: + return False + + def _func_cleanup(self, llm_resp: str, prompt: str = "") -> str: + try: + return llm_resp.split('"')[0].strip() + except Exception as exp: + logger.error(f"{self.cls_name} with error {exp}") + + def _func_fail_default_resp(self) -> str: + pass + + async def run(self, role: "STRole", statements: str, test_input=None, verbose=False) -> str: + def create_prompt_input(role, statements, test_input=None): + prompt_input = [statements, role.scratch.name, role.scratch.name, role.scratch.name] + return prompt_input + + prompt_input = create_prompt_input(role, statements) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "memo_on_convo_v1.txt") + example_output = "Jane Doe was interesting to talk to." + special_instruction = ( + "The output should ONLY contain a string that summarizes anything interesting " + "that the agent may have noticed" + ) + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/st_action.py b/examples/st_game/actions/st_action.py new file mode 100644 index 000000000..0796b5c8d --- /dev/null +++ b/examples/st_game/actions/st_action.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : StanfordTown Action +import json +import time +from abc import abstractmethod +from pathlib import Path +from typing import Any, Optional, Union + +from examples.st_game.utils.const import PROMPTS_DIR +from metagpt.actions.action import Action +from metagpt.config2 import config +from metagpt.logs import logger + + +class STAction(Action): + name: str = "STAction" + prompt_dir: Path = PROMPTS_DIR + fail_default_resp: Optional[str] = None + + @property + def cls_name(self): + return self.__class__.__name__ + + @abstractmethod + def _func_validate(self, llm_resp: str, prompt: str): + raise NotImplementedError + + @abstractmethod + def _func_cleanup(self, llm_resp: str, prompt: str): + raise NotImplementedError + + @abstractmethod + def _func_fail_default_resp(self): + raise NotImplementedError + + def generate_prompt_with_tmpl_filename(self, prompt_input: Union[str, list], tmpl_filename) -> str: + """ + same with `generate_prompt` + Args: + prompt_input: the input we want to feed in (IF THERE ARE MORE THAN ONE INPUT, THIS CAN BE A LIST.) + tmpl_filename: prompt template filename + Returns: + a str prompt that will be sent to LLM server. + """ + if isinstance(prompt_input, str): + prompt_input = [prompt_input] + prompt_input = [str(i) for i in prompt_input] + + f = open(str(self.prompt_dir.joinpath(tmpl_filename)), "r") + prompt = f.read() + f.close() + for count, i in enumerate(prompt_input): + prompt = prompt.replace(f"!!", i) + if "###" in prompt: + prompt = prompt.split("###")[1] + return prompt.strip() + + async def _aask(self, prompt: str) -> str: + return await self.llm.aask(prompt) + + async def _run_gpt35_max_tokens(self, prompt: str, max_tokens: int = 50, retry: int = 3): + for idx in range(retry): + try: + tmp_max_tokens_rsp = getattr(config.llm, "max_token", 1500) + setattr(config.llm, "max_token", max_tokens) + self.llm.use_system_prompt = False # to make it behave like a non-chat completions + + llm_resp = await self._aask(prompt) + + setattr(config.llm, "max_token", tmp_max_tokens_rsp) + logger.info(f"Action: {self.cls_name} llm _run_gpt35_max_tokens raw resp: {llm_resp}") + if self._func_validate(llm_resp, prompt): + return self._func_cleanup(llm_resp, prompt) + except Exception as exp: + logger.warning(f"Action: {self.cls_name} _run_gpt35_max_tokens exp: {exp}") + time.sleep(5) + return self.fail_default_resp + + async def _run_gpt35( + self, prompt: str, example_output: str, special_instruction: str, retry: int = 3 + ) -> Union[bool, Any]: + """same with `gpt_structure.ChatGPT_safe_generate_response`""" + prompt = '"""\n' + prompt + '\n"""\n' + prompt += f"Output the response to the prompt above in json. {special_instruction}\n" + prompt += "Example output json:\n" + prompt += '{"output": "' + str(example_output) + '"}' + + for idx in range(retry): + try: + llm_resp = await self._aask(prompt) + logger.info(f"Action: {self.cls_name} llm _run_gpt35 raw resp: {llm_resp}") + end_idx = llm_resp.strip().rfind("}") + 1 + llm_resp = llm_resp[:end_idx] + llm_resp = json.loads(llm_resp)["output"] + + if self._func_validate(llm_resp, prompt): + return self._func_cleanup(llm_resp, prompt) + except Exception as exp: + logger.warning(f"Action: {self.cls_name} _run_gpt35 exp: {exp}") + time.sleep(5) # usually avoid `Rate limit` + return False + + async def _run_gpt35_wo_extra_prompt(self, prompt: str, retry: int = 3) -> str: + for idx in range(retry): + try: + llm_resp = await self._aask(prompt) + llm_resp = llm_resp.strip() + logger.info(f"Action: {self.cls_name} llm _run_gpt35_wo_extra_prompt raw resp: {llm_resp}") + if self._func_validate(llm_resp, prompt): + return self._func_cleanup(llm_resp, prompt) + except Exception as exp: + logger.warning(f"Action: {self.cls_name} _run_gpt35_wo_extra_prompt exp: {exp}") + time.sleep(5) # usually avoid `Rate limit` + return self.fail_default_resp + + async def run(self, *args, **kwargs): + """Run action""" + raise NotImplementedError("The run method should be implemented in a subclass.") diff --git a/examples/st_game/actions/summarize_conv.py b/examples/st_game/actions/summarize_conv.py new file mode 100644 index 000000000..f35651b35 --- /dev/null +++ b/examples/st_game/actions/summarize_conv.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : summarize the content of agents' conversation + +from examples.st_game.actions.st_action import STAction +from metagpt.logs import logger + + +class SummarizeConv(STAction): + name: str = "SummarizeConv" + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + resp = False + try: + _ = self._func_cleanup(llm_resp, prompt) + resp = True + except Exception: + pass + return resp + + def _func_cleanup(self, llm_resp: str, prompt: str) -> str: + ret = "conversing about " + llm_resp.strip() + return ret + + def _func_fail_default_resp(self) -> str: + return "conversing with a housemate about morning greetings" + + async def run(self, conv: list): + def create_prompt_input(conversation: list): + convo_str = "" + for row in conversation: + convo_str += f'{row[0]}: "{row[1]}"\n' + prompt_input = [convo_str] + return prompt_input + + prompt_input = create_prompt_input(conv) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "summarize_conversation_v1.txt") + + example_output = "conversing about what to eat for lunch" + special_instruction = ( + "The output must continue the sentence above by filling in the tag. " + "Don't start with 'this is a conversation about...' Just finish the sentence " + "but do not miss any important details (including who are chatting)." + ) + output = await self._run_gpt35(prompt, example_output, special_instruction) + logger.info(f"Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/task_decomp.py b/examples/st_game/actions/task_decomp.py new file mode 100644 index 000000000..d9d5ec9fa --- /dev/null +++ b/examples/st_game/actions/task_decomp.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : task_decomp + +import datetime + +from metagpt.logs import logger + +from ..actions.st_action import STAction + + +class TaskDecomp(STAction): + name: str = "TaskDecomp" + + def _func_cleanup(self, llm_resp: str, prompt: str) -> list: + # TODO SOMETHING HERE sometimes fails... See screenshot + temp = [i.strip() for i in llm_resp.split("\n")] + _cr = [] + cr = [] + for count, i in enumerate(temp): + if count != 0: + _cr += [" ".join([j.strip() for j in i.split(" ")][3:])] + else: + _cr += [i] + for count, i in enumerate(_cr): + k = [j.strip() for j in i.split("(duration in minutes:")] + task = k[0] + if task[-1] == ".": + task = task[:-1] + duration = int(k[1].split(",")[0].strip()) + cr += [[task, duration]] + + total_expected_min = int(prompt.split("(total duration in minutes")[-1].split("):")[0].strip()) + + # TODO -- now, you need to make sure that this is the same as the sum of + # the current action sequence. + curr_min_slot = [ + ["dummy", -1], + ] # (task_name, task_index) + for count, i in enumerate(cr): + i_task = i[0] + i_duration = i[1] + + i_duration -= i_duration % 5 + if i_duration > 0: + for j in range(i_duration): + curr_min_slot += [(i_task, count)] + curr_min_slot = curr_min_slot[1:] + + if len(curr_min_slot) > total_expected_min: + last_task = curr_min_slot[60] + for i in range(1, 6): + curr_min_slot[-1 * i] = last_task + elif len(curr_min_slot) < total_expected_min: + last_task = curr_min_slot[-1] + for i in range(total_expected_min - len(curr_min_slot)): + curr_min_slot += [last_task] + + cr_ret = [ + ["dummy", -1], + ] + for task, task_index in curr_min_slot: + if task != cr_ret[-1][0]: + cr_ret += [[task, 1]] + else: + cr_ret[-1][1] += 1 + cr = cr_ret[1:] + + return cr + + def _func_validate(self, llm_resp: str, prompt: str) -> bool: + # TODO -- this sometimes generates error + try: + self._func_cleanup(llm_resp, prompt) + except Exception: + return False + return True + + def _func_fail_default_resp(self) -> int: + fs = [["asleep", 0]] + return fs + + async def run(self, role: "STRole", task_desc: int, truncated_act_dur: int, *args, **kwargs): + def create_prompt_input(role, task, duration): + """ + Today is Saturday June 25. From 00:00 ~ 06:00am, Maeve is + planning on sleeping, 06:00 ~ 07:00am, Maeve is + planning on waking up and doing her morning routine, + and from 07:00am ~08:00am, Maeve is planning on having breakfast. + """ + + curr_f_org_index = role.scratch.get_f_daily_schedule_hourly_org_index() + all_indices = [] + # if curr_f_org_index > 0: + # all_indices += [curr_f_org_index-1] + all_indices += [curr_f_org_index] + if curr_f_org_index + 1 <= len(role.scratch.f_daily_schedule_hourly_org): + all_indices += [curr_f_org_index + 1] + if curr_f_org_index + 2 <= len(role.scratch.f_daily_schedule_hourly_org): + all_indices += [curr_f_org_index + 2] + + curr_time_range = "" + + print("DEBUG") + print(role.scratch.f_daily_schedule_hourly_org) + print(all_indices) + + summ_str = f'Today is {role.scratch.curr_time.strftime("%B %d, %Y")}. ' + summ_str += "From " + for index in all_indices: + print("index", index) + if index < len(role.scratch.f_daily_schedule_hourly_org): + start_min = 0 + for i in range(index): + start_min += role.scratch.f_daily_schedule_hourly_org[i][1] + end_min = start_min + role.scratch.f_daily_schedule_hourly_org[index][1] + start_time = datetime.datetime.strptime("00:00:00", "%H:%M:%S") + datetime.timedelta( + minutes=start_min + ) + end_time = datetime.datetime.strptime("00:00:00", "%H:%M:%S") + datetime.timedelta( + minutes=end_min + ) + start_time_str = start_time.strftime("%H:%M%p") + end_time_str = end_time.strftime("%H:%M%p") + summ_str += ( + f"{start_time_str} ~ {end_time_str}, {role.name} is planning " + f"on {role.scratch.f_daily_schedule_hourly_org[index][0]}, " + ) + if curr_f_org_index + 1 == index: + curr_time_range = f"{start_time_str} ~ {end_time_str}" + summ_str = summ_str[:-2] + "." + + prompt_input = [] + prompt_input += [role.scratch.get_str_iss()] + prompt_input += [summ_str] + # prompt_input += [role.scratch.get_str_curr_date_str()] + prompt_input += [role.scratch.get_str_firstname()] + prompt_input += [role.scratch.get_str_firstname()] + prompt_input += [task] + prompt_input += [curr_time_range] + prompt_input += [duration] + prompt_input += [role.scratch.get_str_firstname()] + return prompt_input + + prompt_input = create_prompt_input(role, task_desc, truncated_act_dur) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "task_decomp_v3.txt") + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=1000) + logger.info(f"Role: {role.name} {self.cls_name} output: {output}") + + fin_output = [] + time_sum = 0 + for i_task, i_duration in output: + time_sum += i_duration + # HM????????? + # if time_sum < duration: + if time_sum <= truncated_act_dur: + fin_output += [[i_task, i_duration]] + else: + break + ftime_sum = 0 + for fi_task, fi_duration in fin_output: + ftime_sum += fi_duration + + # print ("for debugging... line 365", fin_output) + fin_output[-1][1] += truncated_act_dur - ftime_sum + output = fin_output + + task_decomp = output + ret = [] + for decomp_task, duration in task_decomp: + ret += [[f"{task_desc} ({decomp_task})", duration]] + output = ret + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/actions/wake_up.py b/examples/st_game/actions/wake_up.py new file mode 100644 index 000000000..d39115854 --- /dev/null +++ b/examples/st_game/actions/wake_up.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : wake_up + + +from metagpt.logs import logger + +from ..actions.st_action import STAction + + +class WakeUp(STAction): + name: str = "WakeUp" + + def _func_validate(self, llm_resp: str, prompt: str = None) -> bool: + try: + self._func_cleanup(llm_resp, prompt="") + except Exception: + return False + return True + + def _func_cleanup(self, llm_resp: str, prompt: str) -> int: + cr = int(llm_resp.strip().lower().split("am")[0]) + return cr + + def _func_fail_default_resp(self) -> int: + fs = 8 + return fs + + async def run(self, role: "STRole"): + def create_prompt_input(role): + prompt_input = [ + role.scratch.get_str_iss(), + role.scratch.get_str_lifestyle(), + role.scratch.get_str_firstname(), + ] + return prompt_input + + prompt_input = create_prompt_input(role) + prompt = self.generate_prompt_with_tmpl_filename(prompt_input, "wake_up_hour_v1.txt") + self.fail_default_resp = self._func_fail_default_resp() + output = await self._run_gpt35_max_tokens(prompt, max_tokens=5) + logger.info(f"Role: {role.name} Action: {self.cls_name} output: {output}") + return output diff --git a/examples/st_game/memory/__init__.py b/examples/st_game/memory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/st_game/memory/agent_memory.py b/examples/st_game/memory/agent_memory.py new file mode 100644 index 000000000..a66a40efd --- /dev/null +++ b/examples/st_game/memory/agent_memory.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : BasicMemory,AgentMemory实现 + +from datetime import datetime +from pathlib import Path +from typing import Optional + +from pydantic import Field, field_serializer, model_validator + +from metagpt.logs import logger +from metagpt.memory.memory import Memory +from metagpt.schema import Message +from metagpt.utils.common import read_json_file, write_json_file + + +class BasicMemory(Message): + """ + BasicMemory继承于MG的Message类，其中content属性替代description属性 + Message类中对于Chat类型支持的非常好，对于Agent个体的Perceive,Reflection,Plan支持的并不多 + 在Type设计上，我们延续GA的三个种类，但是对于Chat种类的对话进行特别设计（具体怎么设计还没想好） + """ + + memory_id: Optional[str] = Field(default=None) # 记忆ID + memory_count: int = -1 # 第几个记忆，实际数值与Memory相等 + type_count: int = -1 # 第几种记忆，类型为整数 + memory_type: Optional[str] = Field(default=None) # 记忆类型，包含 event,thought,chat三种类型 + depth: int = -1 # 记忆深度，类型为整数 + created: Optional[datetime] = Field(default=None) # 创建时间 + expiration: Optional[datetime] = Field(default=None) # 记忆失效时间，默认为空（） + last_accessed: Optional[datetime] = Field(default=None) # 上一次调用的时间，初始化时候与self.created一致 + subject: Optional[str] = Field(default=None) # 主语 + predicate: Optional[str] = Field(default=None) # 谓语 + object: Optional[str] = Field(default=None) # 宾语 + + description: Optional[str] = Field(default=None) + embedding_key: Optional[str] = Field(default=None) # 内容与self.content一致 + poignancy: int = -1 # importance值 + keywords: list[str] = Field(default=[]) # keywords + filling: list = Field(default=[]) # 装的与之相关联的memory_id的列表 + + @model_validator(mode="before") + @classmethod + def check_values(cls, values): + if "created" in values: + values["last_accessed"] = values["created"] + if "content" in values: + values["description"] = values["content"] + return values + + @field_serializer("created", "expiration") + def transform_time_field(self, time_field: Optional[datetime]) -> str: + if time_field: + time_field = time_field.strftime("%Y-%m-%d %H:%M:%S") + return time_field + + def summary(self): + return self.subject, self.predicate, self.object + + def save_to_dict(self) -> dict: + """ + 将MemoryBasic类转化为字典，用于存储json文件 + 这里需要注意，cause_by跟GA不兼容，所以需要做一个格式转换 + """ + memory_dict = dict() + node_id = self.memory_id + basic_mem_obj = self.model_dump( + include=[ + "node_count", + "type_count", + "type", + "depth", + "created", + "expiration", + "subject", + "predicate", + "object", + "description", + "embedding_key", + "poignancy", + "keywords", + "filling", + "cause_by", + ] + ) + + memory_dict[node_id] = basic_mem_obj + return memory_dict + + +class AgentMemory(Memory): + """ + GA中主要存储三种JSON + 1. embedding.json (Dict embedding_key:embedding) + 2. Node.json (Dict Node_id:Node) + 3. kw_strength.json + """ + + storage: list[BasicMemory] = [] # 重写Storage，存储BasicMemory所有节点 + event_list: list[BasicMemory] = [] # 存储event记忆 + thought_list: list[BasicMemory] = [] # 存储thought记忆 + chat_list: list[BasicMemory] = [] # chat-related memory + + event_keywords: dict[str, list[BasicMemory]] = dict() # 存储keywords + thought_keywords: dict[str, list[BasicMemory]] = dict() + chat_keywords: dict[str, list[BasicMemory]] = dict() + + kw_strength_event: dict[str, int] = dict() + kw_strength_thought: dict[str, int] = dict() + + memory_saved: Optional[Path] = Field(default=None) + embeddings: dict[str, list[float]] = dict() + + def set_mem_path(self, memory_saved: Path): + self.memory_saved = memory_saved + self.load(memory_saved) + + def save(self, memory_saved: Path): + """ + 将MemoryBasic类存储为Nodes.json形式。复现GA中的Kw Strength.json形式 + 这里添加一个路径即可 + TODO 这里在存储时候进行倒序存储，之后需要验证（test_memory通过） + """ + memory_json = dict() + for i in range(len(self.storage)): + memory_node = self.storage[len(self.storage) - i - 1] + memory_node = memory_node.save_to_dict() + memory_json.update(memory_node) + write_json_file(memory_saved.joinpath("nodes.json"), memory_json) + write_json_file(memory_saved.joinpath("embeddings.json"), self.embeddings) + + strength_json = dict() + strength_json["kw_strength_event"] = self.kw_strength_event + strength_json["kw_strength_thought"] = self.kw_strength_thought + write_json_file(memory_saved.joinpath("kw_strength.json"), strength_json) + + def load(self, memory_saved: Path): + """ + 将GA的JSON解析，填充到AgentMemory类之中 + """ + self.embeddings = read_json_file(memory_saved.joinpath("embeddings.json")) + memory_load = read_json_file(memory_saved.joinpath("nodes.json")) + for count in range(len(memory_load.keys())): + node_id = f"node_{str(count + 1)}" + node_details = memory_load[node_id] + node_type = node_details["type"] + created = datetime.strptime(node_details["created"], "%Y-%m-%d %H:%M:%S") + expiration = None + if node_details["expiration"]: + expiration = datetime.strptime(node_details["expiration"], "%Y-%m-%d %H:%M:%S") + + s = node_details["subject"] + p = node_details["predicate"] + o = node_details["object"] + + description = node_details["description"] + embedding_pair = (node_details["embedding_key"], self.embeddings[node_details["embedding_key"]]) + poignancy = node_details["poignancy"] + keywords = set(node_details["keywords"]) + filling = node_details["filling"] + if node_type == "thought": + self.add_thought( + created, expiration, s, p, o, description, keywords, poignancy, embedding_pair, filling + ) + if node_type == "event": + self.add_event(created, expiration, s, p, o, description, keywords, poignancy, embedding_pair, filling) + if node_type == "chat": + self.add_chat(created, expiration, s, p, o, description, keywords, poignancy, embedding_pair, filling) + + strength_keywords_load = read_json_file(memory_saved.joinpath("kw_strength.json")) + if strength_keywords_load["kw_strength_event"]: + self.kw_strength_event = strength_keywords_load["kw_strength_event"] + if strength_keywords_load["kw_strength_thought"]: + self.kw_strength_thought = strength_keywords_load["kw_strength_thought"] + + def add(self, memory_basic: BasicMemory): + """ + Add a new message to storage, while updating the index + 重写add方法，修改原有的Message类为BasicMemory类，并添加不同的记忆类型添加方式 + """ + if memory_basic.memory_id in self.storage: + return + self.storage.append(memory_basic) + if memory_basic.memory_type == "chat": + self.chat_list[0:0] = [memory_basic] + return + if memory_basic.memory_type == "thought": + self.thought_list[0:0] = [memory_basic] + return + if memory_basic.memory_type == "event": + self.event_list[0:0] = [memory_basic] + return + + def add_chat( + self, created, expiration, s, p, o, content, keywords, poignancy, embedding_pair, filling, cause_by="" + ): + """ + 调用add方法，初始化chat，在创建的时候就需要调用embedding函数 + """ + memory_count = len(self.storage) + 1 + type_count = len(self.thought_list) + 1 + memory_type = "chat" + memory_id = f"node_{str(memory_count)}" + depth = 1 + + memory_node = BasicMemory( + memory_id=memory_id, + memory_count=memory_count, + type_count=type_count, + memory_type=memory_type, + depth=depth, + created=created, + expiration=expiration, + subject=s, + predicate=p, + object=o, + description=content, + embedding_key=embedding_pair[0], + poignancy=poignancy, + keywords=keywords, + filling=filling, + cause_by=cause_by + ) + + keywords = [i.lower() for i in keywords] + for kw in keywords: + if kw in self.chat_keywords: + self.chat_keywords[kw][0:0] = [memory_node] + else: + self.chat_keywords[kw] = [memory_node] + + self.add(memory_node) + + self.embeddings[embedding_pair[0]] = embedding_pair[1] + return memory_node + + def add_thought(self, created, expiration, s, p, o, content, keywords, poignancy, embedding_pair, filling): + """ + 调用add方法，初始化thought + """ + memory_count = len(self.storage) + 1 + type_count = len(self.thought_list) + 1 + memory_type = "thought" + memory_id = f"node_{str(memory_count)}" + depth = 1 + + try: + if filling: + depth_list = [memory_node.depth for memory_node in self.storage if memory_node.memory_id in filling] + depth += max(depth_list) + except Exception as exp: + logger.warning(f"filling init occur {exp}") + pass + + memory_node = BasicMemory( + memory_id=memory_id, + memory_count=memory_count, + type_count=type_count, + memory_type=memory_type, + depth=depth, + created=created, + expiration=expiration, + subject=s, + predicate=p, + object=o, + description=content, + embedding_key=embedding_pair[0], + poignancy=poignancy, + keywords=keywords, + filling=filling + ) + + keywords = [i.lower() for i in keywords] + for kw in keywords: + if kw in self.thought_keywords: + self.thought_keywords[kw][0:0] = [memory_node] + else: + self.thought_keywords[kw] = [memory_node] + + self.add(memory_node) + + if f"{p} {o}" != "is idle": + for kw in keywords: + if kw in self.kw_strength_thought: + self.kw_strength_thought[kw] += 1 + else: + self.kw_strength_thought[kw] = 1 + + self.embeddings[embedding_pair[0]] = embedding_pair[1] + return memory_node + + def add_event(self, created, expiration, s, p, o, content, keywords, poignancy, embedding_pair, filling): + """ + 调用add方法，初始化event + """ + memory_count = len(self.storage) + 1 + type_count = len(self.event_list) + 1 + memory_type = "event" + memory_id = f"node_{str(memory_count)}" + depth = 0 + + if "(" in content: + content = " ".join(content.split()[:3]) + " " + content.split("(")[-1][:-1] + + memory_node = BasicMemory( + memory_id=memory_id, + memory_count=memory_count, + type_count=type_count, + memory_type=memory_type, + depth=depth, + created=created, + expiration=expiration, + subject=s, + predicate=p, + object=o, + description=content, + embedding_key=embedding_pair[0], + poignancy=poignancy, + keywords=keywords, + filling=filling + ) + + keywords = [i.lower() for i in keywords] + for kw in keywords: + if kw in self.event_keywords: + self.event_keywords[kw][0:0] = [memory_node] + else: + self.event_keywords[kw] = [memory_node] + + self.add(memory_node) + + if f"{p} {o}" != "is idle": + for kw in keywords: + if kw in self.kw_strength_event: + self.kw_strength_event[kw] += 1 + else: + self.kw_strength_event[kw] = 1 + + self.embeddings[embedding_pair[0]] = embedding_pair[1] + return memory_node + + def get_summarized_latest_events(self, retention): + ret_set = set() + for e_node in self.event_list[:retention]: + ret_set.add(e_node.summary()) + return ret_set + + def get_last_chat(self, target_role_name: str): + if target_role_name.lower() in self.chat_keywords: + return self.chat_keywords[target_role_name.lower()][0] + else: + return False + + def retrieve_relevant_thoughts(self, s_content: str, p_content: str, o_content: str) -> set: + contents = [s_content, p_content, o_content] + + ret = [] + for i in contents: + if i in self.thought_keywords: + ret += self.thought_keywords[i.lower()] + + ret = set(ret) + return ret + + def retrieve_relevant_events(self, s_content: str, p_content: str, o_content: str) -> set: + contents = [s_content, p_content, o_content] + + ret = [] + for i in contents: + if i in self.event_keywords: + ret += self.event_keywords[i] + + ret = set(ret) + return ret diff --git a/examples/st_game/memory/retrieve.py b/examples/st_game/memory/retrieve.py new file mode 100644 index 000000000..eeeecf85a --- /dev/null +++ b/examples/st_game/memory/retrieve.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : Retrieve函数实现 + +import datetime + +from numpy import dot +from numpy.linalg import norm + +from examples.st_game.memory.agent_memory import BasicMemory +from examples.st_game.utils.utils import get_embedding + + +def agent_retrieve( + agent_memory, + curr_time: datetime.datetime, + memory_forget: float, + query: str, + nodes: list[BasicMemory], + topk: int = 4, +) -> list[BasicMemory]: + """ + Retrieve需要集合Role使用,原因在于Role才具有AgentMemory,scratch + 逻辑:Role调用该函数,self.rc.AgentMemory,self.rc.scratch.curr_time,self.rc.scratch.memory_forget + 输入希望查询的内容与希望回顾的条数,返回TopK条高分记忆，即List[BasicMemory] + + Score_lists示例 + { + "memory": memories[i], BasicMemory类 + "importance": memories[i].poignancy + "recency": 衰减因子计算结果 + "relevance": 搜索结果 + } + """ + memories = nodes + agent_memory_embedding = agent_memory.embeddings + memories = sorted(memories, key=lambda memory_node: memory_node.last_accessed, reverse=True) + + score_list = [] + score_list = extract_importance(memories, score_list) + score_list = extract_recency(curr_time, memory_forget, score_list) + score_list = extract_relevance(agent_memory_embedding, query, score_list) + score_list = normalize_score_floats(score_list, 0, 1) + + total_dict = {} + gw = [1, 1, 1] # 三个因素的权重,重要性,近因性,相关性, + for i in range(len(score_list)): + total_score = ( + score_list[i]["importance"] * gw[0] + score_list[i]["recency"] * gw[1] + score_list[i]["relevance"] * gw[2] + ) + total_dict[score_list[i]["memory"].memory_id] = total_score + + result = top_highest_x_values(total_dict, topk) + + return result # 返回的是一个BasicMemory列表 + + +def new_agent_retrieve(role, focus_points: list, n_count=30) -> dict: + """ + 输入为role，关注点列表,返回记忆数量 + 输出为字典，键为focus_point，值为对应的记忆列表 + """ + retrieved = dict() + for focal_pt in focus_points: + nodes = [ + [i.last_accessed, i] + for i in role.memory.event_list + role.memory.thought_list + if "idle" not in i.embedding_key + ] + nodes = sorted(nodes, key=lambda x: x[0]) + nodes = [i for created, i in nodes] + results = agent_retrieve( + role.memory, role.scratch.curr_time, role.scratch.recency_decay, focal_pt, nodes, n_count + ) + final_result = [] + for n in results: + for i in role.memory.storage: + if i.memory_id == n: + i.last_accessed = role.scratch.curr_time + final_result.append(i) + + retrieved[focal_pt] = final_result + + return retrieved + + +def top_highest_x_values(d, x): + """ + 输入字典，Topx + 返回以字典值排序，字典键组成的List[BasicMemory] + """ + top_v = [item[0] for item in sorted(d.items(), key=lambda item: item[1], reverse=True)[:x]] + return top_v + + +def extract_importance(memories, score_list): + """ + 抽取重要性 + """ + for i in range(len(memories)): + score = {"memory": memories[i], "importance": memories[i].poignancy} + score_list.append(score) + return score_list + + +def extract_relevance(agent_memory_embedding, query, score_list): + """ + 抽取相关性 + """ + query_embedding = get_embedding(query) + # 进行 + for i in range(len(score_list)): + node_embedding = agent_memory_embedding[score_list[i]["memory"].embedding_key] + result = cos_sim(node_embedding, query_embedding) + score_list[i]["relevance"] = result + + return score_list + + +def extract_recency(curr_time, memory_forget, score_list): + """ + 抽取近因性，目前使用的现实世界过一天走一个衰减因子 + """ + for i in range(len(score_list)): + day_count = (curr_time - score_list[i]["memory"].created).days + score_list[i]["recency"] = memory_forget**day_count + return score_list + + +def cos_sim(a, b): + """ + 计算余弦相似度 + """ + return dot(a, b) / (norm(a) * norm(b)) + + +def normalize_list_floats(single_list, target_min, target_max): + """ + 单个列表归一化 + """ + if len(single_list) == 0: + return [] + + min_val = min(single_list) + max_val = max(single_list) + range_val = max_val - min_val + + if range_val == 0: + for i in range(len(single_list)): + single_list[i] = (target_max - target_min) / 2 + else: + for i in range(len(single_list)): + single_list[i] = (single_list[i] - min_val) * (target_max - target_min) / range_val + target_min + return single_list + + +def normalize_score_floats(score_list, target_min, target_max): + """ + 整体归一化 + """ + importance_list = [] + relevance_list = [] + recency_list = [] + + for i in range(len(score_list)): + importance_list.append(score_list[i]["importance"]) + relevance_list.append(score_list[i]["relevance"]) + recency_list.append(score_list[i]["recency"]) + + # 进行归一化操作 + importance_list = normalize_list_floats(importance_list, target_min, target_max) + relevance_list = normalize_list_floats(relevance_list, target_min, target_max) + recency_list = normalize_list_floats(recency_list, target_min, target_max) + + for i in range(len(score_list)): + score_list[i]["importance"] = importance_list[i] + score_list[i]["relevance"] = relevance_list[i] + score_list[i]["recency"] = recency_list[i] + + return score_list diff --git a/examples/st_game/memory/scratch.py b/examples/st_game/memory/scratch.py new file mode 100644 index 000000000..b7040ce08 --- /dev/null +++ b/examples/st_game/memory/scratch.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : Scratch类实现（角色信息类） + +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional + +from pydantic import ( + BaseModel, + Field, + field_serializer, + field_validator, + model_validator, +) + +from metagpt.utils.common import read_json_file, write_json_file + + +class Scratch(BaseModel): + # 类别1:人物超参 + vision_r: int = 4 + att_bandwidth: int = 3 + retention: int = 5 + + # 类别2:世界信息 + curr_time: Optional[datetime] = Field(default=None) + curr_tile: Optional[str] = Field(default=None) + daily_plan_req: Optional[str] = Field(default=None) + + # 类别3:人物角色的核心身份 + name: Optional[str] = Field(default=None) + first_name: Optional[str] = Field(default=None) + last_name: Optional[str] = Field(default=None) + age: Optional[int] = Field(default=None) + innate: Optional[str] = Field(default=None) # L0 permanent core traits. + learned: Optional[str] = Field(default=None) # L1 stable traits. + currently: Optional[str] = Field(default=None) # L2 external implementation. + lifestyle: Optional[str] = Field(default=None) + living_area: Optional[str] = Field(default=None) + + # 类别4:旧反思变量 + concept_forget: int = 100 + daily_reflection_time: int = 60 * 3 + daily_reflection_size: int = 5 + overlap_reflect_th: int = 2 + kw_strg_event_reflect_th: int = 4 + kw_strg_thought_reflect_th: int = 4 + + # 类别5:新反思变量 + recency_w: int = 1 + relevance_w: int = 1 + importance_w: int = 1 + recency_decay: float = 0.99 + importance_trigger_max: int = 150 + importance_trigger_curr: int = 150 + importance_ele_n: int = 0 + thought_count: int = 5 + + # 类别6:个人计划 + daily_req: list[str] = Field(default=[]) + f_daily_schedule: list[list[str]] = Field(default=[]) + f_daily_schedule_hourly_org: list[list[str]] = Field(default=[]) + + # 类别7:当前动作 + act_address: Optional[str] = Field(default=None) + act_start_time: Optional[datetime] = Field(default=None) + act_duration: Optional[int] = Field(default=None) + act_description: Optional[str] = Field(default=None) + act_pronunciatio: Optional[str] = Field(default=None) + act_event: tuple[str, Optional[str], Optional[str]] = (None, None, None) + + act_obj_description: Optional[str] = Field(default=None) + act_obj_pronunciatio: Optional[str] = Field(default=None) + act_obj_event: tuple[Optional[str], Optional[str], Optional[str]] = (None, None, None) + + chatting_with: Optional[str] = Field(default=None) + chat: Optional[str] = Field(default=None) + chatting_with_buffer: dict = dict() + chatting_end_time: Optional[datetime] = Field(default=None) + + act_path_set: bool = False + planned_path: list[str] = Field(default=[]) + + @model_validator(mode="after") + @classmethod + def check_values(cls, values): + if "name" in values: + values["act_event"] = (values["name"], None, None) + values["act_obj_event"] = (values["name"], None, None) + return values + + @field_validator("curr_time", "act_start_time", "chatting_end_time") + @classmethod + def check_time_filed(cls, time_filed): + val = datetime.strptime(time_filed, "%B %d, %Y, %H:%M:%S") if time_filed else None + return val + + @field_serializer("curr_time", "act_start_time", "chatting_end_time") + def transform_time_field(self, time_filed: Optional[datetime]) -> str: + if time_filed: + time_filed = time_filed.strftime("%B %d, %Y, %H:%M:%S") + return time_filed + + @classmethod + def set_scratch_path(cls, f_saved: Path): + scratch_load = read_json_file(f_saved) + scratch = Scratch(**scratch_load) + return scratch + + def save(self, out_json: Path): + """ + Save persona's scratch. + + INPUT: + out_json: The file where we wil be saving our persona's state. + OUTPUT: + None + """ + scratch = self.model_dump() + write_json_file(out_json, scratch, encoding="utf-8") + + def get_f_daily_schedule_index(self, advance=0): + """ + We get the current index of self.f_daily_schedule. + + Recall that self.f_daily_schedule stores the decomposed action sequences + up until now, and the hourly sequences of the future action for the rest + of today. Given that self.f_daily_schedule is a list of list where the + inner list is composed of [task, duration], we continue to add up the + duration until we reach "if elapsed > today_min_elapsed" condition. The + index where we stop is the index we will return. + + INPUT + advance: Integer value of the number minutes we want to look into the + future. This allows us to get the index of a future timeframe. + OUTPUT + an integer value for the current index of f_daily_schedule. + """ + # We first calculate teh number of minutes elapsed today. + today_min_elapsed = 0 + today_min_elapsed += self.curr_time.hour * 60 + today_min_elapsed += self.curr_time.minute + today_min_elapsed += advance + + x = 0 + for task, duration in self.f_daily_schedule: + x += duration + x = 0 + for task, duration in self.f_daily_schedule_hourly_org: + x += duration + + # We then calculate the current index based on that. + curr_index = 0 + elapsed = 0 + for task, duration in self.f_daily_schedule: + elapsed += duration + if elapsed > today_min_elapsed: + return curr_index + curr_index += 1 + + return curr_index + + def get_f_daily_schedule_hourly_org_index(self, advance=0): + """ + We get the current index of self.f_daily_schedule_hourly_org. + It is otherwise the same as get_f_daily_schedule_index. + + INPUT + advance: Integer value of the number minutes we want to look into the + future. This allows us to get the index of a future timeframe. + OUTPUT + an integer value for the current index of f_daily_schedule. + """ + # We first calculate teh number of minutes elapsed today. + today_min_elapsed = 0 + today_min_elapsed += self.curr_time.hour * 60 + today_min_elapsed += self.curr_time.minute + today_min_elapsed += advance + # We then calculate the current index based on that. + curr_index = 0 + elapsed = 0 + for task, duration in self.f_daily_schedule_hourly_org: + elapsed += duration + if elapsed > today_min_elapsed: + return curr_index + curr_index += 1 + return curr_index + + def get_str_iss(self): + """ + ISS stands for "identity stable set." This describes the commonset summary + of this persona -- basically, the bare minimum description of the persona + that gets used in almost all prompts that need to call on the persona. + + INPUT + None + OUTPUT + the identity stable set summary of the persona in a string form. + EXAMPLE STR OUTPUT + "Name: Dolores Heitmiller + Age: 28 + Innate traits: hard-edged, independent, loyal + Learned traits: Dolores is a painter who wants live quietly and paint + while enjoying her everyday life. + Currently: Dolores is preparing for her first solo show. She mostly + works from home. + Lifestyle: Dolores goes to bed around 11pm, sleeps for 7 hours, eats + dinner around 6pm. + Daily plan requirement: Dolores is planning to stay at home all day and + never go out." + """ + commonset = "" + commonset += f"Name: {self.name}\n" + commonset += f"Age: {self.age}\n" + commonset += f"Innate traits: {self.innate}\n" + commonset += f"Learned traits: {self.learned}\n" + commonset += f"Currently: {self.currently}\n" + commonset += f"Lifestyle: {self.lifestyle}\n" + commonset += f"Daily plan requirement: {self.daily_plan_req}\n" + commonset += f"Current Date: {self.curr_time.strftime('%A %B %d') if self.curr_time else ''}\n" + return commonset + + def get_str_name(self): + return self.name + + def get_str_firstname(self): + return self.first_name + + def get_str_lastname(self): + return self.last_name + + def get_str_age(self): + return str(self.age) + + def get_str_innate(self): + return self.innate + + def get_str_learned(self): + return self.learned + + def get_str_currently(self): + return self.currently + + def get_str_lifestyle(self): + return self.lifestyle + + def get_str_daily_plan_req(self): + return self.daily_plan_req + + def get_str_curr_date_str(self): + return self.curr_time.strftime("%A %B %d") + + def get_curr_event(self): + if not self.act_address: + return (self.name, None, None) + else: + return self.act_event + + def get_curr_event_and_desc(self): + if not self.act_address: + return (self.name, None, None, None) + else: + return (self.act_event[0], self.act_event[1], self.act_event[2], self.act_description) + + def get_curr_obj_event_and_desc(self): + if not self.act_address: + return ("", None, None, None) + else: + return (self.act_address, self.act_obj_event[1], self.act_obj_event[2], self.act_obj_description) + + def add_new_action( + self, + action_address, + action_duration, + action_description, + action_pronunciatio, + action_event, + chatting_with, + chat, + chatting_with_buffer, + chatting_end_time, + act_obj_description, + act_obj_pronunciatio, + act_obj_event, + act_start_time=None, + ): + self.act_address = action_address + self.act_duration = action_duration + self.act_description = action_description + self.act_pronunciatio = action_pronunciatio + self.act_event = action_event + + self.chatting_with = chatting_with + self.chat = chat + if chatting_with_buffer: + self.chatting_with_buffer.update(chatting_with_buffer) + self.chatting_end_time = chatting_end_time + + self.act_obj_description = act_obj_description + self.act_obj_pronunciatio = act_obj_pronunciatio + self.act_obj_event = act_obj_event + + self.act_start_time = self.curr_time + + self.act_path_set = False + + def act_time_str(self): + """ + Returns a string output of the current time. + + INPUT + None + OUTPUT + A string output of the current time. + EXAMPLE STR OUTPUT + "14:05 P.M." + """ + return self.act_start_time.strftime("%H:%M %p") + + def act_check_finished(self): + """ + Checks whether the self.Action instance has finished. + + INPUT + curr_datetime: Current time. If current time is later than the action's + start time + its duration, then the action has finished. + OUTPUT + Boolean [True]: Action has finished. + Boolean [False]: Action has not finished and is still ongoing. + """ + if not self.act_address: + return True + + if self.chatting_with: + end_time = self.chatting_end_time + else: + x = self.act_start_time + if x.second != 0: + x = x.replace(second=0) + x = x + timedelta(minutes=1) + end_time = x + timedelta(minutes=self.act_duration) + + if end_time.strftime("%H:%M:%S") == self.curr_time.strftime("%H:%M:%S"): + return True + return False + + def act_summarize(self): + """ + Summarize the current action as a dictionary. + + INPUT + None + OUTPUT + ret: A human readable summary of the action. + """ + exp = dict() + exp["persona"] = self.name + exp["address"] = self.act_address + exp["start_datetime"] = self.act_start_time + exp["duration"] = self.act_duration + exp["description"] = self.act_description + exp["pronunciatio"] = self.act_pronunciatio + return exp + + def act_summary_str(self): + """ + Returns a string summary of the current action. Meant to be + human-readable. + + INPUT + None + OUTPUT + ret: A human readable summary of the action. + """ + start_datetime_str = self.act_start_time.strftime("%A %B %d -- %H:%M %p") + ret = f"[{start_datetime_str}]\n" + ret += f"Activity: {self.name} is {self.act_description}\n" + ret += f"Address: {self.act_address}\n" + ret += f"Duration in minutes (e.g., x min): {str(self.act_duration)} min\n" + return ret + + def get_daily_schedule(self, daily_schedule: list[list[str]]): + ret = "" + curr_min_sum = 0 + for row in daily_schedule: + curr_min_sum += row[1] + hour = int(curr_min_sum / 60) + minute = curr_min_sum % 60 + ret += f"{hour:02}:{minute:02} || {row[0]}\n" + return ret + + def get_str_daily_schedule_summary(self): + return self.get_daily_schedule(self.f_daily_schedule) + + def get_str_daily_schedule_hourly_org_summary(self): + return self.get_daily_schedule(self.f_daily_schedule_hourly_org) diff --git a/examples/st_game/memory/spatial_memory.py b/examples/st_game/memory/spatial_memory.py new file mode 100644 index 000000000..c1030cdbb --- /dev/null +++ b/examples/st_game/memory/spatial_memory.py @@ -0,0 +1,115 @@ +""" +Author: Joon Sung Park (joonspk@stanford.edu) + +File: spatial_memory.py +Description: Defines the MemoryTree class that serves as the agents' spatial +memory that aids in grounding their behavior in the game world. +""" +from pathlib import Path + +from pydantic import BaseModel, Field + +from metagpt.utils.common import read_json_file, write_json_file + + +class MemoryTree(BaseModel): + tree: dict = Field(default=dict) + + def set_mem_path(self, f_saved: Path): + self.tree = read_json_file(f_saved) + + def print_tree(self) -> None: + def _print_tree(tree, depth): + dash = " >" * depth + if isinstance(tree, list): + if tree: + print(dash, tree) + return + + for key, val in tree.items(): + if key: + print(dash, key) + _print_tree(val, depth + 1) + + _print_tree(self.tree, 0) + + def save(self, out_json: Path) -> None: + write_json_file(out_json, self.tree) + + def get_str_accessible_sectors(self, curr_world: str) -> str: + """ + Returns a summary string of all the arenas that the persona can access + within the current sector. + + Note that there are places a given persona cannot enter. This information + is provided in the persona sheet. We account for this in this function. + + INPUT + None + OUTPUT + A summary string of all the arenas that the persona can access. + EXAMPLE STR OUTPUT + "bedroom, kitchen, dining room, office, bathroom" + """ + x = ", ".join(list(self.tree[curr_world].keys())) + return x + + def get_str_accessible_sector_arenas(self, sector: str) -> str: + """ + Returns a summary string of all the arenas that the persona can access + within the current sector. + + Note that there are places a given persona cannot enter. This information + is provided in the persona sheet. We account for this in this function. + + INPUT + None + OUTPUT + A summary string of all the arenas that the persona can access. + EXAMPLE STR OUTPUT + "bedroom, kitchen, dining room, office, bathroom" + """ + curr_world, curr_sector = sector.split(":") + if not curr_sector: + return "" + x = ", ".join(list(self.tree[curr_world][curr_sector].keys())) + return x + + def get_str_accessible_arena_game_objects(self, arena: str) -> str: + """ + Get a str list of all accessible game objects that are in the arena. If + temp_address is specified, we return the objects that are available in + that arena, and if not, we return the objects that are in the arena our + persona is currently in. + + INPUT + temp_address: optional arena address + OUTPUT + str list of all accessible game objects in the gmae arena. + EXAMPLE STR OUTPUT + "phone, charger, bed, nightstand" + """ + curr_world, curr_sector, curr_arena = arena.split(":") + + if not curr_arena: + return "" + + try: + x = ", ".join(list(self.tree[curr_world][curr_sector][curr_arena])) + except Exception: + x = ", ".join(list(self.tree[curr_world][curr_sector][curr_arena.lower()])) + return x + + def add_tile_info(self, tile_info: dict) -> None: + if tile_info["world"]: + if tile_info["world"] not in self.tree: + self.tree[tile_info["world"]] = {} + if tile_info["sector"]: + if tile_info["sector"] not in self.tree[tile_info["world"]]: + self.tree[tile_info["world"]][tile_info["sector"]] = {} + if tile_info["arena"]: + if tile_info["arena"] not in self.tree[tile_info["world"]][tile_info["sector"]]: + self.tree[tile_info["world"]][tile_info["sector"]][tile_info["arena"]] = [] + if tile_info["game_object"]: + if tile_info["game_object"] not in self.tree[tile_info["world"]][tile_info["sector"]][tile_info["arena"]]: + self.tree[tile_info["world"]][tile_info["sector"]][tile_info["arena"]] += [tile_info["game_object"]] diff --git a/examples/st_game/plan/__init__.py b/examples/st_game/plan/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/examples/st_game/plan/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/examples/st_game/plan/converse.py b/examples/st_game/plan/converse.py new file mode 100644 index 000000000..0371b3d44 --- /dev/null +++ b/examples/st_game/plan/converse.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : conversation between two agents + +from typing import Tuple + +from examples.st_game.actions.agent_chat_sum_rel import AgentChatSumRel +from examples.st_game.actions.gen_iter_chat_utt import GenIterChatUTT +from examples.st_game.memory.retrieve import new_agent_retrieve +from metagpt.logs import logger + + +async def agent_conversation(init_role: "STRole", target_role: "STRole", conv_rounds: int = 8) -> list[list[str]]: + curr_chat = [] + logger.info(f"Role: {init_role.name} starts a conversation with Role: {target_role.name}") + + for idx in range(conv_rounds): + logger.info(f"Conv round: {idx} between {init_role.name} and {target_role.name}") + scratch = init_role.rc.scratch + target_scratch = target_role.rc.scratch + + focal_points = [f"{target_scratch.name}"] + retrieved = new_agent_retrieve(init_role, focal_points, 50) + relationship = await generate_summarize_agent_relationship(init_role, target_role, retrieved) + logger.info(f"The relationship between {init_role.name} and {target_role.name}: {relationship}") + last_chat = "" + for i in curr_chat[-4:]: + last_chat += ": ".join(i) + "\n" + if last_chat: + focal_points = [f"{relationship}", f"{target_scratch.name} is {target_scratch.act_description}", last_chat] + else: + focal_points = [f"{relationship}", f"{target_scratch.name} is {target_scratch.act_description}"] + retrieved = new_agent_retrieve(init_role, focal_points, 15) + utt, end = await generate_one_utterance(init_role, target_role, retrieved, curr_chat) + + curr_chat += [[scratch.name, utt]] + if end: + break + + focal_points = [f"{scratch.name}"] + retrieved = new_agent_retrieve(target_role, focal_points, 50) + relationship = await generate_summarize_agent_relationship(target_role, init_role, retrieved) + logger.info(f"The relationship between {target_role.name} and {init_role.name}: {relationship}") + last_chat = "" + for i in curr_chat[-4:]: + last_chat += ": ".join(i) + "\n" + if last_chat: + focal_points = [f"{relationship}", f"{scratch.name} is {scratch.act_description}", last_chat] + else: + focal_points = [f"{relationship}", f"{scratch.name} is {scratch.act_description}"] + retrieved = new_agent_retrieve(target_role, focal_points, 15) + utt, end = await generate_one_utterance(target_role, init_role, retrieved, curr_chat) + + curr_chat += [[target_scratch.name, utt]] + if end: + break + + logger.warning(f"Conversations between {target_role.name} and {init_role.name}:") + for row in curr_chat: + logger.info(row) + + return curr_chat + + +async def generate_summarize_agent_relationship(init_role: "STRole", target_role: "STRole", retrieved: dict) -> str: + all_embedding_keys = list() + for key, val in retrieved.items(): + for i in val: + all_embedding_keys += [i.embedding_key] + all_embedding_key_str = "" + for i in all_embedding_keys: + all_embedding_key_str += f"{i}\n" + + summarized_relationship = await AgentChatSumRel().run(init_role, target_role, all_embedding_key_str) + return summarized_relationship + + +async def generate_one_utterance(init_role, target_role, retrieved: dict, curr_chat: list) -> Tuple[str, str]: + # Chat version optimized for speed via batch generation + scratch = init_role.rc.scratch + target_scratch = target_role.rc.scratch + curr_context = ( + f"{scratch.name} " + + f"was {scratch.act_description} " + + f"when {scratch.name} " + + f"saw {target_scratch.name} " + + f"in the middle of {target_scratch.act_description}.\n" + ) + curr_context += f"{scratch.name} " + "is initiating a conversation with " + f"{target_scratch.name}." + + x = await GenIterChatUTT().run(init_role, target_role, retrieved, curr_context, curr_chat) + + return x["utterance"], x["end"] diff --git a/examples/st_game/plan/st_plan.py b/examples/st_game/plan/st_plan.py new file mode 100644 index 000000000..017b3e1fb --- /dev/null +++ b/examples/st_game/plan/st_plan.py @@ -0,0 +1,719 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : st' planning execution + +import datetime +import math +import random +from typing import Tuple, Union + +from metagpt.llm import LLM +from metagpt.logs import logger + +from ..actions.decide_to_talk import DecideToTalk +from ..actions.gen_action_details import GenActionDetails +from ..actions.gen_daily_schedule import GenDailySchedule +from ..actions.gen_hourly_schedule import GenHourlySchedule +from ..actions.new_decomp_schedule import NewDecompSchedule +from ..actions.summarize_conv import SummarizeConv +from ..actions.task_decomp import TaskDecomp +from ..actions.wake_up import WakeUp +from ..memory.retrieve import new_agent_retrieve +from ..plan.converse import agent_conversation +from ..utils.utils import get_embedding + + +async def plan(role: "STRole", roles: dict["STRole"], new_day: bool, retrieved: dict) -> str: + # PART 1: Generate the hourly schedule. + if new_day: + await _long_term_planning(role, new_day) + + # PART 2: If the current action has expired, we want to create a new plan. + act_check_finished = role.scratch.act_check_finished() + logger.info(f"Role: {role.name} act_check_finished is {act_check_finished}") + if act_check_finished: + await _determine_action(role) + + # PART 3: If you perceived an event that needs to be responded to (saw + # another role), and retrieved relevant information. + # Step 1: Retrieved may have multiple events represented in it. The first + # job here is to determine which of the events we want to focus + # on for the role. + # takes the form of a dictionary like this: + # dictionary {["curr_event"] = , + # ["events"] = [, ...], + # ["thoughts"] = [, ...]} + focused_event = False + if retrieved.keys(): + focused_event = _choose_retrieved(role.name, retrieved) + + # Step 2: Once we choose an event, we need to determine whether the + # role will take any actions for the perceived event. There are + # three possible modes of reaction returned by _should_react. + # a) "chat with {target_role.name}" + # b) "react" + # c) False + logger.info(f"Role: {role.name} focused_event: {focused_event}") + if focused_event: + reaction_mode = await _should_react(role, focused_event, roles) + logger.info(f"Role: {role.name} reaction_mode: {reaction_mode}") + if reaction_mode: + # If we do want to chat, then we generate conversation + if reaction_mode[:9] == "chat with": + await _chat_react(role, reaction_mode, roles) + elif reaction_mode[:4] == "wait": + await _wait_react(role, reaction_mode) + + # Step 3: Chat-related state clean up. + # If the persona is not chatting with anyone, we clean up any of the + # chat-related states here. + if role.rc.scratch.act_event[1] != "chat with": + role.rc.scratch.chatting_with = None + role.rc.scratch.chat = None + role.rc.scratch.chatting_end_time = None + # We want to make sure that the persona does not keep conversing with each + # other in an infinite loop. So, chatting_with_buffer maintains a form of + # buffer that makes the persona wait from talking to the same target + # immediately after chatting once. We keep track of the buffer value here. + curr_persona_chat_buffer = role.rc.scratch.chatting_with_buffer + for persona_name, buffer_count in curr_persona_chat_buffer.items(): + if persona_name != role.rc.scratch.chatting_with: + role.rc.scratch.chatting_with_buffer[persona_name] -= 1 + + return role.rc.scratch.act_address + + +def _choose_retrieved(role_name: str, retrieved: dict) -> Union[None, dict]: + """ + Retrieved elements have multiple core "curr_events". We need to choose one + event to which we are going to react to. We pick that event here. + Args: + role_name: Current role instance's name whose action we are determining. + retrieved: A dictionary of that were retrieved from the + the role's associative memory. This dictionary takes the + following form: + dictionary[event.description] = + {["curr_event"] = , + ["events"] = [, ...], + ["thoughts"] = [, ...] } + """ + # Once we are done with the reflection, we might want to build a more + # complex structure here. + + # We do not want to take self events... for now + copy_retrieved = retrieved.copy() + for event_desc, rel_ctx in copy_retrieved.items(): + curr_event = rel_ctx["curr_event"] + if curr_event.subject == role_name: + del retrieved[event_desc] + + # Always choose role first. + priority = [] + for event_desc, rel_ctx in retrieved.items(): + curr_event = rel_ctx["curr_event"] + if ":" not in curr_event.subject and curr_event.subject != role_name: + priority += [rel_ctx] + if priority: + return random.choice(priority) + + # Skip idle. + for event_desc, rel_ctx in retrieved.items(): + if "is idle" not in event_desc: + priority += [rel_ctx] + if priority: + return random.choice(priority) + return None + + +async def _should_react(role: "STRole", retrieved: dict, roles: dict): + """ + Determines what form of reaction the role should exihibit given the + retrieved values. + INPUT + role: Current <"STRole"> instance whose action we are determining. + retrieved: A dictionary of that were retrieved from the + the role's associative memory. This dictionary takes the + following form: + dictionary[event.description] = + {["curr_event"] = , + ["events"] = [, ...], + ["thoughts"] = [, ...] } + roles: A dictionary that contains all role names as keys, and the + <"STRole"> instance as values. + """ + + async def lets_talk(init_role: "STRole", target_role: "STRole", retrieved: dict): + if init_role.name == target_role.name: + logger.info(f"Role: {role.name} _should_react lets_talk meet same role, return False") + return False + + scratch = init_role.rc.scratch + target_scratch = target_role.rc.scratch + if ( + not target_scratch.act_address + or not target_scratch.act_description + or not scratch.act_address + or not scratch.act_description + ): + return False + + if "sleeping" in target_scratch.act_description or "sleeping" in scratch.act_description: + return False + + if scratch.curr_time.hour == 23: + return False + + if "" in target_scratch.act_address: + return False + + if target_scratch.chatting_with or scratch.chatting_with: + return False + + if target_role.name in scratch.chatting_with_buffer: + if scratch.chatting_with_buffer[target_role.name] > 0: + return False + + if await DecideToTalk().run(init_role, target_role, retrieved): + return True + + return False + + async def lets_react(init_role: "STRole", target_role: "STRole", retrieved: dict): + if init_role.name == target_role.name: + logger.info(f"Role: {role.name} _should_react lets_react meet same role, return False") + return False + + scratch = init_role.rc.scratch + target_scratch = target_role.rc.scratch + if ( + not target_scratch.act_address + or not target_scratch.act_description + or not scratch.act_address + or not scratch.act_description + ): + return False + + if "sleeping" in target_scratch.act_description or "sleeping" in scratch.act_description: + return False + + # return False + if scratch.curr_time.hour == 23: + return False + + if "waiting" in target_scratch.act_description: + return False + if scratch.planned_path == []: + return False + + if scratch.act_address != target_scratch.act_address: + return False + + react_mode = await DecideToTalk().run(init_role, target_role, retrieved) + + if react_mode == "1": + wait_until = ( + target_scratch.act_start_time + datetime.timedelta(minutes=target_scratch.act_duration - 1) + ).strftime("%B %d, %Y, %H:%M:%S") + return f"wait: {wait_until}" + elif react_mode == "2": + return False + return "do other things" + else: + return False # "keep" + + # If the role is chatting right now, default to no reaction + scratch = role.rc.scratch + if scratch.chatting_with: + return False + if "" in scratch.act_address: + return False + + # Recall that retrieved takes the following form: + # dictionary {["curr_event"] = } + curr_event = retrieved["curr_event"] + logger.info(f"Role: {role.name} _should_react curr_event.subject: {curr_event.subject}") + + if ":" not in curr_event.subject: + # this is a role event. + if await lets_talk(role, roles[curr_event.subject], retrieved): + return f"chat with {curr_event.subject}" + react_mode = await lets_react(role, roles[curr_event.subject], retrieved) + return react_mode + return False + + +async def _chat_react(role: "STRole", reaction_mode: str, roles: dict["STRole"]): + # There are two roles -- the role who is initiating the conversation + # and the role who is the target. We get the role instances here. + init_role = role + target_role = roles[reaction_mode[9:].strip()] + + # Actually creating the conversation here. + convo, duration_min = await generate_convo(init_role, target_role) # 2222 + convo_summary = await generate_convo_summary(convo) + inserted_act = convo_summary + inserted_act_dur = duration_min + + act_start_time = target_role.rc.scratch.act_start_time + + curr_time = target_role.rc.scratch.curr_time + if curr_time.second != 0: + temp_curr_time = curr_time + datetime.timedelta(seconds=60 - curr_time.second) + chatting_end_time = temp_curr_time + datetime.timedelta(minutes=inserted_act_dur) + else: + chatting_end_time = curr_time + datetime.timedelta(minutes=inserted_act_dur) + + for role, p in [("init", init_role), ("target", target_role)]: + if role == "init": + act_address = f" {target_role.name}" + act_event = (p.name, "chat with", target_role.name) + chatting_with = target_role.name + chatting_with_buffer = {} + chatting_with_buffer[target_role.name] = 800 + elif role == "target": + act_address = f" {init_role.name}" + act_event = (p.name, "chat with", init_role.name) + chatting_with = init_role.name + chatting_with_buffer = {} + chatting_with_buffer[init_role.name] = 800 + + act_pronunciatio = "💬" + act_obj_description = None + act_obj_pronunciatio = None + act_obj_event = (None, None, None) + + await _create_react( + p, + inserted_act, + inserted_act_dur, + act_address, + act_event, + chatting_with, + convo, + chatting_with_buffer, + chatting_end_time, + act_pronunciatio, + act_obj_description, + act_obj_pronunciatio, + act_obj_event, + act_start_time, + ) + + +async def _create_react( + role: "STRole", + inserted_act: str, + inserted_act_dur: int, + act_address: str, + act_event: Tuple, + chatting_with: str, + chat: list, + chatting_with_buffer: dict, + chatting_end_time: datetime, + act_pronunciatio: str, + act_obj_description: str, + act_obj_pronunciatio: str, + act_obj_event: Tuple, + act_start_time=None, +): + p = role + scratch = role.rc.scratch + + min_sum = 0 + for i in range(scratch.get_f_daily_schedule_hourly_org_index()): + min_sum += scratch.f_daily_schedule_hourly_org[i][1] + start_hour = int(min_sum / 60) + + if scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index()][1] >= 120: + end_hour = ( + start_hour + scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index()][1] / 60 + ) + + elif ( + scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index()][1] + + scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index() + 1][1] + ): + end_hour = start_hour + ( + ( + scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index()][1] + + scratch.f_daily_schedule_hourly_org[scratch.get_f_daily_schedule_hourly_org_index() + 1][1] + ) + / 60 + ) + + else: + end_hour = start_hour + 2 + end_hour = int(end_hour) + + dur_sum = 0 + count = 0 + start_index = None + end_index = None + for act, dur in scratch.f_daily_schedule: + if dur_sum >= start_hour * 60 and start_index is None: + start_index = count + if dur_sum >= end_hour * 60 and end_index is None: + end_index = count + dur_sum += dur + count += 1 + + ret = await generate_new_decomp_schedule(p, inserted_act, inserted_act_dur, start_hour, end_hour) + scratch.f_daily_schedule[start_index:end_index] = ret + scratch.add_new_action( + act_address, + inserted_act_dur, + inserted_act, + act_pronunciatio, + act_event, + chatting_with, + chat, + chatting_with_buffer, + chatting_end_time, + act_obj_description, + act_obj_pronunciatio, + act_obj_event, + act_start_time, + ) + + +async def _wait_react(role: "STRole", reaction_mode: str): + scratch = role.rc.scratch + + inserted_act = f'waiting to start {scratch.act_description.split("(")[-1][:-1]}' + end_time = datetime.datetime.strptime(reaction_mode[6:].strip(), "%B %d, %Y, %H:%M:%S") + inserted_act_dur = ( + (end_time.minute + end_time.hour * 60) - (scratch.curr_time.minute + scratch.curr_time.hour * 60) + 1 + ) + + act_address = f" {scratch.curr_tile[0]} {scratch.curr_tile[1]}" + act_event = (role.name, "waiting to start", scratch.act_description.split("(")[-1][:-1]) + chatting_with = None + chat = None + chatting_with_buffer = None + chatting_end_time = None + + act_pronunciatio = "⌛" + act_obj_description = None + act_obj_pronunciatio = None + act_obj_event = (None, None, None) + + await _create_react( + role, + inserted_act, + inserted_act_dur, + act_address, + act_event, + chatting_with, + chat, + chatting_with_buffer, + chatting_end_time, + act_pronunciatio, + act_obj_description, + act_obj_pronunciatio, + act_obj_event, + ) + + +async def generate_convo(init_role: "STRole", target_role: "STRole") -> Union[list, int]: + convo = await agent_conversation(init_role, target_role) + all_utt = "" + + for row in convo: + speaker = row[0] + utt = row[1] + all_utt += f"{speaker}: {utt}\n" + + convo_length = math.ceil(int(len(all_utt) / 8) / 30) + + return convo, convo_length + + +async def generate_convo_summary(conv: list[list[str]]) -> str: + conv_summary = await SummarizeConv().run(conv) + return conv_summary + + +async def generate_new_decomp_schedule( + role: "STRole", inserted_act: str, inserted_act_dur: int, start_hour: int, end_hour: int +): + # Step 1: Setting up the core variables for the function. + #

is the role whose schedule we are editing right now. + scratch = role.rc.scratch + # indicates the number of minutes that have passed today. + today_min_pass = int(scratch.curr_time.hour) * 60 + int(scratch.curr_time.minute) + 1 + + # Step 2: We need to create and . + main_act_dur = [] + truncated_act_dur = [] + dur_sum = 0 # duration sum + count = 0 # enumerate count + truncated_fin = False + + print("DEBUG::: ", scratch.name) + for act, dur in scratch.f_daily_schedule: + if (dur_sum >= start_hour * 60) and (dur_sum < end_hour * 60): + main_act_dur += [[act, dur]] + if dur_sum <= today_min_pass: + truncated_act_dur += [[act, dur]] + elif dur_sum > today_min_pass and not truncated_fin: + # We need to insert that last act, duration list like this one: + # e.g., ['wakes up and completes her morning routine (wakes up...)', 2] + truncated_act_dur += [[scratch.f_daily_schedule[count][0], dur_sum - today_min_pass]] + truncated_act_dur[-1][-1] -= ( + dur_sum - today_min_pass + ) # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass + 1) + print("DEBUG::: ", truncated_act_dur) + + # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass) + truncated_fin = True + dur_sum += dur + count += 1 + + main_act_dur = main_act_dur + + x = ( + truncated_act_dur[-1][0].split("(")[0].strip() + + " (on the way to " + + truncated_act_dur[-1][0].split("(")[-1][:-1] + + ")" + ) + truncated_act_dur[-1][0] = x + + if "(" in truncated_act_dur[-1][0]: + inserted_act = truncated_act_dur[-1][0].split("(")[0].strip() + " (" + inserted_act + ")" + + # To do inserted_act_dur+1 below is an important decision but I'm not sure + # if I understand the full extent of its implications. Might want to + # revisit. + truncated_act_dur += [[inserted_act, inserted_act_dur]] + start_time_hour = datetime.datetime(2022, 10, 31, 0, 0) + datetime.timedelta(hours=start_hour) + end_time_hour = datetime.datetime(2022, 10, 31, 0, 0) + datetime.timedelta(hours=end_hour) + + return await NewDecompSchedule().run( + role, main_act_dur, truncated_act_dur, start_time_hour, end_time_hour, inserted_act, inserted_act_dur + ) + + +async def _long_term_planning(role: "STRole", new_day: bool): + """ + Formulates the role's daily long-term plan if it is the start of a new + day. This basically has two components: first, we create the wake-up hour, + and second, we create the hourly schedule based on it. + INPUT + new_day: Indicates whether the current time signals a "First day", + "New day", or False (for neither). This is important because we + create the roles' long term planning on the new day. + """ + # We start by creating the wake up hour for the role. + wake_up_hour = await WakeUp().run(role) + wake_up_hour = int(wake_up_hour) + logger.info(f"Role: {role.name} long_term_planning, wake_up_hour: {wake_up_hour}") + + # When it is a new day, we start by creating the daily_req of the role. + # Note that the daily_req is a list of strings that describe the role's + # day in broad strokes. + if new_day == "First day": + # Bootstrapping the daily plan for the start of then generation: + # if this is the start of generation (so there is no previous day's + # daily requirement, or if we are on a new day, we want to create a new + # set of daily requirements. + role.scratch.daily_req = await GenDailySchedule().run(role, wake_up_hour) + logger.info(f"Role: {role.name} daily requirements: {role.scratch.daily_req}") + elif new_day == "New day": + revise_identity(role) + + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - TODO + # We need to create a new daily_req here... + role.scratch.daily_req = role.scratch.daily_req + + # Based on the daily_req, we create an hourly schedule for the role, + # which is a list of todo items with a time duration (in minutes) that + # add up to 24 hours. + role.scratch.f_daily_schedule = await GenHourlySchedule().run(role, wake_up_hour) + logger.info(f"Role: {role.name} f_daily_schedule: {role.scratch.f_daily_schedule}") + role.scratch.f_daily_schedule_hourly_org = role.scratch.f_daily_schedule[:] + + # Added March 4 -- adding plan to the memory. + thought = f"This is {role.scratch.name}'s plan for {role.scratch.curr_time.strftime('%A %B %d')}:" + for i in role.scratch.daily_req: + thought += f" {i}," + thought = thought[:-1] + "." + created = role.scratch.curr_time + expiration = role.scratch.curr_time + datetime.timedelta(days=30) + s, p, o = (role.scratch.name, "plan", role.scratch.curr_time.strftime("%A %B %d")) + keywords = set(["plan"]) + thought_poignancy = 5 + thought_embedding_pair = (thought, get_embedding(thought)) + role.a_mem.add_thought( + created, expiration, s, p, o, thought, keywords, thought_poignancy, thought_embedding_pair, None + ) + + # print("Sleeping for 20 seconds...") + # time.sleep(10) + # print("Done sleeping!") + + +async def _determine_action(role: "STRole"): + """ + Creates the next action sequence for the role. + The main goal of this function is to run "add_new_action" on the role's + scratch space, which sets up all the action related variables for the next + action. + As a part of this, the role may need to decompose its hourly schedule as + needed. + INPUT + role: Current instance whose action we are determining. + """ + + def determine_decomp(act_desp, act_dura): + """ + Given an action description and its duration, we determine whether we need + to decompose it. If the action is about the agent sleeping, we generally + do not want to decompose it, so that's what we catch here. + + INPUT: + act_desp: the description of the action (e.g., "sleeping") + act_dura: the duration of the action in minutes. + OUTPUT: + a boolean. True if we need to decompose, False otherwise. + """ + if "sleep" not in act_desp and "bed" not in act_desp: + return True + elif "sleeping" in act_desp or "asleep" in act_desp or "in bed" in act_desp: + return False + elif "sleep" in act_desp or "bed" in act_desp: + if act_dura > 60: + return False + return True + + # The goal of this function is to get us the action associated with + # . As a part of this, we may need to decompose some large + # chunk actions. + # Importantly, we try to decompose at least two hours worth of schedule at + # any given point. + curr_index = role.scratch.get_f_daily_schedule_index() + curr_index_60 = role.scratch.get_f_daily_schedule_index(advance=60) + + logger.info(f"f_daily_schedule: {role.scratch.f_daily_schedule}") + # * Decompose * + # During the first hour of the day, we need to decompose two hours + # sequence. We do that here. + if curr_index == 0: + # This portion is invoked if it is the first hour of the day. + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] + if act_dura >= 60: + # We decompose if the next action is longer than an hour, and fits the + # criteria described in determine_decomp. + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index : curr_index + 1] = await TaskDecomp().run(role, act_desp, act_dura) + if curr_index_60 + 1 < len(role.scratch.f_daily_schedule): + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60 + 1] + if act_dura >= 60: + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index_60 + 1 : curr_index_60 + 2] = await TaskDecomp().run( + role, act_desp, act_dura + ) + + if curr_index_60 < len(role.scratch.f_daily_schedule): + # If it is not the first hour of the day, this is always invoked (it is + # also invoked during the first hour of the day -- to double up so we can + # decompose two hours in one go). Of course, we need to have something to + # decompose as well, so we check for that too. + if role.scratch.curr_time.hour < 23: + # And we don't want to decompose after 11 pm. + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60] + if act_dura >= 60: + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index_60 : curr_index_60 + 1] = await TaskDecomp().run( + role, act_desp, act_dura + ) + # * End of Decompose * + + # Generate an instance from the action description and duration. By + # this point, we assume that all the relevant actions are decomposed and + # ready in f_daily_schedule. + print("DEBUG LJSDLFSKJF") + for i in role.scratch.f_daily_schedule: + print(i) + print(curr_index) + print(len(role.scratch.f_daily_schedule)) + print(role.scratch.name) + print("------") + + # 1440 + x_emergency = 0 + for i in role.scratch.f_daily_schedule: + x_emergency += i[1] + # print ("x_emergency", x_emergency) + + if 1440 - x_emergency > 0: + print("x_emergency__AAA", x_emergency) + role.scratch.f_daily_schedule += [["sleeping", 1440 - x_emergency]] + + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] + + new_action_details = await GenActionDetails().run(role, act_desp, act_dura) + # Adding the action to role's queue. + role.scratch.add_new_action(**new_action_details) + + +def revise_identity(role: "STRole"): + p_name = role.scratch.name + + focal_points = [ + f"{p_name}'s plan for {role.scratch.get_str_curr_date_str()}.", + f"Important recent events for {p_name}'s life.", + ] + retrieved = new_agent_retrieve(role, focal_points) + + statements = "[Statements]\n" + for key, val in retrieved.items(): + for i in val: + statements += f"{i.created.strftime('%A %B %d -- %H:%M %p')}: {i.embedding_key}\n" + + # print (";adjhfno;asdjao;idfjo;af", p_name) + plan_prompt = statements + "\n" + plan_prompt += f"Given the statements above, is there anything that {p_name} should remember as they plan for" + plan_prompt += f" *{role.scratch.curr_time.strftime('%A %B %d')}*? " + plan_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement)\n\n" + plan_prompt += f"Write the response from {p_name}'s perspective." + plan_note = LLM().ask(plan_prompt) + # print (plan_note) + + thought_prompt = statements + "\n" + thought_prompt += ( + f"Given the statements above, how might we summarize {p_name}'s feelings about their days up to now?\n\n" + ) + thought_prompt += f"Write the response from {p_name}'s perspective." + thought_note = LLM().ask(thought_prompt) + # print (thought_note) + + currently_prompt = ( + f"{p_name}'s status from {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" + ) + currently_prompt += f"{role.scratch.currently}\n\n" + currently_prompt += f"{p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" + currently_prompt += (plan_note + thought_note).replace("\n", "") + "\n\n" + currently_prompt += f"It is now {role.scratch.curr_time.strftime('%A %B %d')}. Given the above, write {p_name}'s status for {role.scratch.curr_time.strftime('%A %B %d')} that reflects {p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}. Write this in third-person talking about {p_name}." + currently_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement).\n\n" + currently_prompt += "Follow this format below:\nStatus: " + # print ("DEBUG ;adjhfno;asdjao;asdfsidfjo;af", p_name) + # print (currently_prompt) + new_currently = LLM().ask(currently_prompt) + # print (new_currently) + # print (new_currently[10:]) + + role.scratch.currently = new_currently + + daily_req_prompt = role.scratch.get_str_iss() + "\n" + daily_req_prompt += f"Today is {role.scratch.curr_time.strftime('%A %B %d')}. Here is {role.scratch.name}'s plan today in broad-strokes (with the time of the day. e.g., have a lunch at 12:00 pm, watch TV from 7 to 8 pm).\n\n" + daily_req_prompt += "Follow this format (the list should have 4~6 items but no more):\n" + daily_req_prompt += "1. wake up and complete the morning routine at

is the role whose schedule we are editing right now. + scratch = role.rc.scratch + # indicates the number of minutes that have passed today. + today_min_pass = int(scratch.curr_time.hour) * 60 + int(scratch.curr_time.minute) + 1 + + # Step 2: We need to create and . + main_act_dur = [] + truncated_act_dur = [] + dur_sum = 0 # duration sum + count = 0 # enumerate count + truncated_fin = False + + print("DEBUG::: ", scratch.name) + for act, dur in scratch.f_daily_schedule: + if (dur_sum >= start_hour * 60) and (dur_sum < end_hour * 60): + main_act_dur += [[act, dur]] + if dur_sum <= today_min_pass: + truncated_act_dur += [[act, dur]] + elif dur_sum > today_min_pass and not truncated_fin: + # We need to insert that last act, duration list like this one: + # e.g., ['wakes up and completes her morning routine (wakes up...)', 2] + truncated_act_dur += [[scratch.f_daily_schedule[count][0], dur_sum - today_min_pass]] + truncated_act_dur[-1][-1] -= ( + dur_sum - today_min_pass + ) # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass + 1) + print("DEBUG::: ", truncated_act_dur) + + # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass) + truncated_fin = True + dur_sum += dur + count += 1 + + main_act_dur = main_act_dur + + x = ( + truncated_act_dur[-1][0].split("(")[0].strip() + + " (on the way to " + + truncated_act_dur[-1][0].split("(")[-1][:-1] + + ")" + ) + truncated_act_dur[-1][0] = x + + if "(" in truncated_act_dur[-1][0]: + inserted_act = truncated_act_dur[-1][0].split("(")[0].strip() + " (" + inserted_act + ")" + + # To do inserted_act_dur+1 below is an important decision but I'm not sure + # if I understand the full extent of its implications. Might want to + # revisit. + truncated_act_dur += [[inserted_act, inserted_act_dur]] + start_time_hour = datetime.datetime(2022, 10, 31, 0, 0) + datetime.timedelta(hours=start_hour) + end_time_hour = datetime.datetime(2022, 10, 31, 0, 0) + datetime.timedelta(hours=end_hour) + + return await NewDecompSchedule().run( + role, main_act_dur, truncated_act_dur, start_time_hour, end_time_hour, inserted_act, inserted_act_dur + ) + + +async def _long_term_planning(role: "STRole", new_day: bool): + """ + Formulates the role's daily long-term plan if it is the start of a new + day. This basically has two components: first, we create the wake-up hour, + and second, we create the hourly schedule based on it. + INPUT + new_day: Indicates whether the current time signals a "First day", + "New day", or False (for neither). This is important because we + create the roles' long term planning on the new day. + """ + # We start by creating the wake up hour for the role. + wake_up_hour = await WakeUp().run(role) + wake_up_hour = int(wake_up_hour) + logger.info(f"Role: {role.name} long_term_planning, wake_up_hour: {wake_up_hour}") + + # When it is a new day, we start by creating the daily_req of the role. + # Note that the daily_req is a list of strings that describe the role's + # day in broad strokes. + if new_day == "First day": + # Bootstrapping the daily plan for the start of then generation: + # if this is the start of generation (so there is no previous day's + # daily requirement, or if we are on a new day, we want to create a new + # set of daily requirements. + role.scratch.daily_req = await GenDailySchedule().run(role, wake_up_hour) + logger.info(f"Role: {role.name} daily requirements: {role.scratch.daily_req}") + elif new_day == "New day": + revise_identity(role) + + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - TODO + # We need to create a new daily_req here... + role.scratch.daily_req = role.scratch.daily_req + + # Based on the daily_req, we create an hourly schedule for the role, + # which is a list of todo items with a time duration (in minutes) that + # add up to 24 hours. + role.scratch.f_daily_schedule = await GenHourlySchedule().run(role, wake_up_hour) + logger.info(f"Role: {role.name} f_daily_schedule: {role.scratch.f_daily_schedule}") + role.scratch.f_daily_schedule_hourly_org = role.scratch.f_daily_schedule[:] + + # Added March 4 -- adding plan to the memory. + thought = f"This is {role.scratch.name}'s plan for {role.scratch.curr_time.strftime('%A %B %d')}:" + for i in role.scratch.daily_req: + thought += f" {i}," + thought = thought[:-1] + "." + created = role.scratch.curr_time + expiration = role.scratch.curr_time + datetime.timedelta(days=30) + s, p, o = (role.scratch.name, "plan", role.scratch.curr_time.strftime("%A %B %d")) + keywords = set(["plan"]) + thought_poignancy = 5 + thought_embedding_pair = (thought, get_embedding(thought)) + role.a_mem.add_thought( + created, expiration, s, p, o, thought, keywords, thought_poignancy, thought_embedding_pair, None + ) + + # print("Sleeping for 20 seconds...") + # time.sleep(10) + # print("Done sleeping!") + + +async def _determine_action(role: "STRole"): + """ + Creates the next action sequence for the role. + The main goal of this function is to run "add_new_action" on the role's + scratch space, which sets up all the action related variables for the next + action. + As a part of this, the role may need to decompose its hourly schedule as + needed. + INPUT + role: Current instance whose action we are determining. + """ + + def determine_decomp(act_desp, act_dura): + """ + Given an action description and its duration, we determine whether we need + to decompose it. If the action is about the agent sleeping, we generally + do not want to decompose it, so that's what we catch here. + + INPUT: + act_desp: the description of the action (e.g., "sleeping") + act_dura: the duration of the action in minutes. + OUTPUT: + a boolean. True if we need to decompose, False otherwise. + """ + if "sleep" not in act_desp and "bed" not in act_desp: + return True + elif "sleeping" in act_desp or "asleep" in act_desp or "in bed" in act_desp: + return False + elif "sleep" in act_desp or "bed" in act_desp: + if act_dura > 60: + return False + return True + + # The goal of this function is to get us the action associated with + # . As a part of this, we may need to decompose some large + # chunk actions. + # Importantly, we try to decompose at least two hours worth of schedule at + # any given point. + curr_index = role.scratch.get_f_daily_schedule_index() + curr_index_60 = role.scratch.get_f_daily_schedule_index(advance=60) + + logger.info(f"f_daily_schedule: {role.scratch.f_daily_schedule}") + # * Decompose * + # During the first hour of the day, we need to decompose two hours + # sequence. We do that here. + if curr_index == 0: + # This portion is invoked if it is the first hour of the day. + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] + if act_dura >= 60: + # We decompose if the next action is longer than an hour, and fits the + # criteria described in determine_decomp. + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index : curr_index + 1] = await TaskDecomp().run(role, act_desp, act_dura) + if curr_index_60 + 1 < len(role.scratch.f_daily_schedule): + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60 + 1] + if act_dura >= 60: + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index_60 + 1 : curr_index_60 + 2] = await TaskDecomp().run( + role, act_desp, act_dura + ) + + if curr_index_60 < len(role.scratch.f_daily_schedule): + # If it is not the first hour of the day, this is always invoked (it is + # also invoked during the first hour of the day -- to double up so we can + # decompose two hours in one go). Of course, we need to have something to + # decompose as well, so we check for that too. + if role.scratch.curr_time.hour < 23: + # And we don't want to decompose after 11 pm. + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60] + if act_dura >= 60: + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index_60 : curr_index_60 + 1] = await TaskDecomp().run( + role, act_desp, act_dura + ) + # * End of Decompose * + + # Generate an instance from the action description and duration. By + # this point, we assume that all the relevant actions are decomposed and + # ready in f_daily_schedule. + print("DEBUG LJSDLFSKJF") + for i in role.scratch.f_daily_schedule: + print(i) + print(curr_index) + print(len(role.scratch.f_daily_schedule)) + print(role.scratch.name) + print("------") + + # 1440 + x_emergency = 0 + for i in role.scratch.f_daily_schedule: + x_emergency += i[1] + # print ("x_emergency", x_emergency) + + if 1440 - x_emergency > 0: + print("x_emergency__AAA", x_emergency) + role.scratch.f_daily_schedule += [["sleeping", 1440 - x_emergency]] + + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] + + new_action_details = await GenActionDetails().run(role, act_desp, act_dura) + # Adding the action to role's queue. + role.scratch.add_new_action(**new_action_details) + + +def revise_identity(role: "STRole"): + p_name = role.scratch.name + + focal_points = [ + f"{p_name}'s plan for {role.scratch.get_str_curr_date_str()}.", + f"Important recent events for {p_name}'s life.", + ] + retrieved = new_agent_retrieve(role, focal_points) + + statements = "[Statements]\n" + for key, val in retrieved.items(): + for i in val: + statements += f"{i.created.strftime('%A %B %d -- %H:%M %p')}: {i.embedding_key}\n" + + # print (";adjhfno;asdjao;idfjo;af", p_name) + plan_prompt = statements + "\n" + plan_prompt += f"Given the statements above, is there anything that {p_name} should remember as they plan for" + plan_prompt += f" *{role.scratch.curr_time.strftime('%A %B %d')}*? " + plan_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement)\n\n" + plan_prompt += f"Write the response from {p_name}'s perspective." + plan_note = LLM().ask(plan_prompt) + # print (plan_note) + + thought_prompt = statements + "\n" + thought_prompt += ( + f"Given the statements above, how might we summarize {p_name}'s feelings about their days up to now?\n\n" + ) + thought_prompt += f"Write the response from {p_name}'s perspective." + thought_note = LLM().ask(thought_prompt) + # print (thought_note) + + currently_prompt = ( + f"{p_name}'s status from {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" + ) + currently_prompt += f"{role.scratch.currently}\n\n" + currently_prompt += f"{p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" + currently_prompt += (plan_note + thought_note).replace("\n", "") + "\n\n" + currently_prompt += f"It is now {role.scratch.curr_time.strftime('%A %B %d')}. Given the above, write {p_name}'s status for {role.scratch.curr_time.strftime('%A %B %d')} that reflects {p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}. Write this in third-person talking about {p_name}." + currently_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement).\n\n" + currently_prompt += "Follow this format below:\nStatus: " + # print ("DEBUG ;adjhfno;asdjao;asdfsidfjo;af", p_name) + # print (currently_prompt) + new_currently = LLM().ask(currently_prompt) + # print (new_currently) + # print (new_currently[10:]) + + role.scratch.currently = new_currently + + daily_req_prompt = role.scratch.get_str_iss() + "\n" + daily_req_prompt += f"Today is {role.scratch.curr_time.strftime('%A %B %d')}. Here is {role.scratch.name}'s plan today in broad-strokes (with the time of the day. e.g., have a lunch at 12:00 pm, watch TV from 7 to 8 pm).\n\n" + daily_req_prompt += "Follow this format (the list should have 4~6 items but no more):\n" + daily_req_prompt += "1. wake up and complete the morning routine at

is the role whose schedule we are editing right now. + scratch = role.rc.scratch + # indicates the number of minutes that have passed today. + today_min_pass = int(scratch.curr_time.hour) * 60 + int(scratch.curr_time.minute) + 1 + + # Step 2: We need to create and . + main_act_dur = [] + truncated_act_dur = [] + dur_sum = 0 # duration sum + count = 0 # enumerate count + truncated_fin = False + + print("DEBUG::: ", scratch.name) + for act, dur in scratch.f_daily_schedule: + if (dur_sum >= start_hour * 60) and (dur_sum < end_hour * 60): + main_act_dur += [[act, dur]] + if dur_sum <= today_min_pass: + truncated_act_dur += [[act, dur]] + elif dur_sum > today_min_pass and not truncated_fin: + # We need to insert that last act, duration list like this one: + # e.g., ['wakes up and completes her morning routine (wakes up...)', 2] + truncated_act_dur += [[scratch.f_daily_schedule[count][0], dur_sum - today_min_pass]] + truncated_act_dur[-1][-1] -= ( + dur_sum - today_min_pass + ) # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass + 1) + print("DEBUG::: ", truncated_act_dur) + + # DEC 7 DEBUG;.. is the +1 the right thing to do??? + # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass) + truncated_fin = True + dur_sum += dur + count += 1 + + main_act_dur = main_act_dur + + x = ( + truncated_act_dur[-1][0].split("(")[0].strip() + + " (on the way to " + + truncated_act_dur[-1][0].split("(")[-1][:-1] + + ")" + ) + truncated_act_dur[-1][0] = x + + if "(" in truncated_act_dur[-1][0]: + inserted_act = truncated_act_dur[-1][0].split("(")[0].strip() + " (" + inserted_act + ")" + + # To do inserted_act_dur+1 below is an important decision but I'm not sure + # if I understand the full extent of its implications. Might want to + # revisit. + truncated_act_dur += [[inserted_act, inserted_act_dur]] + start_time_hour = datetime.datetime(2022, 10, 31, 0, 0) + datetime.timedelta(hours=start_hour) + end_time_hour = datetime.datetime(2022, 10, 31, 0, 0) + datetime.timedelta(hours=end_hour) + + return await NewDecompSchedule().run( + role, main_act_dur, truncated_act_dur, start_time_hour, end_time_hour, inserted_act, inserted_act_dur + ) + + +async def _long_term_planning(role: "STRole", new_day: bool): + """ + Formulates the role's daily long-term plan if it is the start of a new + day. This basically has two components: first, we create the wake-up hour, + and second, we create the hourly schedule based on it. + INPUT + new_day: Indicates whether the current time signals a "First day", + "New day", or False (for neither). This is important because we + create the roles' long term planning on the new day. + """ + # We start by creating the wake up hour for the role. + wake_up_hour = await WakeUp().run(role) + wake_up_hour = int(wake_up_hour) + logger.info(f"Role: {role.name} long_term_planning, wake_up_hour: {wake_up_hour}") + + # When it is a new day, we start by creating the daily_req of the role. + # Note that the daily_req is a list of strings that describe the role's + # day in broad strokes. + if new_day == "First day": + # Bootstrapping the daily plan for the start of then generation: + # if this is the start of generation (so there is no previous day's + # daily requirement, or if we are on a new day, we want to create a new + # set of daily requirements. + role.scratch.daily_req = await GenDailySchedule().run(role, wake_up_hour) + logger.info(f"Role: {role.name} daily requirements: {role.scratch.daily_req}") + elif new_day == "New day": + revise_identity(role) + + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - TODO + # We need to create a new daily_req here... + role.scratch.daily_req = role.scratch.daily_req + + # Based on the daily_req, we create an hourly schedule for the role, + # which is a list of todo items with a time duration (in minutes) that + # add up to 24 hours. + role.scratch.f_daily_schedule = await GenHourlySchedule().run(role, wake_up_hour) + logger.info(f"Role: {role.name} f_daily_schedule: {role.scratch.f_daily_schedule}") + role.scratch.f_daily_schedule_hourly_org = role.scratch.f_daily_schedule[:] + + # Added March 4 -- adding plan to the memory. + thought = f"This is {role.scratch.name}'s plan for {role.scratch.curr_time.strftime('%A %B %d')}:" + for i in role.scratch.daily_req: + thought += f" {i}," + thought = thought[:-1] + "." + created = role.scratch.curr_time + expiration = role.scratch.curr_time + datetime.timedelta(days=30) + s, p, o = (role.scratch.name, "plan", role.scratch.curr_time.strftime("%A %B %d")) + keywords = set(["plan"]) + thought_poignancy = 5 + thought_embedding_pair = (thought, get_embedding(thought)) + role.a_mem.add_thought( + created, expiration, s, p, o, thought, keywords, thought_poignancy, thought_embedding_pair, None + ) + + # print("Sleeping for 20 seconds...") + # time.sleep(10) + # print("Done sleeping!") + + +async def _determine_action(role: "STRole"): + """ + Creates the next action sequence for the role. + The main goal of this function is to run "add_new_action" on the role's + scratch space, which sets up all the action related variables for the next + action. + As a part of this, the role may need to decompose its hourly schedule as + needed. + INPUT + role: Current instance whose action we are determining. + """ + + def determine_decomp(act_desp, act_dura): + """ + Given an action description and its duration, we determine whether we need + to decompose it. If the action is about the agent sleeping, we generally + do not want to decompose it, so that's what we catch here. + + INPUT: + act_desp: the description of the action (e.g., "sleeping") + act_dura: the duration of the action in minutes. + OUTPUT: + a boolean. True if we need to decompose, False otherwise. + """ + if "sleep" not in act_desp and "bed" not in act_desp: + return True + elif "sleeping" in act_desp or "asleep" in act_desp or "in bed" in act_desp: + return False + elif "sleep" in act_desp or "bed" in act_desp: + if act_dura > 60: + return False + return True + + # The goal of this function is to get us the action associated with + # . As a part of this, we may need to decompose some large + # chunk actions. + # Importantly, we try to decompose at least two hours worth of schedule at + # any given point. + curr_index = role.scratch.get_f_daily_schedule_index() + curr_index_60 = role.scratch.get_f_daily_schedule_index(advance=60) + + logger.info(f"f_daily_schedule: {role.scratch.f_daily_schedule}") + # * Decompose * + # During the first hour of the day, we need to decompose two hours + # sequence. We do that here. + if curr_index == 0: + # This portion is invoked if it is the first hour of the day. + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] + if act_dura >= 60: + # We decompose if the next action is longer than an hour, and fits the + # criteria described in determine_decomp. + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index : curr_index + 1] = await TaskDecomp().run(role, act_desp, act_dura) + if curr_index_60 + 1 < len(role.scratch.f_daily_schedule): + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60 + 1] + if act_dura >= 60: + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index_60 + 1 : curr_index_60 + 2] = await TaskDecomp().run( + role, act_desp, act_dura + ) + + if curr_index_60 < len(role.scratch.f_daily_schedule): + # If it is not the first hour of the day, this is always invoked (it is + # also invoked during the first hour of the day -- to double up so we can + # decompose two hours in one go). Of course, we need to have something to + # decompose as well, so we check for that too. + if role.scratch.curr_time.hour < 23: + # And we don't want to decompose after 11 pm. + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60] + if act_dura >= 60: + if determine_decomp(act_desp, act_dura): + role.scratch.f_daily_schedule[curr_index_60 : curr_index_60 + 1] = await TaskDecomp().run( + role, act_desp, act_dura + ) + # * End of Decompose * + + # Generate an instance from the action description and duration. By + # this point, we assume that all the relevant actions are decomposed and + # ready in f_daily_schedule. + print("DEBUG LJSDLFSKJF") + for i in role.scratch.f_daily_schedule: + print(i) + print(curr_index) + print(len(role.scratch.f_daily_schedule)) + print(role.scratch.name) + print("------") + + # 1440 + x_emergency = 0 + for i in role.scratch.f_daily_schedule: + x_emergency += i[1] + # print ("x_emergency", x_emergency) + + if 1440 - x_emergency > 0: + print("x_emergency__AAA", x_emergency) + role.scratch.f_daily_schedule += [["sleeping", 1440 - x_emergency]] + + act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] + + new_action_details = await GenActionDetails().run(role, act_desp, act_dura) + # Adding the action to role's queue. + role.scratch.add_new_action(**new_action_details) + + +def revise_identity(role: "STRole"): + p_name = role.scratch.name + + focal_points = [ + f"{p_name}'s plan for {role.scratch.get_str_curr_date_str()}.", + f"Important recent events for {p_name}'s life.", + ] + retrieved = new_agent_retrieve(role, focal_points) + + statements = "[Statements]\n" + for key, val in retrieved.items(): + for i in val: + statements += f"{i.created.strftime('%A %B %d -- %H:%M %p')}: {i.embedding_key}\n" + + # print (";adjhfno;asdjao;idfjo;af", p_name) + plan_prompt = statements + "\n" + plan_prompt += f"Given the statements above, is there anything that {p_name} should remember as they plan for" + plan_prompt += f" *{role.scratch.curr_time.strftime('%A %B %d')}*? " + plan_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement)\n\n" + plan_prompt += f"Write the response from {p_name}'s perspective." + plan_note = LLM().ask(plan_prompt) + # print (plan_note) + + thought_prompt = statements + "\n" + thought_prompt += ( + f"Given the statements above, how might we summarize {p_name}'s feelings about their days up to now?\n\n" + ) + thought_prompt += f"Write the response from {p_name}'s perspective." + thought_note = LLM().ask(thought_prompt) + # print (thought_note) + + currently_prompt = ( + f"{p_name}'s status from {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" + ) + currently_prompt += f"{role.scratch.currently}\n\n" + currently_prompt += f"{p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" + currently_prompt += (plan_note + thought_note).replace("\n", "") + "\n\n" + currently_prompt += f"It is now {role.scratch.curr_time.strftime('%A %B %d')}. Given the above, write {p_name}'s status for {role.scratch.curr_time.strftime('%A %B %d')} that reflects {p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}. Write this in third-person talking about {p_name}." + currently_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement).\n\n" + currently_prompt += "Follow this format below:\nStatus: " + # print ("DEBUG ;adjhfno;asdjao;asdfsidfjo;af", p_name) + # print (currently_prompt) + new_currently = LLM().ask(currently_prompt) + # print (new_currently) + # print (new_currently[10:]) + + role.scratch.currently = new_currently + + daily_req_prompt = role.scratch.get_str_iss() + "\n" + daily_req_prompt += f"Today is {role.scratch.curr_time.strftime('%A %B %d')}. Here is {role.scratch.name}'s plan today in broad-strokes (with the time of the day. e.g., have a lunch at 12:00 pm, watch TV from 7 to 8 pm).\n\n" + daily_req_prompt += "Follow this format (the list should have 4~6 items but no more):\n" + daily_req_prompt += "1. wake up and complete the morning routine at

is the role whose schedule we are editing right now. - scratch = role.rc.scratch - # indicates the number of minutes that have passed today. - today_min_pass = int(scratch.curr_time.hour) * 60 + int(scratch.curr_time.minute) + 1 - - # Step 2: We need to create and . - main_act_dur = [] - truncated_act_dur = [] - dur_sum = 0 # duration sum - count = 0 # enumerate count - truncated_fin = False - - print("DEBUG::: ", scratch.name) - for act, dur in scratch.f_daily_schedule: - if (dur_sum >= start_hour * 60) and (dur_sum < end_hour * 60): - main_act_dur += [[act, dur]] - if dur_sum <= today_min_pass: - truncated_act_dur += [[act, dur]] - elif dur_sum > today_min_pass and not truncated_fin: - # We need to insert that last act, duration list like this one: - # e.g., ['wakes up and completes her morning routine (wakes up...)', 2] - truncated_act_dur += [[scratch.f_daily_schedule[count][0], dur_sum - today_min_pass]] - truncated_act_dur[-1][-1] -= ( - dur_sum - today_min_pass - ) # DEC 7 DEBUG;.. is the +1 the right thing to do??? - # DEC 7 DEBUG;.. is the +1 the right thing to do??? - # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass + 1) - print("DEBUG::: ", truncated_act_dur) - - # DEC 7 DEBUG;.. is the +1 the right thing to do??? - # truncated_act_dur[-1][-1] -= (dur_sum - today_min_pass) - truncated_fin = True - dur_sum += dur - count += 1 - - main_act_dur = main_act_dur - - x = ( - truncated_act_dur[-1][0].split("(")[0].strip() - + " (on the way to " - + truncated_act_dur[-1][0].split("(")[-1][:-1] - + ")" - ) - truncated_act_dur[-1][0] = x - - if "(" in truncated_act_dur[-1][0]: - inserted_act = truncated_act_dur[-1][0].split("(")[0].strip() + " (" + inserted_act + ")" - - # To do inserted_act_dur+1 below is an important decision but I'm not sure - # if I understand the full extent of its implications. Might want to - # revisit. - truncated_act_dur += [[inserted_act, inserted_act_dur]] - start_time_hour = datetime.datetime(2022, 10, 31, 0, 0) + datetime.timedelta(hours=start_hour) - end_time_hour = datetime.datetime(2022, 10, 31, 0, 0) + datetime.timedelta(hours=end_hour) - - return await NewDecompSchedule().run( - role, main_act_dur, truncated_act_dur, start_time_hour, end_time_hour, inserted_act, inserted_act_dur - ) - - -async def _long_term_planning(role: "STRole", new_day: bool): - """ - Formulates the role's daily long-term plan if it is the start of a new - day. This basically has two components: first, we create the wake-up hour, - and second, we create the hourly schedule based on it. - INPUT - new_day: Indicates whether the current time signals a "First day", - "New day", or False (for neither). This is important because we - create the roles' long term planning on the new day. - """ - # We start by creating the wake up hour for the role. - wake_up_hour = await WakeUp().run(role) - wake_up_hour = int(wake_up_hour) - logger.info(f"Role: {role.name} long_term_planning, wake_up_hour: {wake_up_hour}") - - # When it is a new day, we start by creating the daily_req of the role. - # Note that the daily_req is a list of strings that describe the role's - # day in broad strokes. - if new_day == "First day": - # Bootstrapping the daily plan for the start of then generation: - # if this is the start of generation (so there is no previous day's - # daily requirement, or if we are on a new day, we want to create a new - # set of daily requirements. - role.scratch.daily_req = await GenDailySchedule().run(role, wake_up_hour) - logger.info(f"Role: {role.name} daily requirements: {role.scratch.daily_req}") - elif new_day == "New day": - revise_identity(role) - - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - TODO - # We need to create a new daily_req here... - role.scratch.daily_req = role.scratch.daily_req - - # Based on the daily_req, we create an hourly schedule for the role, - # which is a list of todo items with a time duration (in minutes) that - # add up to 24 hours. - role.scratch.f_daily_schedule = await GenHourlySchedule().run(role, wake_up_hour) - logger.info(f"Role: {role.name} f_daily_schedule: {role.scratch.f_daily_schedule}") - role.scratch.f_daily_schedule_hourly_org = role.scratch.f_daily_schedule[:] - - # Added March 4 -- adding plan to the memory. - thought = f"This is {role.scratch.name}'s plan for {role.scratch.curr_time.strftime('%A %B %d')}:" - for i in role.scratch.daily_req: - thought += f" {i}," - thought = thought[:-1] + "." - created = role.scratch.curr_time - expiration = role.scratch.curr_time + datetime.timedelta(days=30) - s, p, o = (role.scratch.name, "plan", role.scratch.curr_time.strftime("%A %B %d")) - keywords = set(["plan"]) - thought_poignancy = 5 - thought_embedding_pair = (thought, get_embedding(thought)) - role.a_mem.add_thought( - created, expiration, s, p, o, thought, keywords, thought_poignancy, thought_embedding_pair, None - ) - - # print("Sleeping for 20 seconds...") - # time.sleep(10) - # print("Done sleeping!") - - -async def _determine_action(role: "STRole"): - """ - Creates the next action sequence for the role. - The main goal of this function is to run "add_new_action" on the role's - scratch space, which sets up all the action related variables for the next - action. - As a part of this, the role may need to decompose its hourly schedule as - needed. - INPUT - role: Current instance whose action we are determining. - """ - - def determine_decomp(act_desp, act_dura): - """ - Given an action description and its duration, we determine whether we need - to decompose it. If the action is about the agent sleeping, we generally - do not want to decompose it, so that's what we catch here. - - INPUT: - act_desp: the description of the action (e.g., "sleeping") - act_dura: the duration of the action in minutes. - OUTPUT: - a boolean. True if we need to decompose, False otherwise. - """ - if "sleep" not in act_desp and "bed" not in act_desp: - return True - elif "sleeping" in act_desp or "asleep" in act_desp or "in bed" in act_desp: - return False - elif "sleep" in act_desp or "bed" in act_desp: - if act_dura > 60: - return False - return True - - # The goal of this function is to get us the action associated with - # . As a part of this, we may need to decompose some large - # chunk actions. - # Importantly, we try to decompose at least two hours worth of schedule at - # any given point. - curr_index = role.scratch.get_f_daily_schedule_index() - curr_index_60 = role.scratch.get_f_daily_schedule_index(advance=60) - - logger.info(f"f_daily_schedule: {role.scratch.f_daily_schedule}") - # * Decompose * - # During the first hour of the day, we need to decompose two hours - # sequence. We do that here. - if curr_index == 0: - # This portion is invoked if it is the first hour of the day. - act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] - if act_dura >= 60: - # We decompose if the next action is longer than an hour, and fits the - # criteria described in determine_decomp. - if determine_decomp(act_desp, act_dura): - role.scratch.f_daily_schedule[curr_index : curr_index + 1] = await TaskDecomp().run( - role, act_desp, act_dura - ) - if curr_index_60 + 1 < len(role.scratch.f_daily_schedule): - act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60 + 1] - if act_dura >= 60: - if determine_decomp(act_desp, act_dura): - role.scratch.f_daily_schedule[curr_index_60 + 1 : curr_index_60 + 2] = await TaskDecomp().run( - role, act_desp, act_dura - ) - - if curr_index_60 < len(role.scratch.f_daily_schedule): - # If it is not the first hour of the day, this is always invoked (it is - # also invoked during the first hour of the day -- to double up so we can - # decompose two hours in one go). Of course, we need to have something to - # decompose as well, so we check for that too. - if role.scratch.curr_time.hour < 23: - # And we don't want to decompose after 11 pm. - act_desp, act_dura = role.scratch.f_daily_schedule[curr_index_60] - if act_dura >= 60: - if determine_decomp(act_desp, act_dura): - role.scratch.f_daily_schedule[curr_index_60 : curr_index_60 + 1] = await TaskDecomp().run( - role, act_desp, act_dura - ) - # * End of Decompose * - - # Generate an instance from the action description and duration. By - # this point, we assume that all the relevant actions are decomposed and - # ready in f_daily_schedule. - print("DEBUG LJSDLFSKJF") - for i in role.scratch.f_daily_schedule: - print(i) - print(curr_index) - print(len(role.scratch.f_daily_schedule)) - print(role.scratch.name) - print("------") - - # 1440 - x_emergency = 0 - for i in role.scratch.f_daily_schedule: - x_emergency += i[1] - # print ("x_emergency", x_emergency) - - if 1440 - x_emergency > 0: - print("x_emergency__AAA", x_emergency) - role.scratch.f_daily_schedule += [["sleeping", 1440 - x_emergency]] - - act_desp, act_dura = role.scratch.f_daily_schedule[curr_index] - - new_action_details = await GenActionDetails().run(role, act_desp, act_dura) - # Adding the action to role's queue. - role.scratch.add_new_action(**new_action_details) - - -def revise_identity(role: "STRole"): - p_name = role.scratch.name - - focal_points = [ - f"{p_name}'s plan for {role.scratch.get_str_curr_date_str()}.", - f"Important recent events for {p_name}'s life.", - ] - retrieved = new_agent_retrieve(role, focal_points) - - statements = "[Statements]\n" - for key, val in retrieved.items(): - for i in val: - statements += f"{i.created.strftime('%A %B %d -- %H:%M %p')}: {i.embedding_key}\n" - - # print (";adjhfno;asdjao;idfjo;af", p_name) - plan_prompt = statements + "\n" - plan_prompt += f"Given the statements above, is there anything that {p_name} should remember as they plan for" - plan_prompt += f" *{role.scratch.curr_time.strftime('%A %B %d')}*? " - plan_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement)\n\n" - plan_prompt += f"Write the response from {p_name}'s perspective." - plan_note = LLM().ask(plan_prompt) - # print (plan_note) - - thought_prompt = statements + "\n" - thought_prompt += ( - f"Given the statements above, how might we summarize {p_name}'s feelings about their days up to now?\n\n" - ) - thought_prompt += f"Write the response from {p_name}'s perspective." - thought_note = LLM().ask(thought_prompt) - # print (thought_note) - - currently_prompt = ( - f"{p_name}'s status from {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" - ) - currently_prompt += f"{role.scratch.currently}\n\n" - currently_prompt += f"{p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}:\n" - currently_prompt += (plan_note + thought_note).replace("\n", "") + "\n\n" - currently_prompt += f"It is now {role.scratch.curr_time.strftime('%A %B %d')}. Given the above, write {p_name}'s status for {role.scratch.curr_time.strftime('%A %B %d')} that reflects {p_name}'s thoughts at the end of {(role.scratch.curr_time - datetime.timedelta(days=1)).strftime('%A %B %d')}. Write this in third-person talking about {p_name}." - currently_prompt += "If there is any scheduling information, be as specific as possible (include date, time, and location if stated in the statement).\n\n" - currently_prompt += "Follow this format below:\nStatus: " - # print ("DEBUG ;adjhfno;asdjao;asdfsidfjo;af", p_name) - # print (currently_prompt) - new_currently = LLM().ask(currently_prompt) - # print (new_currently) - # print (new_currently[10:]) - - role.scratch.currently = new_currently - - daily_req_prompt = role.scratch.get_str_iss() + "\n" - daily_req_prompt += f"Today is {role.scratch.curr_time.strftime('%A %B %d')}. Here is {role.scratch.name}'s plan today in broad-strokes (with the time of the day. e.g., have a lunch at 12:00 pm, watch TV from 7 to 8 pm).\n\n" - daily_req_prompt += "Follow this format (the list should have 4~6 items but no more):\n" - daily_req_prompt += "1. wake up and complete the morning routine at