From 1047415a9b916869814c051e3def8fa71dd36ebe Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 9 Oct 2023 17:21:24 +0800 Subject: [PATCH] add plan unittest and update logs --- examples/st_game/actions/st_action.py | 6 +-- examples/st_game/plan/converse.py | 15 +++--- examples/st_game/plan/st_plan.py | 10 ++-- examples/st_game/roles/st_role.py | 42 +++++++++------ examples/st_game/run_st_game.py | 2 +- examples/st_game/stanford_town.py | 16 ++++++ examples/st_game/tests/plan/__init__.py | 3 ++ examples/st_game/tests/plan/test_converse.py | 55 ++++++++++++++++++++ examples/st_game/tests/plan/test_st_plan.py | 36 +++++++++++++ 9 files changed, 152 insertions(+), 33 deletions(-) create mode 100644 examples/st_game/tests/plan/__init__.py create mode 100644 examples/st_game/tests/plan/test_converse.py create mode 100644 examples/st_game/tests/plan/test_st_plan.py diff --git a/examples/st_game/actions/st_action.py b/examples/st_game/actions/st_action.py index 231d2f85d..d9274abf4 100644 --- a/examples/st_game/actions/st_action.py +++ b/examples/st_game/actions/st_action.py @@ -82,7 +82,7 @@ class STAction(Action): CONFIG.max_tokens_rsp = tmp_max_tokens_rsp self.llm.model = tmp_model_name - logger.info(f"llm _run_text_davinci raw resp: {llm_resp}") + logger.info(f"Action: {self.cls_name} llm _run_text_davinci raw resp: {llm_resp}") if self._func_validate(llm_resp, prompt): return self._func_cleanup(llm_resp, prompt) return self.fail_default_resp @@ -101,7 +101,7 @@ class STAction(Action): for idx in range(retry): try: llm_resp = self._ask(prompt) - logger.info(f"llm _run_gpt35 raw resp: {llm_resp}") + logger.info(f"Action: {self.cls_name} llm _run_gpt35 raw resp: {llm_resp}") end_idx = llm_resp.strip().rfind("}") + 1 llm_resp = llm_resp[:end_idx] llm_resp = json.loads(llm_resp)["output"] @@ -118,7 +118,7 @@ class STAction(Action): for idx in range(retry): try: llm_resp = self._ask(prompt).strip() - logger.info(f"llm _run_gpt35_wo_extra_prompt raw resp: {llm_resp}") + logger.info(f"Action: {self.cls_name} llm _run_gpt35_wo_extra_prompt raw resp: {llm_resp}") if self._func_validate(llm_resp, prompt): return self._func_cleanup(llm_resp, prompt) except Exception as exp: diff --git a/examples/st_game/plan/converse.py b/examples/st_game/plan/converse.py index e598e792c..2b2877a7b 100644 --- a/examples/st_game/plan/converse.py +++ b/examples/st_game/plan/converse.py @@ -12,19 +12,20 @@ from examples.st_game.actions.agent_chat_sum_rel import AgentChatSumRel from examples.st_game.actions.gen_iter_chat_utt import GenIterChatUTT -def agent_conversation(maze: Maze, init_role: "STRole", target_role: "STRole") -> list[str]: +def agent_conversation(maze: Maze, init_role: "STRole", target_role: "STRole") -> list[list[str]]: curr_chat = [] logger.info(f"Role: {init_role.name} starts a conversation with Role: {target_role.name}") conv_rounds = 8 for idx in range(conv_rounds): + logger.info(f"Conv round: {idx} between {init_role.name} and {target_role.name}") scratch = init_role._rc.scratch target_scratch = target_role._rc.scratch focal_points = [f"{target_scratch.name}"] retrieved = new_agent_retrieve(init_role, focal_points, 50) relationship = generate_summarize_agent_relationship(init_role, target_role, retrieved) - print("-------- relationship: ", relationship) + logger.info(f"The relationship between {init_role.name} and {target_role.name}: {relationship}") last_chat = "" for i in curr_chat[-4:]: last_chat += ": ".join(i) + "\n" @@ -45,7 +46,7 @@ def agent_conversation(maze: Maze, init_role: "STRole", target_role: "STRole") - focal_points = [f"{scratch.name}"] retrieved = new_agent_retrieve(target_role, focal_points, 50) relationship = generate_summarize_agent_relationship(target_role, init_role, retrieved) - print("-------- relationship: ", relationship) + logger.info(f"The relationship between {target_role.name} and {init_role.name}: {relationship}") last_chat = "" for i in curr_chat[-4:]: last_chat += ": ".join(i) + "\n" @@ -63,10 +64,9 @@ def agent_conversation(maze: Maze, init_role: "STRole", target_role: "STRole") - if end: break - print("July 23 PU") + logger.warning(f"Conversations between {target_role.name} and {init_role.name}:") for row in curr_chat: - print(row) - print("July 23 FIN") + logger.info(row) return curr_chat @@ -101,9 +101,6 @@ def generate_one_utterance(maze: Maze, init_role, target_role, retrieved: dict, f"is initiating a conversation with " + f"{target_scratch.name}.") - print("July 23 5") x = GenIterChatUTT().run(maze, init_role, target_role, retrieved, curr_context, curr_chat) - print("July 23 6") - return x["utterance"], x["end"] diff --git a/examples/st_game/plan/st_plan.py b/examples/st_game/plan/st_plan.py index dfdf484fb..82c4269e1 100644 --- a/examples/st_game/plan/st_plan.py +++ b/examples/st_game/plan/st_plan.py @@ -23,7 +23,7 @@ from ..utils.utils import get_embedding from ..memory.retrieve import new_agent_retrieve -def plan(role: "STRole", maze: Maze, roles: list["STRole"], new_day: bool, retrieved: dict) -> str: +def plan(role: "STRole", maze: Maze, roles: dict["STRole"], new_day: bool, retrieved: dict) -> str: # PART 1: Generate the hourly schedule. if new_day: _long_term_planning(role, new_day) @@ -52,6 +52,7 @@ def plan(role: "STRole", maze: Maze, roles: list["STRole"], new_day: bool, retri # a) "chat with {target_role.name}" # b) "react" # c) False + logger.info(f"Role: {role.name} focused_event: {focused_event}") if focused_event: reaction_mode = _should_react(role, focused_event, roles) logger.info(f"Role: {role.name} reaction_mode: {reaction_mode}") @@ -235,7 +236,7 @@ def _should_react(role: "STRole", retrieved: dict, roles: dict): return False -def _chat_react(maze: Maze, role: "STRole", reaction_mode: str, roles: list["STRole"]): +def _chat_react(maze: Maze, role: "STRole", reaction_mode: str, roles: dict["STRole"]): # There are two roles -- the role who is initiating the conversation # and the role who is the target. We get the role instances here. init_role = role @@ -244,7 +245,7 @@ def _chat_react(maze: Maze, role: "STRole", reaction_mode: str, roles: list["STR # Actually creating the conversation here. convo, duration_min = generate_convo(maze, init_role, target_role) # 2222 - convo_summary = generate_convo_summary(init_role, convo) + convo_summary = generate_convo_summary(convo) inserted_act = convo_summary inserted_act_dur = duration_min @@ -366,7 +367,6 @@ def _wait_react(role: "STRole", reaction_mode: str): def generate_convo(maze: Maze, init_role: "STRole", target_role: "STRole") -> Union[list, int]: - curr_loc = maze.access_tile(init_role._rc.scratch.curr_tile) convo = agent_conversation(maze, init_role, target_role) all_utt = "" @@ -380,7 +380,7 @@ def generate_convo(maze: Maze, init_role: "STRole", target_role: "STRole") -> Un return convo, convo_length -def generate_convo_summary(role: "STRole", conv: list) -> str: +def generate_convo_summary(conv: list[list[str]]) -> str: conv_summary = SummarizeConv().run(conv) return conv_summary diff --git a/examples/st_game/roles/st_role.py b/examples/st_game/roles/st_role.py index 62b74d453..5132f62f2 100644 --- a/examples/st_game/roles/st_role.py +++ b/examples/st_game/roles/st_role.py @@ -71,16 +71,10 @@ class STRole(Role): super(STRole, self).__init__(name=name, profile=profile) + + self.role_storage_path = STORAGE_PATH.joinpath(f"{sim_code}/personas/{self.name}") self._rc = STRoleContext() - memory_saved = str(STORAGE_PATH.joinpath(f"{sim_code}/personas/{self.name}/" - f"bootstrap_memory/associative_memory")) - self._rc.memory.set_mem_path(memory_saved) - sp_mem_saved = str(STORAGE_PATH.joinpath(f"{sim_code}/personas/{self.name}/" - f"bootstrap_memory/spatial_memory.json")) - self._rc.spatial_memory.set_mem_path(f_saved=sp_mem_saved) - scratch_f_saved = str(STORAGE_PATH.joinpath(f"{sim_code}/personas/{self.name}/" - f"bootstrap_memory/scratch.json")) - self._rc.scratch.set_scratch_path(f_saved=scratch_f_saved) + self.load_from() # load role's memory self._init_actions([]) @@ -121,17 +115,35 @@ class STRole(Role): def memory(self): return self._rc.memory - def load_from(self, folder: Path): + def load_from(self): """ - load role data from `storage/{simulation_name}/personas/{role_name} + load role data from `storage/{simulation_name}/personas/{role_name}` """ - pass + memory_saved = str(self.role_storage_path.joinpath("bootstrap_memory/associative_memory")) + self._rc.memory.set_mem_path(memory_saved) - def save_into(self, folder: Path): + sp_mem_saved = str(self.role_storage_path.joinpath("bootstrap_memory/spatial_memory.json")) + self._rc.spatial_memory.set_mem_path(f_saved=sp_mem_saved) + + scratch_f_saved = str(self.role_storage_path.joinpath("bootstrap_memory/scratch.json")) + self._rc.scratch.set_scratch_path(f_saved=scratch_f_saved) + + logger.info(f"Role: {self.name} loaded role's memory from {str(self.role_storage_path)}") + + def save_into(self): """ - save role data from `storage/{simulation_name}/personas/{role_name} + save role data from `storage/{simulation_name}/personas/{role_name}` """ - pass + memory_saved = str(self.role_storage_path.joinpath("bootstrap_memory/associative_memory")) + self._rc.memory.save(memory_saved) + + sp_mem_saved = str(self.role_storage_path.joinpath("bootstrap_memory/spatial_memory.json")) + self._rc.spatial_memory.save(sp_mem_saved) + + scratch_f_saved = str(self.role_storage_path.joinpath("bootstrap_memory/scratch.json")) + self._rc.scratch.save(scratch_f_saved) + + logger.info(f"Role: {self.name} saved role's memory into {str(self.role_storage_path)}") async def _observe(self) -> int: if not self._rc.env: diff --git a/examples/st_game/run_st_game.py b/examples/st_game/run_st_game.py index d36ff92ca..af98da50c 100644 --- a/examples/st_game/run_st_game.py +++ b/examples/st_game/run_st_game.py @@ -30,7 +30,7 @@ async def startup(idea: str, has_inner_voice = True if idx == 0 else False role = STRole(name=role_name, sim_code=sim_code, - profile=f"STMember_{idx}", + profile=role_name, step=reverie_meta.get("step", 0), start_date=reverie_meta.get("start_date"), curr_time=reverie_meta.get("curr_time"), diff --git a/examples/st_game/stanford_town.py b/examples/st_game/stanford_town.py index d259dbc36..c565afd80 100644 --- a/examples/st_game/stanford_town.py +++ b/examples/st_game/stanford_town.py @@ -25,3 +25,19 @@ class StanfordTown(SoftwareCompany): self.environment.publish_message( Message(role="User", content=idea, cause_by=UserRequirement) ) + + async def run(self, n_round: int = 3): + """Run company until target round or no money""" + while n_round > 0: + # self._save() + n_round -= 1 + logger.debug(f"{n_round=}") + self._check_balance() + await self.environment.run() + + # save simulation result including environment and roles after all rounds + roles = self.environment.get_roles() + for profile, role in roles.items(): + role.save_into() + + return self.environment.history diff --git a/examples/st_game/tests/plan/__init__.py b/examples/st_game/tests/plan/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/examples/st_game/tests/plan/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/examples/st_game/tests/plan/test_converse.py b/examples/st_game/tests/plan/test_converse.py new file mode 100644 index 000000000..a39452334 --- /dev/null +++ b/examples/st_game/tests/plan/test_converse.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : unittest of roles conversation + +from typing import Tuple + +from examples.st_game.roles.st_role import STRole +from examples.st_game.maze import Maze +from examples.st_game.utils.const import STORAGE_PATH +from examples.st_game.utils.mg_ga_transform import get_reverie_meta +from examples.st_game.utils.utils import copy_folder +from examples.st_game.plan.converse import agent_conversation + + +def init_two_roles(fork_sim_code: str = "July1_the_ville_isabella_maria_klaus-step-3-8") -> Tuple["STRole"]: + sim_code = "unittest_sim" + + copy_folder(str(STORAGE_PATH.joinpath(fork_sim_code)), str(STORAGE_PATH.joinpath(sim_code))) + + reverie_meta = get_reverie_meta(fork_sim_code) + role_ir_name = "Isabella Rodriguez" + role_km_name = "Klaus Mueller" + + role_ir = STRole(name=role_ir_name, + sim_code=sim_code, + profile=role_ir_name, + step=reverie_meta.get("step"), + start_date=reverie_meta.get("start_date"), + curr_time=reverie_meta.get("curr_time"), + sec_per_step=reverie_meta.get("sec_per_step")) + + role_km = STRole(name=role_km_name, + sim_code=sim_code, + profile=role_km_name, + step=reverie_meta.get("step"), + start_date=reverie_meta.get("start_date"), + curr_time=reverie_meta.get("curr_time"), + sec_per_step=reverie_meta.get("sec_per_step")) + + return role_ir, role_km + + +def test_agent_conversation(): + role_ir, role_km = init_two_roles() + + maze = Maze() + curr_chat = agent_conversation(maze, role_ir, role_km) + assert len(curr_chat) % 2 == 0 + + meet = False + for conv in curr_chat: + if "Valentine's Day party" in conv[1]: + # conv[0] speaker, conv[1] utterance + meet = True + assert meet diff --git a/examples/st_game/tests/plan/test_st_plan.py b/examples/st_game/tests/plan/test_st_plan.py new file mode 100644 index 000000000..26901d35d --- /dev/null +++ b/examples/st_game/tests/plan/test_st_plan.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : unittest of st_plan + +from examples.st_game.plan.st_plan import _should_react, _choose_retrieved, _wait_react + +from examples.st_game.tests.plan.test_converse import init_two_roles + + +def test_should_react(): + role_ir, role_km = init_two_roles() + roles = { + role_ir.name: role_ir, + role_km.name: role_km + } + + observed = role_ir.observe() + retrieved = role_ir.retrieve(observed) + + focused_event = _choose_retrieved(role_ir, retrieved) + + reaction_mode = _should_react(role_ir, focused_event, roles) # chat with Isabella Rodriguez + assert "chat with" in reaction_mode + + +def test_wait_react(): + role_ir, role_km = init_two_roles("July1_the_ville_isabella_maria_klaus-step-3-1") + reaction_mode = "wait: February 13, 2023, 00:01:30" + f_daily_schedule = role_ir.scratch.f_daily_schedule + # [['sleeping', 360], ['waking up and completing her morning routine (getting out of bed)', 5], ['sleeping', 180]] + + _wait_react(role_ir, reaction_mode) + new_f_daily_schedule = role_ir.scratch.f_daily_schedule + # [['sleeping', 360], ['waking up and completing her morning routine (getting out of bed)', 5], + # ['waking up and completing her morning routine (brushing her teeth)', 5], ['sleeping', 180]] + assert len(f_daily_schedule) == len(new_f_daily_schedule)