From ef81791f3268a519751e7ca0aad3b5d902a187ee Mon Sep 17 00:00:00 2001 From: Aria F <51890782+ariafyy@users.noreply.github.com> Date: Mon, 2 Oct 2023 13:42:31 +0800 Subject: [PATCH 1/4] =?UTF-8?q?#=20feat=EF=BC=9Aconvert=20txt=20file=20to?= =?UTF-8?q?=20json=20for=20evals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/werewolf_game/evals/utils.py | 123 ++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 examples/werewolf_game/evals/utils.py diff --git a/examples/werewolf_game/evals/utils.py b/examples/werewolf_game/evals/utils.py new file mode 100644 index 000000000..8705c1992 --- /dev/null +++ b/examples/werewolf_game/evals/utils.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- + + +""" +__info："utils for eval" +__author:"[Aria](https://github.com/ariafyy)" +__update："convert txt to json" +""" +from metagpt.const import WORKSPACE_ROOT, PROJECT_ROOT +import re, json + + +class Utils(object): + def __init__(self): + pass + + def _action(self, text: str) -> str: + """ + # get action + input: I vote to eliminate Player3 + output: vote + """ + text = text.lower() + if "vote" in text: + action = "vote" + return action + if "verify" in text: + action = "verify" + return action + if "kill" in text: + action = "kill" + return action + else: + action = "chat" + return action + + def _life(self, text: str) -> str: + """ + # get life + input: Kill Player6 + output: dead + """ + text = text.lower() + if re.search(r'\b(eliminated|killed|kill)\b', text, re.I): + life = 'dead' + dead_role = re.findall(r'\[(.*?)\]', text) + if re.search(r'no one was killed', text, re.I): + return "alive", [] + else: + return life, dead_role + else: + life = "alive" + return life, [] + + def txt2data(self, file): + """ + input: .txt file + output: data for json format + """ + result = {} + count = 0 + day = -1 + flag = False + + with open(file, "r") as f: + lines = f.readlines() + for line in lines: + if "Moderator(Moderator): 0" in line: + flag = True + if flag: + if "It’s dark, everyone close your eyes." in line: + day += 1 + count = 0 + data = {} + parts = line.split("|") + data["role"] = parts[0].strip().split(":")[0] + data["day"] = day + data["turn"] = count + if len(parts) > 1: + data["text"] = parts[1].strip() + data["action"] = self._action(data["text"]) + data["life"], data["dead_role"] = self._life(data["text"]) + else: + continue + key = "day_{}".format(day) + if key not in result: + result[key] = [] + result[key].append(data) + count += 1 + return result + + def data2json(self, in_file): + """ + input:.txt + output:.json + + + output examples: + { + "day_0": [ + { + "role": "Moderator(Moderator)", + "day": 0, + "turn": 0, + "text": "It’s dark, everyone close your eyes. I will talk with you/your team secretly at night.", + "action": "chat", + "life": "alive", + "dead_role": [] + },{}] + ... + } + """ + result = self.txt2data(in_file) + with open(WORKSPACE_ROOT / 'werewolf_transcript.json', "w", encoding='utf-8') as f: + json.dump(result, f, ensure_ascii=False, indent=2) + f.write('\n') + + +if __name__ == '__main__': + txt_path = WORKSPACE_ROOT / "werewolf_transcript.txt" + log_path = PROJECT_ROOT / "logs/log.txt" + Utils().data2json(txt_path) From 26ca9641bf291f9cba08ad4bdca390479efaff3e Mon Sep 17 00:00:00 2001 From: Aria F <51890782+ariafyy@users.noreply.github.com> Date: Thu, 5 Oct 2023 12:21:30 +0800 Subject: [PATCH 2/4] # fix: merge comments from mg --- examples/werewolf_game/evals/utils.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/examples/werewolf_game/evals/utils.py b/examples/werewolf_game/evals/utils.py index 8705c1992..2e83b38f6 100644 --- a/examples/werewolf_game/evals/utils.py +++ b/examples/werewolf_game/evals/utils.py @@ -1,17 +1,13 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- - - -""" -__info："utils for eval" -__author:"[Aria](https://github.com/ariafyy)" -__update："convert txt to json" -""" +''' +Filename: MetaGPT/examples/werewolf_game/evals/utils.py +Created Date: Oct 2, 2023 +Author: [Aria](https://github.com/ariafyy) +''' from metagpt.const import WORKSPACE_ROOT, PROJECT_ROOT import re, json -class Utils(object): +class Utils: def __init__(self): pass @@ -92,10 +88,6 @@ class Utils(object): def data2json(self, in_file): """ - input:.txt - output:.json - - output examples: { "day_0": [ @@ -111,9 +103,14 @@ class Utils(object): ... } """ + result = self.txt2data(in_file) + self._save_json(result) + return result + + def _save_json(self, data): with open(WORKSPACE_ROOT / 'werewolf_transcript.json', "w", encoding='utf-8') as f: - json.dump(result, f, ensure_ascii=False, indent=2) + json.dump(data, f, ensure_ascii=False, indent=2) f.write('\n') From c55608cb7691a3e2d95605ecf4762889742859e4 Mon Sep 17 00:00:00 2001 From: Aria F <51890782+ariafyy@users.noreply.github.com> Date: Tue, 10 Oct 2023 20:29:19 +0800 Subject: [PATCH 3/4] feat: polish log files for evals --- examples/werewolf_game/evals/utils.py | 148 +++++++++----------------- 1 file changed, 48 insertions(+), 100 deletions(-) diff --git a/examples/werewolf_game/evals/utils.py b/examples/werewolf_game/evals/utils.py index 2e83b38f6..ac2eecdf6 100644 --- a/examples/werewolf_game/evals/utils.py +++ b/examples/werewolf_game/evals/utils.py @@ -1,120 +1,68 @@ ''' Filename: MetaGPT/examples/werewolf_game/evals/utils.py -Created Date: Oct 2, 2023 +Created Date: Oct 10, 2023 Author: [Aria](https://github.com/ariafyy) ''' from metagpt.const import WORKSPACE_ROOT, PROJECT_ROOT -import re, json +import re class Utils: + """Utils: utils of logs""" + def __init__(self): pass - def _action(self, text: str) -> str: - """ - # get action - input: I vote to eliminate Player3 - output: vote - """ - text = text.lower() - if "vote" in text: - action = "vote" - return action - if "verify" in text: - action = "verify" - return action - if "kill" in text: - action = "kill" - return action - else: - action = "chat" - return action + def polish_log(self, in_logfile, out_txtfile): + """polish logs for evaluation""" + pattern_text = r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \| (\w+) +\| ([\w\.]+:\w+:\d+) - (.*\S)" + pattern_player = r"(Player(\d{1}): \w+)" + pattern_start = False + json_start = False - def _life(self, text: str) -> str: - """ - # get life - input: Kill Player6 - output: dead - """ - text = text.lower() - if re.search(r'\b(eliminated|killed|kill)\b', text, re.I): - life = 'dead' - dead_role = re.findall(r'\[(.*?)\]', text) - if re.search(r'no one was killed', text, re.I): - return "alive", [] - else: - return life, dead_role - else: - life = "alive" - return life, [] + with open(in_logfile, "r") as f, open(out_txtfile, "w") as out: + for line in f.readlines(): + matches = re.match(pattern_text, line) + if matches: + message = matches.group(4).strip() + pattern_start = True + json_start = False - def txt2data(self, file): - """ - input: .txt file - output: data for json format - """ - result = {} - count = 0 - day = -1 - flag = False - - with open(file, "r") as f: - lines = f.readlines() - for line in lines: - if "Moderator(Moderator): 0" in line: - flag = True - if flag: - if "It’s dark, everyone close your eyes." in line: - day += 1 - count = 0 - data = {} - parts = line.split("|") - data["role"] = parts[0].strip().split(":")[0] - data["day"] = day - data["turn"] = count - if len(parts) > 1: - data["text"] = parts[1].strip() - data["action"] = self._action(data["text"]) - data["life"], data["dead_role"] = self._life(data["text"]) + if "Moderator(Moderator) ready to InstructSpeak" not in message and "Moderator(Moderator) ready to ParseSpeak" not in message and "Total running cost:" not in message: + out.write("- " + message + '\n') else: - continue - key = "day_{}".format(day) - if key not in result: - result[key] = [] - result[key].append(data) - count += 1 - return result + out.write('\n') - def data2json(self, in_file): - """ - output examples: - { - "day_0": [ - { - "role": "Moderator(Moderator)", - "day": 0, - "turn": 0, - "text": "It’s dark, everyone close your eyes. I will talk with you/your team secretly at night.", - "action": "chat", - "life": "alive", - "dead_role": [] - },{}] - ... - } - """ + elif pattern_start and not matches: + if "gpt-4 may update over time" in line: + line = "" + out.write(line) - result = self.txt2data(in_file) - self._save_json(result) - return result + elif line.strip().startswith("{"): + out.write(line.strip()) + json_start = True - def _save_json(self, data): - with open(WORKSPACE_ROOT / 'werewolf_transcript.json', "w", encoding='utf-8') as f: - json.dump(data, f, ensure_ascii=False, indent=2) - f.write('\n') + elif json_start and not line.strip().endswith("}"): + out.write(line.strip()) + + elif json_start and line.strip().endswith("}"): + out.write(line.strip()) + json_start = False + + elif line.startswith("(User):"): + out.write(line) + + elif line.startswith("********** STEP:"): + out.write(line) + + elif re.search(pattern_player, line): + out.write(line) + + else: + out.write("\n") if __name__ == '__main__': - txt_path = WORKSPACE_ROOT / "werewolf_transcript.txt" - log_path = PROJECT_ROOT / "logs/log.txt" - Utils().data2json(txt_path) + in_logfile = PROJECT_ROOT / "logs/log.txt" + out_txtfile = "input your wish path" + Utils().polish_log(in_logfile, out_txtfile) From 6f641d63107f82071beb19899504e0cbbacf19d5 Mon Sep 17 00:00:00 2001 From: Aria F <51890782+ariafyy@users.noreply.github.com> Date: Wed, 11 Oct 2023 10:45:58 +0800 Subject: [PATCH 4/4] #refactor: refactor Utils class for log parsing --- examples/werewolf_game/evals/utils.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/examples/werewolf_game/evals/utils.py b/examples/werewolf_game/evals/utils.py index ac2eecdf6..d788496a3 100644 --- a/examples/werewolf_game/evals/utils.py +++ b/examples/werewolf_game/evals/utils.py @@ -1,6 +1,6 @@ ''' Filename: MetaGPT/examples/werewolf_game/evals/utils.py -Created Date: Oct 10, 2023 +Created Date: Oct 11, 2023 Author: [Aria](https://github.com/ariafyy) ''' from metagpt.const import WORKSPACE_ROOT, PROJECT_ROOT @@ -10,10 +10,8 @@ import re class Utils: """Utils: utils of logs""" - def __init__(self): - pass - - def polish_log(self, in_logfile, out_txtfile): + @staticmethod + def polish_log(in_logfile, out_txtfile): """polish logs for evaluation""" pattern_text = r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \| (\w+) +\| ([\w\.]+:\w+:\d+) - (.*\S)" pattern_player = r"(Player(\d{1}): \w+)" @@ -49,13 +47,7 @@ class Utils: out.write(line.strip()) json_start = False - elif line.startswith("(User):"): - out.write(line) - - elif line.startswith("********** STEP:"): - out.write(line) - - elif re.search(pattern_player, line): + elif line.startswith("(User):") or line.startswith("********** STEP:") or re.search(pattern_player,line): out.write(line) else: