apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-05-27 14:25:20 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

update aug result summarization

Browse source
This commit is contained in:
Yizhou Chi 2024-09-24 20:48:44 +08:00
parent 0a9ab5cd1b
commit 6344046c31
2 changed files with 18 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

6
expo/experimenter/aug.py
View file

@ -49,9 +49,5 @@ class AugExperimenter(Experimenter):
"args": vars(self.args),
}
)
scores = [result["score_dict"]["test_score"] for result in results]
avg_score = sum(scores) / len(scores)
best_score = max(scores) if not self.args.low_is_better else min(scores)
best_score_idx = scores.index(best_score)
results.insert(0, {"avg_score": avg_score, "best_score": best_score, "best_score_idx": best_score_idx})
results = self.summarize_results(results)
self.save_result(results)

29
expo/experimenter/experimenter.py
View file

@ -47,18 +47,7 @@ class Experimenter:
score_dict = {"train_score": -1, "dev_score": -1, "test_score": -1, "score": -1}
return score_dict
async def run_experiment(self):
state = self.state
user_requirement = state["requirement"]
results = []
for i in range(self.args.num_experiments):
di = ResearchAssistant(node_id="0", use_reflection=self.args.reflection)
score_dict = await self.run_di(di, user_requirement, run_idx=i)
results.append(
{"idx": i, "score_dict": score_dict, "user_requirement": user_requirement, "args": vars(self.args)}
)
self.save_result(results) # save intermediate results
def summarize_results(self, results):
dev_scores = [result["score_dict"]["dev_score"] for result in results]
best_dev_score = (
max(dev_scores)
@ -85,6 +74,22 @@ class Experimenter:
"global_best_test_score": global_best_score,
},
)
return results
async def run_experiment(self):
state = self.state
user_requirement = state["requirement"]
results = []
for i in range(self.args.num_experiments):
di = ResearchAssistant(node_id="0", use_reflection=self.args.reflection)
score_dict = await self.run_di(di, user_requirement, run_idx=i)
results.append(
{"idx": i, "score_dict": score_dict, "user_requirement": user_requirement, "args": vars(self.args)}
)
self.save_result(results) # save intermediate results
results = self.summarize_results(results)
self.save_result(results)
def evaluate_prediction(self, split, state):