From 02c7c4ea47d7db109d9f0bd4a5d6c1c2d2027834 Mon Sep 17 00:00:00 2001 From: didi <84363704+didiforgithub@users.noreply.github.com> Date: Fri, 23 Aug 2024 14:21:04 +0800 Subject: [PATCH] =?UTF-8?q?Update=20=E6=B7=B7=E4=B9=B1=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/ags/w_action_node/evaluator.py | 2 +- examples/ags/w_action_node/graph.py | 2 +- examples/ags/w_action_node/operator_an.py | 4 ++++ .../{graphs => optimized}/gsm8k/basic/graph.py | 0 .../{graphs => optimized}/gsm8k/basic/operator.py | 0 .../{graphs => optimized}/gsm8k/basic/prompt.py | 0 .../gsm8k/graphs}/round_1/graph.py | 0 .../gsm8k/graphs}/round_1/operator.py | 0 .../gsm8k/graphs}/round_1/operator_an.py | 0 .../gsm8k/graphs}/round_1/prompt.py | 0 .../{graphs => optimized}/gsm8k/handcraft/graph.py | 0 .../{graphs => optimized}/hotpotqa/basic/graph.py | 0 .../hotpotqa/basic/operator.py | 0 .../{graphs => optimized}/hotpotqa/basic/prompt.py | 0 .../{graphs => optimized}/hotpotqa/graph.py | 0 .../{graphs => optimized}/humaneval/basic/graph.py | 0 .../humaneval/basic/operator.py | 0 .../humaneval/basic/prompt.py | 0 .../{graphs => optimized}/humaneval/graph.py | 0 examples/ags/w_action_node/optimizer.py | 14 +++++++++++--- 20 files changed, 17 insertions(+), 5 deletions(-) rename examples/ags/w_action_node/{graphs => optimized}/gsm8k/basic/graph.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/gsm8k/basic/operator.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/gsm8k/basic/prompt.py (100%) rename examples/ags/w_action_node/{graphs/gsm8k/optimized => optimized/gsm8k/graphs}/round_1/graph.py (100%) rename examples/ags/w_action_node/{graphs/gsm8k/optimized => optimized/gsm8k/graphs}/round_1/operator.py (100%) rename examples/ags/w_action_node/{graphs/gsm8k/optimized => optimized/gsm8k/graphs}/round_1/operator_an.py (100%) rename examples/ags/w_action_node/{graphs/gsm8k/optimized => optimized/gsm8k/graphs}/round_1/prompt.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/gsm8k/handcraft/graph.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/hotpotqa/basic/graph.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/hotpotqa/basic/operator.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/hotpotqa/basic/prompt.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/hotpotqa/graph.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/humaneval/basic/graph.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/humaneval/basic/operator.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/humaneval/basic/prompt.py (100%) rename examples/ags/w_action_node/{graphs => optimized}/humaneval/graph.py (100%) diff --git a/examples/ags/w_action_node/evaluator.py b/examples/ags/w_action_node/evaluator.py index 75875c5b5..42e2a7d96 100644 --- a/examples/ags/w_action_node/evaluator.py +++ b/examples/ags/w_action_node/evaluator.py @@ -6,7 +6,7 @@ from typing import Literal # TODO 完成实验数据集的手动划分 -DatasetType = Literal["humaneval", "gsm8k", "hotpotqa", "drop", "mmlu"] +DatasetType = Literal["HumanEval", "MMBP", "Gsm8K", "MATH", "HotpotQa", "MMLU"] class Evaluator: diff --git a/examples/ags/w_action_node/graph.py b/examples/ags/w_action_node/graph.py index 4b6286192..24865a2a5 100644 --- a/examples/ags/w_action_node/graph.py +++ b/examples/ags/w_action_node/graph.py @@ -8,7 +8,7 @@ from typing import Literal from metagpt.llm import LLM from metagpt.utils.cost_manager import CostManager -DatasetType = Literal["humaneval", "gsm8k", "hotpotqa", "drop", "mmlu"] +DatasetType = Literal["HumanEval", "MMBP", "Gsm8K", "MATH", "HotpotQa", "MMLU"] cost_manager = CostManager() diff --git a/examples/ags/w_action_node/operator_an.py b/examples/ags/w_action_node/operator_an.py index 30ef27986..d5e47db4d 100644 --- a/examples/ags/w_action_node/operator_an.py +++ b/examples/ags/w_action_node/operator_an.py @@ -86,3 +86,7 @@ class ReflectionTestOp(BaseModel): refined_solution: str = Field( default="", description="Corrective solution for code execution errors or test case failures" ) + + +class Optimize(BaseModel): + graph: str = Field(default="", description="graph") diff --git a/examples/ags/w_action_node/graphs/gsm8k/basic/graph.py b/examples/ags/w_action_node/optimized/gsm8k/basic/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/basic/graph.py rename to examples/ags/w_action_node/optimized/gsm8k/basic/graph.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/basic/operator.py b/examples/ags/w_action_node/optimized/gsm8k/basic/operator.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/basic/operator.py rename to examples/ags/w_action_node/optimized/gsm8k/basic/operator.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/basic/prompt.py b/examples/ags/w_action_node/optimized/gsm8k/basic/prompt.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/basic/prompt.py rename to examples/ags/w_action_node/optimized/gsm8k/basic/prompt.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/graph.py b/examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/graph.py rename to examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/graph.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/operator.py b/examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/operator.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/operator.py rename to examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/operator.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/operator_an.py b/examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/operator_an.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/operator_an.py rename to examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/operator_an.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/prompt.py b/examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/prompt.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/prompt.py rename to examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/prompt.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/handcraft/graph.py b/examples/ags/w_action_node/optimized/gsm8k/handcraft/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/handcraft/graph.py rename to examples/ags/w_action_node/optimized/gsm8k/handcraft/graph.py diff --git a/examples/ags/w_action_node/graphs/hotpotqa/basic/graph.py b/examples/ags/w_action_node/optimized/hotpotqa/basic/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/hotpotqa/basic/graph.py rename to examples/ags/w_action_node/optimized/hotpotqa/basic/graph.py diff --git a/examples/ags/w_action_node/graphs/hotpotqa/basic/operator.py b/examples/ags/w_action_node/optimized/hotpotqa/basic/operator.py similarity index 100% rename from examples/ags/w_action_node/graphs/hotpotqa/basic/operator.py rename to examples/ags/w_action_node/optimized/hotpotqa/basic/operator.py diff --git a/examples/ags/w_action_node/graphs/hotpotqa/basic/prompt.py b/examples/ags/w_action_node/optimized/hotpotqa/basic/prompt.py similarity index 100% rename from examples/ags/w_action_node/graphs/hotpotqa/basic/prompt.py rename to examples/ags/w_action_node/optimized/hotpotqa/basic/prompt.py diff --git a/examples/ags/w_action_node/graphs/hotpotqa/graph.py b/examples/ags/w_action_node/optimized/hotpotqa/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/hotpotqa/graph.py rename to examples/ags/w_action_node/optimized/hotpotqa/graph.py diff --git a/examples/ags/w_action_node/graphs/humaneval/basic/graph.py b/examples/ags/w_action_node/optimized/humaneval/basic/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/humaneval/basic/graph.py rename to examples/ags/w_action_node/optimized/humaneval/basic/graph.py diff --git a/examples/ags/w_action_node/graphs/humaneval/basic/operator.py b/examples/ags/w_action_node/optimized/humaneval/basic/operator.py similarity index 100% rename from examples/ags/w_action_node/graphs/humaneval/basic/operator.py rename to examples/ags/w_action_node/optimized/humaneval/basic/operator.py diff --git a/examples/ags/w_action_node/graphs/humaneval/basic/prompt.py b/examples/ags/w_action_node/optimized/humaneval/basic/prompt.py similarity index 100% rename from examples/ags/w_action_node/graphs/humaneval/basic/prompt.py rename to examples/ags/w_action_node/optimized/humaneval/basic/prompt.py diff --git a/examples/ags/w_action_node/graphs/humaneval/graph.py b/examples/ags/w_action_node/optimized/humaneval/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/humaneval/graph.py rename to examples/ags/w_action_node/optimized/humaneval/graph.py diff --git a/examples/ags/w_action_node/optimizer.py b/examples/ags/w_action_node/optimizer.py index a88f6baa3..6b6ed942a 100644 --- a/examples/ags/w_action_node/optimizer.py +++ b/examples/ags/w_action_node/optimizer.py @@ -21,6 +21,11 @@ DatasetType = Literal["HumanEval", "MMBP", "Gsm8K", "MATH", "HotpotQa", "MMLU"] evaluator = Evaluator(eval_path="eval") +# prompt = GENERATE_PROMPT.format(problem_description=problem_description) +# node = await ActionNode.from_pydantic(GenerateOp).fill(context=prompt, mode="context_fill", llm=self.llm) +# response = node.instruct_content.model_dump() +# return response + class Optimizer: def __init__(self, dataset: DatasetType, llm: LLM, operators: List, optimized_path: str = None) -> None: @@ -28,6 +33,7 @@ class Optimizer: self.dataset = dataset self.graph = None # 初始化为 None，稍后加载 self.operators = operators + self.optimize_prompt = "" self._optimized_path = optimized_path self.root_path = f"{self._optimized_path}/{self.dataset}" self.sample = 6 # sample 含义是什么？ @@ -37,9 +43,10 @@ class Optimizer: def _initialize(self): """ - 基于数据集、操作符初始化 operator 跟 graph + 基于数据集、操作符初始化optimize prompt, operator 跟 graph """ - basic_path = f"{config_iterate_path}/{self.dataset}/basic" + + basic_path = f"{self.root_path}/basic" required_files = ["operator.py", "graph.py", "prompt.py"] def check_files_exist(basic_path, required_files): @@ -104,6 +111,7 @@ class Optimizer: 动态读取指定轮次的 Prompt和Graph。 """ # 构建 prompt.py 文件的相对路径 + # examples/ags/w_action_node/optimized/gsm8k/graphs/round_1 prompt_file_path = os.path.join(graphs_path, "prompt.py") graph_file_path = os.path.join(graphs_path, "graph.py") @@ -228,7 +236,7 @@ class Optimizer: """ # TODO 读取basic模版（从对应的dataset文件夹 {dataset}/basic/operator.py, graph.py, prompt.py ），Operator几乎不用动 # TODO 动Prompt内容；动Graph连接 - graph_path = f"{self.root_path}/graphs" + graph_path = f"{self._optimized_path}/{self.dataset}/graphs" f"{graph_path}/round_{self.round + 1}" # TODO 填充Optimize 逻辑