diff --git a/examples/ags/w_action_node/evaluator.py b/examples/ags/w_action_node/evaluator.py index 75875c5b5..42e2a7d96 100644 --- a/examples/ags/w_action_node/evaluator.py +++ b/examples/ags/w_action_node/evaluator.py @@ -6,7 +6,7 @@ from typing import Literal # TODO 完成实验数据集的手动划分 -DatasetType = Literal["humaneval", "gsm8k", "hotpotqa", "drop", "mmlu"] +DatasetType = Literal["HumanEval", "MMBP", "Gsm8K", "MATH", "HotpotQa", "MMLU"] class Evaluator: diff --git a/examples/ags/w_action_node/graph.py b/examples/ags/w_action_node/graph.py index 4b6286192..24865a2a5 100644 --- a/examples/ags/w_action_node/graph.py +++ b/examples/ags/w_action_node/graph.py @@ -8,7 +8,7 @@ from typing import Literal from metagpt.llm import LLM from metagpt.utils.cost_manager import CostManager -DatasetType = Literal["humaneval", "gsm8k", "hotpotqa", "drop", "mmlu"] +DatasetType = Literal["HumanEval", "MMBP", "Gsm8K", "MATH", "HotpotQa", "MMLU"] cost_manager = CostManager() diff --git a/examples/ags/w_action_node/operator_an.py b/examples/ags/w_action_node/operator_an.py index 30ef27986..d5e47db4d 100644 --- a/examples/ags/w_action_node/operator_an.py +++ b/examples/ags/w_action_node/operator_an.py @@ -86,3 +86,7 @@ class ReflectionTestOp(BaseModel): refined_solution: str = Field( default="", description="Corrective solution for code execution errors or test case failures" ) + + +class Optimize(BaseModel): + graph: str = Field(default="", description="graph") diff --git a/examples/ags/w_action_node/graphs/gsm8k/basic/graph.py b/examples/ags/w_action_node/optimized/gsm8k/basic/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/basic/graph.py rename to examples/ags/w_action_node/optimized/gsm8k/basic/graph.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/basic/operator.py b/examples/ags/w_action_node/optimized/gsm8k/basic/operator.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/basic/operator.py rename to examples/ags/w_action_node/optimized/gsm8k/basic/operator.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/basic/prompt.py b/examples/ags/w_action_node/optimized/gsm8k/basic/prompt.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/basic/prompt.py rename to examples/ags/w_action_node/optimized/gsm8k/basic/prompt.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/graph.py b/examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/graph.py rename to examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/graph.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/operator.py b/examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/operator.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/operator.py rename to examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/operator.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/operator_an.py b/examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/operator_an.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/operator_an.py rename to examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/operator_an.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/prompt.py b/examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/prompt.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/optimized/round_1/prompt.py rename to examples/ags/w_action_node/optimized/gsm8k/graphs/round_1/prompt.py diff --git a/examples/ags/w_action_node/graphs/gsm8k/handcraft/graph.py b/examples/ags/w_action_node/optimized/gsm8k/handcraft/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/gsm8k/handcraft/graph.py rename to examples/ags/w_action_node/optimized/gsm8k/handcraft/graph.py diff --git a/examples/ags/w_action_node/graphs/hotpotqa/basic/graph.py b/examples/ags/w_action_node/optimized/hotpotqa/basic/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/hotpotqa/basic/graph.py rename to examples/ags/w_action_node/optimized/hotpotqa/basic/graph.py diff --git a/examples/ags/w_action_node/graphs/hotpotqa/basic/operator.py b/examples/ags/w_action_node/optimized/hotpotqa/basic/operator.py similarity index 100% rename from examples/ags/w_action_node/graphs/hotpotqa/basic/operator.py rename to examples/ags/w_action_node/optimized/hotpotqa/basic/operator.py diff --git a/examples/ags/w_action_node/graphs/hotpotqa/basic/prompt.py b/examples/ags/w_action_node/optimized/hotpotqa/basic/prompt.py similarity index 100% rename from examples/ags/w_action_node/graphs/hotpotqa/basic/prompt.py rename to examples/ags/w_action_node/optimized/hotpotqa/basic/prompt.py diff --git a/examples/ags/w_action_node/graphs/hotpotqa/graph.py b/examples/ags/w_action_node/optimized/hotpotqa/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/hotpotqa/graph.py rename to examples/ags/w_action_node/optimized/hotpotqa/graph.py diff --git a/examples/ags/w_action_node/graphs/humaneval/basic/graph.py b/examples/ags/w_action_node/optimized/humaneval/basic/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/humaneval/basic/graph.py rename to examples/ags/w_action_node/optimized/humaneval/basic/graph.py diff --git a/examples/ags/w_action_node/graphs/humaneval/basic/operator.py b/examples/ags/w_action_node/optimized/humaneval/basic/operator.py similarity index 100% rename from examples/ags/w_action_node/graphs/humaneval/basic/operator.py rename to examples/ags/w_action_node/optimized/humaneval/basic/operator.py diff --git a/examples/ags/w_action_node/graphs/humaneval/basic/prompt.py b/examples/ags/w_action_node/optimized/humaneval/basic/prompt.py similarity index 100% rename from examples/ags/w_action_node/graphs/humaneval/basic/prompt.py rename to examples/ags/w_action_node/optimized/humaneval/basic/prompt.py diff --git a/examples/ags/w_action_node/graphs/humaneval/graph.py b/examples/ags/w_action_node/optimized/humaneval/graph.py similarity index 100% rename from examples/ags/w_action_node/graphs/humaneval/graph.py rename to examples/ags/w_action_node/optimized/humaneval/graph.py diff --git a/examples/ags/w_action_node/optimizer.py b/examples/ags/w_action_node/optimizer.py index a88f6baa3..6b6ed942a 100644 --- a/examples/ags/w_action_node/optimizer.py +++ b/examples/ags/w_action_node/optimizer.py @@ -21,6 +21,11 @@ DatasetType = Literal["HumanEval", "MMBP", "Gsm8K", "MATH", "HotpotQa", "MMLU"] evaluator = Evaluator(eval_path="eval") +# prompt = GENERATE_PROMPT.format(problem_description=problem_description) +# node = await ActionNode.from_pydantic(GenerateOp).fill(context=prompt, mode="context_fill", llm=self.llm) +# response = node.instruct_content.model_dump() +# return response + class Optimizer: def __init__(self, dataset: DatasetType, llm: LLM, operators: List, optimized_path: str = None) -> None: @@ -28,6 +33,7 @@ class Optimizer: self.dataset = dataset self.graph = None # 初始化为 None,稍后加载 self.operators = operators + self.optimize_prompt = "" self._optimized_path = optimized_path self.root_path = f"{self._optimized_path}/{self.dataset}" self.sample = 6 # sample 含义是什么? @@ -37,9 +43,10 @@ class Optimizer: def _initialize(self): """ - 基于数据集、操作符初始化 operator 跟 graph + 基于数据集、操作符初始化optimize prompt, operator 跟 graph """ - basic_path = f"{config_iterate_path}/{self.dataset}/basic" + + basic_path = f"{self.root_path}/basic" required_files = ["operator.py", "graph.py", "prompt.py"] def check_files_exist(basic_path, required_files): @@ -104,6 +111,7 @@ class Optimizer: 动态读取指定轮次的 Prompt和Graph。 """ # 构建 prompt.py 文件的相对路径 + # examples/ags/w_action_node/optimized/gsm8k/graphs/round_1 prompt_file_path = os.path.join(graphs_path, "prompt.py") graph_file_path = os.path.join(graphs_path, "graph.py") @@ -228,7 +236,7 @@ class Optimizer: """ # TODO 读取basic模版(从对应的dataset文件夹 {dataset}/basic/operator.py, graph.py, prompt.py ),Operator几乎不用动 # TODO 动Prompt内容;动Graph连接 - graph_path = f"{self.root_path}/graphs" + graph_path = f"{self._optimized_path}/{self.dataset}/graphs" f"{graph_path}/round_{self.round + 1}" # TODO 填充Optimize 逻辑