Update

metagpt/ext/aflow/README.md

@@ -25,7 +25,7 @@ ## Datasets
 ## Quick Start
 
 1. Configure your search in `optimize.py`:
-   - Open `examples/aflow/scripts/optimize.py`
+   - Open `metagpt/ext/aflow/scripts/optimize.py`
    - Set the following parameters:
      ```python
      dataset = "HumanEval"  # Choose from: "HumanEval", "MBPP", "GSM8K", "MATH", "HotpotQA", "DROP" or your custom dataset name
@@ -37,19 +37,19 @@ ## Quick Start
      max_rounds = 20  # Maximum number of optimization rounds
      ```
    - Adjust these parameters according to your specific requirements and dataset
-2. Set up parameters in `config/config2.yaml` (see `examples/aflow/config2.example.yaml` for reference)
+2. Set up parameters in `config/config2.yaml` (see `metagpt/ext/aflow/config2.example.yaml` for reference)
 3. Set the operator you want to use in `optimize.py` and in `xxxx`
 4. Download the init round of six datasets and put them in `xxxxxx`
 5. Add your custom dataset and corresponding evaluation function:
 
-- Create a new Python file in the `examples/aflow/benchmark/` directory, named `{custom_dataset_name}.py`
+- Create a new Python file in the `metagpt/ext/aflow/benchmark/` directory, named `{custom_dataset_name}.py`
 - Implement the following key functions in this new file:
   - `load_data`: for loading the dataset
   - `evaluate_problem`: for evaluating a single problem solution
   - `evaluate_all_problems`: for evaluating all problems
   - `save_results_to_csv`: for saving evaluation results
   - `optimize_{custom_dataset_name}_evaluation`: main evaluation function that integrates the above functionalities
-- Add your custom dataset name and config val_list in `examples/aflow/scripts/evaluator.py`
+- Add your custom dataset name and config val_list in `metagpt/ext/aflow/scripts/evaluator.py`
 
 
 ## License

metagpt/ext/aflow/config2.example.yaml

@@ -1,12 +0,0 @@
-models:
- "<model_name>": # model: "gpt-4-turbo"  # or gpt-3.5-turbo
-   api_type: "openai"  # or azure / ollama / groq etc.
-   base_url: "<your base url>" 
-   api_key: "<your api key>"
-   temperature: 0
- "<model_name>":  
-   api_type: "openai"  
-   base_url: "<your base url>"
-   api_key: "<your api key>"
-   temperature: 0
-CALC_USAGE: True 

metagpt/ext/aflow/data/download_data.py

@@ -42,26 +42,23 @@ def process_dataset(url: str, filename: str, extract_path: str) -> None:
 
 # Define the datasets to be downloaded
 # Users can modify this list to choose which datasets to download
-datasets_to_download: List[Dict[str, str]] = [
-    {
-        "name": "datasets",
-        "url": "https://drive.google.com/uc?export=download&id=1tXp5cLw89egeKRwDuood2TPqoEWd8_C0",
+datasets_to_download: Dict[str, Dict[str, str]] = {
+    "datasets": {
+        "url": "https://drive.google.com/uc?export=download&id=1DNoegtZiUhWtvkd2xoIuElmIi4ah7k8e",
         "filename": "aflow_data.tar.gz",
-        "extract_path": "examples/aflow/data"
+        "extract_path": "metagpt/ext/aflow/data"
     },
-    {
-        "name": "results",
-        "url": "",  # Please fill in the correct URL
+    "results": {
+        "url": "",  # 请填入正确的URL
         "filename": "result.tar.gz",
-        "extract_path": "examples/aflow/data/results"
+        "extract_path": "metagpt/ext/aflow/data/results"
     },
-    {
-        "name": "initial_rounds",
-        "url": "",  # Please fill in the correct URL
-        "filename": "first_round.tar.gz",
-        "extract_path": "examples/aflow/scripts/optimized"
+    "initial_rounds": {
+        "url": "https://drive.google.com/uc?export=download&id=1UBoW4WBWjX2gs4I_jq3ALdXeLdwDJMdP",
+        "filename": "initial_rounds.tar.gz",
+        "extract_path": "metagpt/ext/aflow/scripts/optimized"
     }
-]
+}
 
 def download(datasets):
     """Main function to process all selected datasets."""

metagpt/ext/aflow/scripts/evaluator.py

@@ -55,5 +55,5 @@ class Evaluator:
         return graph(name=dataset, llm_config=llm_config, dataset=dataset_config)
 
     def _get_data_path(self, dataset: DatasetType, test: bool) -> str:
-        base_path = f"examples/aflow/data/{dataset.lower()}"
+        base_path = f"metagpt/ext/aflow/data/{dataset.lower()}"
         return f"{base_path}_test.jsonl" if test else f"{base_path}_validate.jsonl"

metagpt/ext/aflow/scripts/optimized/GSM8K/workflows/round_2/graph.py

@@ -0,0 +1,33 @@
+from typing import Literal
+import metagpt.ext.aflow.scripts.optimized.GSM8K.workflows.template.operator as operator
+import metagpt.ext.aflow.scripts.optimized.GSM8K.workflows.round_2.prompt as prompt_custom
+from metagpt.provider.llm_provider_registry import create_llm_instance
+from metagpt.utils.cost_manager import CostManager
+
+DatasetType = Literal["HumanEval", "MBPP", "GSM8K", "MATH", "HotpotQA", "DROP"]
+
+class Workflow:
+    def __init__(
+        self,
+        name: str,
+        llm_config,
+        dataset: DatasetType,
+    ) -> None:
+        self.name = name
+        self.dataset = dataset
+        self.llm = create_llm_instance(llm_config)
+        self.llm.cost_manager = CostManager()
+        self.custom = operator.Custom(self.llm)
+        self.sc_ensemble = operator.ScEnsemble(self.llm)
+
+    async def __call__(self, problem: str):
+        """
+        Implementation of the workflow
+        """
+        solutions = []
+        for _ in range(3):
+            solution = await self.custom(input=problem, instruction=prompt_custom.SOLVE_PROMPT)
+            solutions.append(solution['response'])
+        
+        final_solution = await self.sc_ensemble(solutions=solutions, problem=problem)
+        return final_solution['response'], self.llm.cost_manager.total_cost

metagpt/ext/aflow/scripts/optimized/GSM8K/workflows/round_2/prompt.py

@@ -0,0 +1,12 @@
+SOLVE_PROMPT = """
+You are a mathematical problem solver. Your task is to solve the given problem step by step, showing all your work. After solving the problem, provide the final numerical answer without any units or explanations. Make sure to:
+
+1. Break down the problem into clear steps.
+2. Show all calculations.
+3. Use proper mathematical notation.
+4. Double-check your work for accuracy.
+5. Provide only the final numerical answer at the end, with no additional text.
+
+Solve the following problem:
+
+"""

metagpt/ext/aflow/scripts/utils.py

@@ -56,7 +56,7 @@ def extract_test_cases_from_jsonl(
     entry_point: str, dataset: str = "HumanEval"
 ):
     if dataset == "HumanEval":
-        file_path = "examples/aflow/data/humaneval_public_test.jsonl"
+        file_path = "metagpt/ext/aflow/data/humaneval_public_test.jsonl"
     # Retain the original hardcoded test cases
         hardcoded_cases = {
         "find_zero": "",
@@ -71,7 +71,7 @@ def extract_test_cases_from_jsonl(
         "starts_one_ends":""
         }
     elif dataset == "MBPP":
-        file_path = "examples/aflow/data/mbpp_public_test.jsonl"
+        file_path = "metagpt/ext/aflow/data/mbpp_public_test.jsonl"
         hardcoded_cases = {
             "remove_odd": "",
             "replace_spaces": "",