diff --git a/.gitignore b/.gitignore index aa5edd74a..7c64829ad 100644 --- a/.gitignore +++ b/.gitignore @@ -162,6 +162,7 @@ examples/graph_store.json examples/image__vector_store.json examples/index_store.json .chroma +.chroma_exp_data *~$* workspace/* tmp diff --git a/config/config2.example.yaml b/config/config2.example.yaml index f1158775b..330b73680 100644 --- a/config/config2.example.yaml +++ b/config/config2.example.yaml @@ -74,6 +74,10 @@ s3: secure: false bucket: "test" +exp_pool: + enable_read: false + enable_write: false + persist_path: .chroma_exp_data # The directory. azure_tts_subscription_key: "YOUR_SUBSCRIPTION_KEY" azure_tts_region: "eastus" diff --git a/examples/data/exp_pool/engineer_exps.json b/examples/data/exp_pool/engineer_exps.json new file mode 100644 index 000000000..89c0bc186 --- /dev/null +++ b/examples/data/exp_pool/engineer_exps.json @@ -0,0 +1,16 @@ +[{ + "req": [{ + "role": "user", + "content": "\n# Current Plan\n{'goal': \"Please write a 1048 game using JavaScript and HTML code without using any frameworks, user can play with keyboard. Refer to the system design located at '/tmp/system_design.json' and the project schedule at '/tmp/project_schedule.json' for detailed information.\", 'tasks': []}\n\n# Current Task\n\n\n# Instruction\nBased on the context, write a plan or modify an existing plan to achieve the goal. A plan consists of one to 3 tasks.\nIf plan is created, you should track the progress and update the plan accordingly, such as Plan.finish_current_task, Plan.append_task, Plan.reset_task, Plan.replace_task, etc.\nWhen presented a current task, tackle the task using the available commands.\nPay close attention to new user message, review the conversation history, use RoleZero.reply_to_human to respond to new user requirement.\nNote:\n1. If you keeping encountering errors, unexpected situation, or you are not sure of proceeding, use RoleZero.ask_human to ask for help.\n2. Carefully review your progress at the current task, if your actions so far has not fulfilled the task instruction, you should continue with current task. Otherwise, finish current task.\n3. Each time you finish a task, use RoleZero.reply_to_human to report your progress.\n4. Each time you write a code in your response, write with the Editor directly without preparing a repetitive code block beforehand.\n5. Take on ONE task and write ONE code file in each response. DON'T attempt all tasks in one response.\n6. When not specified, you should write files in a folder named \"src\". If you know the project path, then write in a \"src\" folder under the project path.\n7. When provided system design or project schedule, you MUST read them first before making a plan, then adhere to them in your implementation, especially in the programming language, package, or framework. You MUST implement all code files prescribed in the system design or project schedule. You can create a plan first with each task corresponding to implementing one code file.\n8. Write at most one file per task, do your best to implement THE ONLY ONE FILE. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE.\n9. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets.\n10. When provided system design, YOU MUST FOLLOW \"Data structures and interfaces\". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design.\n11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO.\n12. To modify code in a file, read the entire file, make changes, and update the file with the complete code, ensuring that no line numbers are included in the final write.\n13. When a system design or project schedule is provided, at the end of the plan, add a CodeRview Task for each file; for example, if there are three files, add three CodeRview Tasks. For each CodeRview Task, just call ReviewAndRewriteCode.run.\n" + }], + "resp": "\nExplanation: To begin the task of creating a 2048 game using JavaScript and HTML, I need to first read the system design and project schedule documents to understand the specific requirements and architecture outlined for the game development.\n\n```json\n[\n {\n \"command_name\": \"Editor.read\",\n \"args\": {\n \"path\": \"/tmp/system_design.json\"\n }\n },\n {\n \"command_name\": \"Editor.read\",\n \"args\": {\n \"path\": \"/tmp/project_schedule.json\"\n }\n }\n]\n```\n" +}, { + "req": [{ + "role": "user", + "content": "Command Editor.read executed: file_path='/tmp/system_design.json' block_content='001|{\"Implementation approach\":\"We will implement the 2048 game using plain JavaScript and HTML, ensuring no frameworks are used. The game logic will handle tile movements, merging, and game state updates. The UI will be simple and clean, with a responsive design to fit different screen sizes. We will use CSS for styling and ensure the game is playable with keyboard arrow keys. The game will display the current score, have a restart button, and show a game over message when no more moves are possible.\",\"File list\":[\"index.html\",\"style.css\",\"script.js\"],\"Data structures and interfaces\":\"\\\\nclassDiagram\\\\n class Game {\\\\n -grid: int[][]\\\\n -score: int\\\\n +init(): void\\\\n +move(direction: str): void\\\\n +merge(direction: str): void\\\\n +isGameOver(): bool\\\\n +restart(): void\\\\n }\\\\n class UI {\\\\n -game: Game\\\\n +init(): void\\\\n +update(): void\\\\n +showGameOver(): void\\\\n +bindEvents(): void\\\\n }\\\\n Game --> UI\\\\n\",\"Program call flow\":\"\\\\nsequenceDiagram\\\\n participant U as UI\\\\n participant G as Game\\\\n U->>G: init()\\\\n G-->>U: return\\\\n U->>U: bindEvents()\\\\n U->>G: move(direction)\\\\n G->>G: merge(direction)\\\\n G->>U: update()\\\\n U->>U: update()\\\\n U->>G: isGameOver()\\\\n G-->>U: return bool\\\\n alt Game Over\\\\n U->>U: showGameOver()\\\\n end\\\\n U->>G: restart()\\\\n G-->>U: return\\\\n\",\"Anything UNCLEAR\":\"Clarify if there are any specific design preferences or additional features required beyond the basic 2048 game functionality.\"}\\n'\n\nCommand Editor.read executed: file_path='/tmp/project_schedule.json' block_content='001|{\"Required packages\":[\"No third-party dependencies required\"],\"Required Other language third-party packages\":[\"No third-party dependencies required\"],\"Logic Analysis\":[[\"script.js\",\"Contains Game and UI classes, and their methods: init, move, merge, isGameOver, restart, update, showGameOver, bindEvents\"],[\"index.html\",\"Contains the HTML structure for the game UI\"],[\"style.css\",\"Contains the CSS styles for the game UI\"]],\"Task list\":[\"index.html\",\"style.css\",\"script.js\"],\"Full API spec\":\"\",\"Shared Knowledge\":\"The `script.js` file will contain the core game logic and UI handling. The `index.html` file will provide the structure for the game, and `style.css` will handle the styling.\",\"Anything UNCLEAR\":\"Clarify if there are any specific design preferences or additional features required beyond the basic 2048 game functionality.\"}\\n'" + }, { + "role": "user", + "content": "\n# Current Plan\n{'goal': \"Please write a 1048 game using JavaScript and HTML code without using any frameworks, user can play with keyboard. Refer to the system design located at '/tmp/system_design.json' and the project schedule at '/tmp/project_schedule.json' for detailed information.\", 'tasks': []}\n\n# Current Task\n\n\n# Instruction\nBased on the context, write a plan or modify an existing plan to achieve the goal. A plan consists of one to 3 tasks.\nIf plan is created, you should track the progress and update the plan accordingly, such as Plan.finish_current_task, Plan.append_task, Plan.reset_task, Plan.replace_task, etc.\nWhen presented a current task, tackle the task using the available commands.\nPay close attention to new user message, review the conversation history, use RoleZero.reply_to_human to respond to new user requirement.\nNote:\n1. If you keeping encountering errors, unexpected situation, or you are not sure of proceeding, use RoleZero.ask_human to ask for help.\n2. Carefully review your progress at the current task, if your actions so far has not fulfilled the task instruction, you should continue with current task. Otherwise, finish current task.\n3. Each time you finish a task, use RoleZero.reply_to_human to report your progress.\n4. Each time you write a code in your response, write with the Editor directly without preparing a repetitive code block beforehand.\n5. Take on ONE task and write ONE code file in each response. DON'T attempt all tasks in one response.\n6. When not specified, you should write files in a folder named \"src\". If you know the project path, then write in a \"src\" folder under the project path.\n7. When provided system design or project schedule, you MUST read them first before making a plan, then adhere to them in your implementation, especially in the programming language, package, or framework. You MUST implement all code files prescribed in the system design or project schedule. You can create a plan first with each task corresponding to implementing one code file.\n8. Write at most one file per task, do your best to implement THE ONLY ONE FILE. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE.\n9. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets.\n10. When provided system design, YOU MUST FOLLOW \"Data structures and interfaces\". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design.\n11. Write out EVERY CODE DETAIL, DON'T LEAVE TODO.\n12. To modify code in a file, read the entire file, make changes, and update the file with the complete code, ensuring that no line numbers are included in the final write.\n13. When a system design or project schedule is provided, at the end of the plan, add a CodeRview Task for each file; for example, if there are three files, add three CodeRview Tasks. For each CodeRview Task, just call ReviewAndRewriteCode.run.\n" + }], + "resp": "\nExplanation: Based on the system design and project schedule, I will create a plan to implement the 2048 game. The plan will consist of tasks to create the necessary files (`index.html`, `style.css`, and `script.js`) and review each file to ensure they meet the design specifications.\n\nHere's the plan:\n\n1. **Task 1**: Create `index.html` - This file will contain the HTML structure necessary for the game's UI.\n2. **Task 2**: Create `style.css` - This file will define the CSS styles to make the game visually appealing and responsive.\n3. **Task 3**: Create `script.js` - This file will contain the JavaScript code for the game logic and UI interactions.\n4. **Code Review Tasks**: Review each file to ensure they meet the project requirements and adhere to the system design.\n\nLet's start by appending the first task to the plan.\n\n```json\n[\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"1\",\n \"dependent_task_ids\": [],\n \"instruction\": \"Create the index.html file with the basic HTML structure for the 2048 game.\",\n \"assignee\": \"Alex\"\n }\n },\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"2\",\n \"dependent_task_ids\": [\"1\"],\n \"instruction\": \"Create the style.css file with the necessary CSS to style the 2048 game.\",\n \"assignee\": \"Alex\"\n }\n },\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"3\",\n \"dependent_task_ids\": [\"1\", \"2\"],\n \"instruction\": \"Create the script.js file containing the JavaScript logic for the 2048 game.\",\n \"assignee\": \"Alex\"\n }\n },\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"4\",\n \"dependent_task_ids\": [\"1\"],\n \"instruction\": \"Use ReviewAndRewriteCode to review the code in index.html to ensure it meets the design specifications.\",\n \"assignee\": \"Alex\"\n }\n },\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"5\",\n \"dependent_task_ids\": [\"2\"],\n \"instruction\": \"Use ReviewAndRewriteCode to review the code in style.css to ensure it meets the design specifications.\",\n \"assignee\": \"Alex\"\n }\n },\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"6\",\n \"dependent_task_ids\": [\"3\"],\n \"instruction\": \"Use ReviewAndRewriteCode to review the code in script.js to ensure it meets the design specifications.\",\n \"assignee\": \"Alex\"\n }\n }\n]\n```\n" +}] \ No newline at end of file diff --git a/examples/data/exp_pool/team_leader_exps.json b/examples/data/exp_pool/team_leader_exps.json new file mode 100644 index 000000000..125f0a48d --- /dev/null +++ b/examples/data/exp_pool/team_leader_exps.json @@ -0,0 +1,22 @@ +[{ + "req": [{ + "role": "user", + "content": "\n# Current Plan\n{'goal': \"from to {''}: Write a 1024 game using JavaScript and HTML code without using any frameworks, user can play with keyboard.\", 'tasks': []}\n\n# Current Task\n\n\n# Instruction\nYou are a team leader, and you are responsible for drafting tasks and routing tasks to your team members.\nYour team member:\nTim: Team Leader, \nAlice: Product Manager, efficiently create a successful product that meets market demands and user expectations\nBob: Architect, design a concise, usable, complete software system\nEve: Project Manager, break down tasks according to PRD/technical design, generate a task list, and analyze task dependencies to start with the prerequisite modules\nAlex: Engineer, Take on game, app, and web development\nDavid: DataAnalyst, Take on any data-related tasks, such as data analysis, machine learning, deep learning, web browsing, web scraping, web searching, web deployment, terminal operation, git and github operation, etc.\n\nYou should NOT assign consecutive tasks to the same team member, instead, assign an aggregated task (or the complete requirement) and let the team member to decompose it.\nWhen creating a new plan involving multiple members, create all tasks at once.\nIf plan is created, you should track the progress based on team member feedback message, and update plan accordingly, such as Plan.finish_current_task, Plan.reset_task, Plan.replace_task, etc.\nYou should use TeamLeader.publish_team_message to team members, asking them to start their task. DONT omit any necessary info such as path, link, environment, programming language, framework, requirement, constraint from original content to team members because you are their sole info source.\nPay close attention to new user message, review the conversation history, use RoleZero.reply_to_human to respond to the user directly, DON'T ask your team members.\n\nNote:\n1. If the requirement is a pure DATA-RELATED requirement, such as bug fixes, issue reporting, environment setup, terminal operations, pip install, web browsing, web scraping, web searching, web imitation, data science, data analysis, machine learning, deep learning, text-to-image etc. DON'T decompose it, assign a single task with the original user requirement as instruction directly to Data Analyst.\n2. If the requirement is developing a software, game, app, or website, excluding the above data-related tasks, you should decompose the requirement into multiple tasks and assign them to different team members based on their expertise, usually the sequence of Product Manager -> Architect -> Project Manager -> Engineer -> (optional: QaEngine if present) -> (optional: DataAnalyst if user requests deployment), each assigned ONE task. When publishing message to Product Manager, you should directly copy the full original user requirement.\n3. If the requirement contains both DATA-RELATED part mentioned in 1 and software development part mentioned in 2, you should decompose the software development part and assign them to different team members based on their expertise, and assign the DATA-RELATED part to Data Analyst David directly.\n4. If the requirement is a common-sense, logical, or math problem, you should respond directly without assigning any task to team members.\n5. If you think the requirement is not clear or ambiguous, you should ask the user for clarification immediately. Assign tasks only after all info is clear.\n6. It is helpful for Engineer to have both the system design and the project schedule for writing the code, so include paths of both files (if available) and remind Engineer to definitely read them when publishing message to Engineer.\n7. If the requirement is writing a TRD and software framework, you should assign it to Architect. When publishing message to Architect, you should directly copy the full original user requirement.\n" + }], + "resp": "\nExplanation: The requirement is about software development. Assign each tasks to a different team member based on their expertise. When publishing message to Product Manager, we copy original user requirement directly to ensure no information loss.\n```json\n[\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"1\",\n \"dependent_task_ids\": [],\n \"instruction\": \"Create a product requirement document (PRD) outlining the features, user interface, and user experience of the CLI python snake game.\",\n \"assignee\": \"Alice\"\n }\n },\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"2\",\n \"dependent_task_ids\": [\"1\"],\n \"instruction\": \"Design the software architecture for the CLI snake game, including the choice of programming language, libraries, and data flow.\",\n \"assignee\": \"Bob\"\n }\n },\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"3\",\n \"dependent_task_ids\": [\"2\"],\n \"instruction\": \"Break down the architecture into manageable tasks, identify task dependencies, and prepare a detailed task list for implementation.\",\n \"assignee\": \"Eve\"\n }\n },\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"4\",\n \"dependent_task_ids\": [\"3\"],\n \"instruction\": \"Implement the core game logic for the CLI snake game, including snake movement, food generation, and score tracking.\",\n \"assignee\": \"Alex\"\n }\n },\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"5\",\n \"dependent_task_ids\": [\"4\"],\n \"instruction\": \"Write comprehensive tests for the game logic and user interface to ensure functionality and reliability.\",\n \"assignee\": \"Edward\"\n }\n },\n {\n \"command_name\": \"TeamLeader.publish_message\",\n \"args\": {\n \"content\": \"Create a cli snake game using Python\",\n \"send_to\": \"Alice\"\n }\n },\n {\n \"command_name\": \"RoleZero.reply_to_human\",\n \"args\": {\n \"content\": \"I have assigned the tasks to the team members. Alice will create the PRD, Bob will design the software architecture, Eve will break down the architecture into tasks, Alex will implement the core game logic, and Edward will write comprehensive tests. The team will work on the project accordingly\"\n }\n },\n {\n \"command_name\": \"end\"\n }\n]\n```\n" +}, { + "req": [{ + "role": "user", + "content": "\n# Current Plan\n{'goal': \"from to {''}: Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.\", 'tasks': []}\n\n# Current Task\n\n\n# Instruction\nYou are a team leader, and you are responsible for drafting tasks and routing tasks to your team members.\nYour team member:\nTim: Team Leader, \nAlice: Product Manager, efficiently create a successful product that meets market demands and user expectations\nBob: Architect, design a concise, usable, complete software system\nEve: Project Manager, break down tasks according to PRD/technical design, generate a task list, and analyze task dependencies to start with the prerequisite modules\nAlex: Engineer, Take on game, app, and web development\nDavid: DataAnalyst, Take on any data-related tasks, such as data analysis, machine learning, deep learning, web browsing, web scraping, web searching, web deployment, terminal operation, git and github operation, etc.\n\nYou should NOT assign consecutive tasks to the same team member, instead, assign an aggregated task (or the complete requirement) and let the team member to decompose it.\nWhen creating a new plan involving multiple members, create all tasks at once.\nIf plan is created, you should track the progress based on team member feedback message, and update plan accordingly, such as Plan.finish_current_task, Plan.reset_task, Plan.replace_task, etc.\nYou should use TeamLeader.publish_team_message to team members, asking them to start their task. DONT omit any necessary info such as path, link, environment, programming language, framework, requirement, constraint from original content to team members because you are their sole info source.\nPay close attention to new user message, review the conversation history, use RoleZero.reply_to_human to respond to the user directly, DON'T ask your team members.\n\nNote:\n1. If the requirement is a pure DATA-RELATED requirement, such as bug fixes, issue reporting, environment setup, terminal operations, pip install, web browsing, web scraping, web searching, web imitation, data science, data analysis, machine learning, deep learning, text-to-image etc. DON'T decompose it, assign a single task with the original user requirement as instruction directly to Data Analyst.\n2. If the requirement is developing a software, game, app, or website, excluding the above data-related tasks, you should decompose the requirement into multiple tasks and assign them to different team members based on their expertise, usually the sequence of Product Manager -> Architect -> Project Manager -> Engineer -> (optional: QaEngine if present) -> (optional: DataAnalyst if user requests deployment), each assigned ONE task. When publishing message to Product Manager, you should directly copy the full original user requirement.\n3. If the requirement contains both DATA-RELATED part mentioned in 1 and software development part mentioned in 2, you should decompose the software development part and assign them to different team members based on their expertise, and assign the DATA-RELATED part to Data Analyst David directly.\n4. If the requirement is a common-sense, logical, or math problem, you should respond directly without assigning any task to team members.\n5. If you think the requirement is not clear or ambiguous, you should ask the user for clarification immediately. Assign tasks only after all info is clear.\n6. It is helpful for Engineer to have both the system design and the project schedule for writing the code, so include paths of both files (if available) and remind Engineer to definitely read them when publishing message to Engineer.\n7. If the requirement is writing a TRD and software framework, you should assign it to Architect. When publishing message to Architect, you should directly copy the full original user requirement.\n" + }], + "resp": "\nExplanation: DON'T decompose requirement if it is a DATA-RELATED task, assign a single task directly to Data Analyst David. He will manage the decomposition and implementation.\n```json\n[\n {\n \"command_name\": \"Plan.append_task\",\n \"args\": {\n \"task_id\": \"1\",\n \"dependent_task_ids\": [],\n \"instruction\": \"Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.\",\n \"assignee\": \"David\"\n }\n },\n {\n \"command_name\": \"TeamLeader.publish_message\",\n \"args\": {\n \"content\": \"Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.\",\n \"send_to\": \"David\"\n }\n },\n {\n \"command_name\": \"RoleZero.reply_to_human\",\n \"args\": {\n \"content\": \"I have assigned the task to David. He will break down the task further by himself and starts solving it.\"\n }\n },\n {\n \"command_name\": \"end\"\n }\n]\n```\n" +}, { + "req": "Conversation History:\n[\n ...,\n {'role': 'assistant', 'content': 'from Alice(Product Manager) to {'Bob'}: {'docs': {'20240424153821.json': {'root_path': 'docs/prd', 'filename': '20240424153821.json', 'content': '{\"Language\":\"en_us\",\"Programming Language\":\"Python\",\"Original Requirements\":\"create a cli snake game\",\"Project Name\":\"snake_game\",\"Product Goals\":[\"Develop an intuitive and addictive snake game\",...], ...}}}}},\n]\n", + "resp": "\nExplanation: You received a message from Alice, the Product Manager, that she has completed the PRD, use Plan.finish_current_task to mark her task as finished and moves the plan to the next task. Based on plan status, next task is for Bob (Architect), publish a message asking him to start. The message content should contain important path info.\n```json\n[\n {\n \"command_name\": \"Plan.finish_current_task\",\n \"args\": {}\n },\n {\n \"command_name\": \"TeamLeader.publish_message\",\n \"args\": {\n \"content\": \"Please design the software architecture for the snake game based on the PRD created by Alice. The PRD is at 'docs/prd/20240424153821.json'. Include the choice of programming language, libraries, and data flow, etc.\",\n \"send_to\": \"Bob\"\n }\n },\n {\n \"command_name\": \"RoleZero.reply_to_human\",\n \"args\": {\n \"content\": \"Alice has completed the PRD. I have marked her task as finished and sent the PRD to Bob. Bob will work on the software architecture.\"\n }\n },\n {\n \"command_name\": \"end\"\n }\n]\n```\n" +}, { + "req": [{ + "role": "user", + "content": "\n# Current Plan\n{'goal': \"from to {''}: how does the project go?\", 'tasks': []}\n\n# Current Task\n\n\n# Instruction\nYou are a team leader, and you are responsible for drafting tasks and routing tasks to your team members.\nYour team member:\nTim: Team Leader, \nAlice: Product Manager, efficiently create a successful product that meets market demands and user expectations\nBob: Architect, design a concise, usable, complete software system\nEve: Project Manager, break down tasks according to PRD/technical design, generate a task list, and analyze task dependencies to start with the prerequisite modules\nAlex: Engineer, Take on game, app, and web development\nDavid: DataAnalyst, Take on any data-related tasks, such as data analysis, machine learning, deep learning, web browsing, web scraping, web searching, web deployment, terminal operation, git and github operation, etc.\n\nYou should NOT assign consecutive tasks to the same team member, instead, assign an aggregated task (or the complete requirement) and let the team member to decompose it.\nWhen creating a new plan involving multiple members, create all tasks at once.\nIf plan is created, you should track the progress based on team member feedback message, and update plan accordingly, such as Plan.finish_current_task, Plan.reset_task, Plan.replace_task, etc.\nYou should use TeamLeader.publish_team_message to team members, asking them to start their task. DONT omit any necessary info such as path, link, environment, programming language, framework, requirement, constraint from original content to team members because you are their sole info source.\nPay close attention to new user message, review the conversation history, use RoleZero.reply_to_human to respond to the user directly, DON'T ask your team members.\n\nNote:\n1. If the requirement is a pure DATA-RELATED requirement, such as bug fixes, issue reporting, environment setup, terminal operations, pip install, web browsing, web scraping, web searching, web imitation, data science, data analysis, machine learning, deep learning, text-to-image etc. DON'T decompose it, assign a single task with the original user requirement as instruction directly to Data Analyst.\n2. If the requirement is developing a software, game, app, or website, excluding the above data-related tasks, you should decompose the requirement into multiple tasks and assign them to different team members based on their expertise, usually the sequence of Product Manager -> Architect -> Project Manager -> Engineer -> (optional: QaEngine if present) -> (optional: DataAnalyst if user requests deployment), each assigned ONE task. When publishing message to Product Manager, you should directly copy the full original user requirement.\n3. If the requirement contains both DATA-RELATED part mentioned in 1 and software development part mentioned in 2, you should decompose the software development part and assign them to different team members based on their expertise, and assign the DATA-RELATED part to Data Analyst David directly.\n4. If the requirement is a common-sense, logical, or math problem, you should respond directly without assigning any task to team members.\n5. If you think the requirement is not clear or ambiguous, you should ask the user for clarification immediately. Assign tasks only after all info is clear.\n6. It is helpful for Engineer to have both the system design and the project schedule for writing the code, so include paths of both files (if available) and remind Engineer to definitely read them when publishing message to Engineer.\n7. If the requirement is writing a TRD and software framework, you should assign it to Architect. When publishing message to Architect, you should directly copy the full original user requirement.\n" + }], + "resp": "\nExplanation: The user is asking for a general update on the project status. Give a straight answer about the current task the team is working on and provide a summary of the completed tasks.\n```json\n[\n {\n \"command_name\": \"RoleZero.reply_to_human\",\n \"args\": {\n \"content\": \"The team is currently working on ... We have completed ...\"\n }\n },\n {\n \"command_name\": \"end\"\n }\n]\n```\n" +}] \ No newline at end of file diff --git a/examples/exp_pool/README.md b/examples/exp_pool/README.md new file mode 100644 index 000000000..d405bfa22 --- /dev/null +++ b/examples/exp_pool/README.md @@ -0,0 +1,19 @@ +# Experience Pool + +## Prerequisites +- Ensure the RAG module is installed: https://docs.deepwisdom.ai/main/en/guide/in_depth_guides/rag_module.html +- Set both `enable_read` and `enable_write` to `true` in the `exp_pool` section of `config2.yaml` + +## Example Files + +### 1. decorator.py +Showcases the implementation of the `@exp_cache` decorator. + +### 2. init_exp_pool.py +Demonstrates the process of initializing the experience pool. + +### 3. manager.py +Illustrates CRUD (Create, Read, Update, Delete) operations for managing experiences in the pool. + +### 4. scorer.py +Outlines methods for evaluating and scoring experiences within the pool. diff --git a/examples/exp_pool/decorator.py b/examples/exp_pool/decorator.py new file mode 100644 index 000000000..d25949e8d --- /dev/null +++ b/examples/exp_pool/decorator.py @@ -0,0 +1,28 @@ +""" +This script demonstrates how to automatically store experiences using @exp_cache and query the stored experiences. +""" + +import asyncio +import uuid + +from metagpt.exp_pool import exp_cache, exp_manager +from metagpt.logs import logger + + +@exp_cache() +async def produce(req=""): + return f"{req} {uuid.uuid4().hex}" + + +async def main(): + req = "Water" + + resp = await produce(req=req) + logger.info(f"The response of `produce({req})` is: {resp}") + + exps = await exp_manager.query_exps(req) + logger.info(f"Find experiences: {exps}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/exp_pool/init_exp_pool.py b/examples/exp_pool/init_exp_pool.py new file mode 100644 index 000000000..14c415be7 --- /dev/null +++ b/examples/exp_pool/init_exp_pool.py @@ -0,0 +1,94 @@ +"""Init experience pool. + +Put some useful experiences into the experience pool. +""" + +import asyncio +import json +from pathlib import Path + +from metagpt.const import EXAMPLE_DATA_PATH +from metagpt.exp_pool import exp_manager +from metagpt.exp_pool.schema import EntryType, Experience, Metric, Score +from metagpt.logs import logger +from metagpt.utils.common import aread + + +async def load_file(filepath) -> list[dict]: + """Asynchronously loads and parses a JSON file. + + Args: + filepath: Path to the JSON file. + + Returns: + A list of dictionaries parsed from the JSON file. + """ + + return json.loads(await aread(filepath)) + + +async def add_exp(req: str, resp: str, tag: str, metric: Metric = None): + """Adds a new experience to the experience pool. + + Args: + req: The request string. + resp: The response string. + tag: A tag for categorizing the experience. + metric: Optional metric for the experience. Defaults to a score of 10. + + """ + + exp = Experience( + req=req, + resp=resp, + entry_type=EntryType.MANUAL, + tag=tag, + metric=metric or Metric(score=Score(val=10, reason="Manual")), + ) + + exp_manager.config.exp_pool.enable_write = True + exp_manager.create_exp(exp) + logger.info(f"New experience created for the request `{req[:10]}`.") + + +async def add_exps(exps: list, tag: str): + """Adds multiple experiences to the experience pool. + + Args: + exps: A list of experience dictionaries. + tag: A tag for categorizing the experiences. + + """ + + tasks = [add_exp(req=json.dumps(exp["req"]), resp=exp["resp"], tag=tag) for exp in exps] + await asyncio.gather(*tasks) + + +async def add_exps_from_file(tag: str, filepath: Path): + """Loads experiences from a file and adds them to the experience pool. + + Args: + tag: A tag for categorizing the experiences. + filepath: Path to the file containing experiences. + + """ + + exps = await load_file(filepath) + await add_exps(exps, tag) + + +def query_exps_count(): + """Queries and logs the total count of experiences in the pool.""" + + count = exp_manager.get_exps_count() + logger.info(f"Experiences Count: {count}") + + +async def main(): + await add_exps_from_file("TeamLeader.llm_cached_aask", EXAMPLE_DATA_PATH / "exp_pool/team_leader_exps.json"), + await add_exps_from_file("Engineer2.llm_cached_aask", EXAMPLE_DATA_PATH / "exp_pool/engineer_exps.json") + query_exps_count() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/exp_pool/manager.py b/examples/exp_pool/manager.py new file mode 100644 index 000000000..ae998214a --- /dev/null +++ b/examples/exp_pool/manager.py @@ -0,0 +1,30 @@ +""" +Demonstrate the creation and querying of experiences. + +This script creates a new experience, logs its creation, and then queries for experiences matching the same request. +""" + +import asyncio + +from metagpt.exp_pool import exp_manager +from metagpt.exp_pool.schema import EntryType, Experience +from metagpt.logs import logger + + +async def main(): + # Define the simple request and response + req = "Simple req" + resp = "Simple resp" + + # Add the new experience + exp = Experience(req=req, resp=resp, entry_type=EntryType.MANUAL) + exp_manager.create_exp(exp) + logger.info(f"New experience created for the request `{req}`.") + + # Query for experiences matching the request + exps = await exp_manager.query_exps(req) + logger.info(f"Got experiences: {exps}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/exp_pool/scorer.py b/examples/exp_pool/scorer.py new file mode 100644 index 000000000..aafcee63f --- /dev/null +++ b/examples/exp_pool/scorer.py @@ -0,0 +1,44 @@ +import asyncio + +from metagpt.exp_pool.scorers import SimpleScorer + +# Request to implement quicksort in Python +REQ = "Write a program to implement quicksort in python." + +# First response: Quicksort implementation without base case +RESP1 = """ +def quicksort(arr): + return quicksort([x for x in arr[1:] if x <= arr[0]]) + [arr[0]] + quicksort([x for x in arr[1:] if x > arr[0]]) +""" + +# Second response: Quicksort implementation with base case +RESP2 = """ +def quicksort(arr): + if len(arr) <= 1: + return arr + return quicksort([x for x in arr[1:] if x <= arr[0]]) + [arr[0]] + quicksort([x for x in arr[1:] if x > arr[0]]) +""" + + +async def simple(): + """Evaluates two quicksort implementations using SimpleScorer. + + Example: + { + "val": 3, + "reason": "The response attempts to implement quicksort but contains a critical flaw: it lacks a base case to terminate the recursion, which will lead to a maximum recursion depth exceeded error for non-empty lists. Additionally, the function does not handle empty lists properly. A correct implementation should include a base case to handle lists of length 0 or 1." + } + """ + + scorer = SimpleScorer() + + await scorer.evaluate(req=REQ, resp=RESP1) + await scorer.evaluate(req=REQ, resp=RESP2) + + +async def main(): + await simple() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/write_novel.py b/examples/write_novel.py index a6e9ce05d..f49918fbb 100644 --- a/examples/write_novel.py +++ b/examples/write_novel.py @@ -50,9 +50,9 @@ async def generate_novel(): "Fill the empty nodes with your own ideas. Be creative! Use your own words!" "I will tip you $100,000 if you write a good novel." ) - novel_node = await ActionNode.from_pydantic(Novel).fill(context=instruction, llm=LLM()) + novel_node = await ActionNode.from_pydantic(Novel).fill(req=instruction, llm=LLM()) chap_node = await ActionNode.from_pydantic(Chapters).fill( - context=f"### instruction\n{instruction}\n### novel\n{novel_node.content}", llm=LLM() + req=f"### instruction\n{instruction}\n### novel\n{novel_node.content}", llm=LLM() ) print(chap_node.instruct_content) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index b760c96d8..8733947f5 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -90,7 +90,7 @@ class Action(SerializationMixin, ContextMixin, BaseModel): msgs = args[0] context = "## History Messages\n" context += "\n".join([f"{idx}: {i}" for idx, i in enumerate(reversed(msgs))]) - return await self.node.fill(context=context, llm=self.llm) + return await self.node.fill(req=context, llm=self.llm) async def run(self, *args, **kwargs): """Run action""" diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 48372f790..c1de16656 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -18,6 +18,8 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action_outcls_registry import register_action_outcls from metagpt.const import MARKDOWN_TITLE_PREFIX, USE_CONFIG_TIMEOUT +from metagpt.exp_pool import exp_cache +from metagpt.exp_pool.serializers import ActionNodeSerializer from metagpt.llm import BaseLLM from metagpt.logs import logger from metagpt.provider.postprocess.llm_output_postprocess import llm_output_postprocess @@ -465,9 +467,11 @@ class ActionNode: return self + @exp_cache(serializer=ActionNodeSerializer()) async def fill( self, - context, + *, + req, llm, schema="json", mode="auto", @@ -478,7 +482,7 @@ class ActionNode: ): """Fill the node(s) with mode. - :param context: Everything we should know when filling node. + :param req: Everything we should know when filling node. :param llm: Large Language Model with pre-defined system message. :param schema: json/markdown, determine example and output format. - raw: free form text @@ -497,7 +501,7 @@ class ActionNode: :return: self """ self.set_llm(llm) - self.set_context(context) + self.set_context(req) if self.schema: schema = self.schema diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 7fac6710b..62b0d435d 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -178,12 +178,12 @@ class WriteDesign(Action): ) async def _new_system_design(self, context): - node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, schema=self.prompt_schema) + node = await DESIGN_API_NODE.fill(req=context, llm=self.llm, schema=self.prompt_schema) return node async def _merge(self, prd_doc, system_design_doc): context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content) - node = await REFINED_DESIGN_NODE.fill(context=context, llm=self.llm, schema=self.prompt_schema) + node = await REFINED_DESIGN_NODE.fill(req=context, llm=self.llm, schema=self.prompt_schema) system_design_doc.content = node.instruct_content.model_dump_json() return system_design_doc diff --git a/metagpt/actions/generate_questions.py b/metagpt/actions/generate_questions.py index c96a37649..bf0ba6277 100644 --- a/metagpt/actions/generate_questions.py +++ b/metagpt/actions/generate_questions.py @@ -22,4 +22,4 @@ class GenerateQuestions(Action): name: str = "GenerateQuestions" async def run(self, context) -> ActionNode: - return await QUESTIONS.fill(context=context, llm=self.llm) + return await QUESTIONS.fill(req=context, llm=self.llm) diff --git a/metagpt/actions/prepare_interview.py b/metagpt/actions/prepare_interview.py index 04cc954d2..0a7eb6581 100644 --- a/metagpt/actions/prepare_interview.py +++ b/metagpt/actions/prepare_interview.py @@ -22,4 +22,4 @@ class PrepareInterview(Action): name: str = "PrepareInterview" async def run(self, context): - return await QUESTIONS.fill(context=context, llm=self.llm) + return await QUESTIONS.fill(req=context, llm=self.llm) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index e810a5dbe..355a3853d 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -151,12 +151,12 @@ class WriteTasks(Action): return task_doc async def _run_new_tasks(self, context: str): - node = await PM_NODE.fill(context, self.llm, schema=self.prompt_schema) + node = await PM_NODE.fill(req=context, llm=self.llm, schema=self.prompt_schema) return node async def _merge(self, system_design_doc, task_doc) -> Document: context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_task=task_doc.content) - node = await REFINED_PM_NODE.fill(context, self.llm, schema=self.prompt_schema) + node = await REFINED_PM_NODE.fill(req=context, llm=self.llm, schema=self.prompt_schema) task_doc.content = node.instruct_content.model_dump_json() return task_doc diff --git a/metagpt/actions/write_code_an_draft.py b/metagpt/actions/write_code_an_draft.py index ce030b0e9..4c3fd4c19 100644 --- a/metagpt/actions/write_code_an_draft.py +++ b/metagpt/actions/write_code_an_draft.py @@ -578,7 +578,7 @@ class WriteCodeAN(Action): async def run(self, context): self.llm.system_prompt = "You are an outstanding engineer and can implement any code" - return await WRITE_MOVE_NODE.fill(context=context, llm=self.llm, schema="json") + return await WRITE_MOVE_NODE.fill(req=context, llm=self.llm, schema="json") async def main(): diff --git a/metagpt/actions/write_code_plan_and_change_an.py b/metagpt/actions/write_code_plan_and_change_an.py index 31482a94d..989df52f2 100644 --- a/metagpt/actions/write_code_plan_and_change_an.py +++ b/metagpt/actions/write_code_plan_and_change_an.py @@ -229,7 +229,7 @@ class WriteCodePlanAndChange(Action): code=await self.get_old_codes(), ) logger.info("Writing code plan and change..") - return await WRITE_CODE_PLAN_AND_CHANGE_NODE.fill(context=context, llm=self.llm, schema="json") + return await WRITE_CODE_PLAN_AND_CHANGE_NODE.fill(req=context, llm=self.llm, schema="json") async def get_old_codes(self) -> str: old_codes = await self.repo.srcs.get_all() diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 4d29d8c6f..42154837a 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -211,7 +211,7 @@ class WritePRD(Action): context = CONTEXT_TEMPLATE.format(requirements=requirement, project_name=project_name) exclude = [PROJECT_NAME.key] if project_name else [] node = await WRITE_PRD_NODE.fill( - context=context, llm=self.llm, exclude=exclude, schema=self.prompt_schema + req=context, llm=self.llm, exclude=exclude, schema=self.prompt_schema ) # schema=schema return node @@ -238,7 +238,7 @@ class WritePRD(Action): async def _is_bugfix(self, context: str) -> bool: if not self.repo.code_files_exists(): return False - node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm) + node = await WP_ISSUE_TYPE_NODE.fill(req=context, llm=self.llm) return node.get("issue_type") == "BUG" async def get_related_docs(self, req: Document, docs: list[Document]) -> list[Document]: @@ -248,14 +248,14 @@ class WritePRD(Action): async def _is_related(self, req: Document, old_prd: Document) -> bool: context = NEW_REQ_TEMPLATE.format(old_prd=old_prd.content, requirements=req.content) - node = await WP_IS_RELATIVE_NODE.fill(context, self.llm) + node = await WP_IS_RELATIVE_NODE.fill(req=context, llm=self.llm) return node.get("is_relative") == "YES" async def _merge(self, req: Document, related_doc: Document) -> Document: if not self.project_name: self.project_name = Path(self.project_path).name prompt = NEW_REQ_TEMPLATE.format(requirements=req.content, old_prd=related_doc.content) - node = await REFINED_PRD_NODE.fill(context=prompt, llm=self.llm, schema=self.prompt_schema) + node = await REFINED_PRD_NODE.fill(req=prompt, llm=self.llm, schema=self.prompt_schema) related_doc.content = node.instruct_content.model_dump_json() await self._rename_workspace(node) return related_doc diff --git a/metagpt/actions/write_review.py b/metagpt/actions/write_review.py index db8512946..907a1e990 100644 --- a/metagpt/actions/write_review.py +++ b/metagpt/actions/write_review.py @@ -36,4 +36,4 @@ class WriteReview(Action): name: str = "WriteReview" async def run(self, context): - return await WRITE_REVIEW_NODE.fill(context=context, llm=self.llm, schema="json") + return await WRITE_REVIEW_NODE.fill(req=context, llm=self.llm, schema="json") diff --git a/metagpt/config2.py b/metagpt/config2.py index 717fe63a9..6588a6036 100644 --- a/metagpt/config2.py +++ b/metagpt/config2.py @@ -13,6 +13,7 @@ from pydantic import BaseModel, model_validator from metagpt.configs.browser_config import BrowserConfig from metagpt.configs.embedding_config import EmbeddingConfig +from metagpt.configs.exp_pool_config import ExperiencePoolConfig from metagpt.configs.llm_config import LLMConfig, LLMType from metagpt.configs.mermaid_config import MermaidConfig from metagpt.configs.redis_config import RedisConfig @@ -71,6 +72,9 @@ class Config(CLIParams, YamlModel): enable_longterm_memory: bool = False code_review_k_times: int = 2 + # Experience Pool Parameters + exp_pool: ExperiencePoolConfig = ExperiencePoolConfig() + # Will be removed in the future metagpt_tti_url: str = "" language: str = "English" diff --git a/metagpt/configs/exp_pool_config.py b/metagpt/configs/exp_pool_config.py new file mode 100644 index 000000000..786558ed9 --- /dev/null +++ b/metagpt/configs/exp_pool_config.py @@ -0,0 +1,9 @@ +from pydantic import Field + +from metagpt.utils.yaml_model import YamlModel + + +class ExperiencePoolConfig(YamlModel): + enable_read: bool = Field(default=False, description="Enable to read from experience pool.") + enable_write: bool = Field(default=False, description="Enable to write to experience pool.") + persist_path: str = Field(default=".chroma_exp_data", description="The persist path for experience pool.") diff --git a/metagpt/exp_pool/__init__.py b/metagpt/exp_pool/__init__.py new file mode 100644 index 000000000..aeeb94b38 --- /dev/null +++ b/metagpt/exp_pool/__init__.py @@ -0,0 +1,6 @@ +"""Experience pool init.""" + +from metagpt.exp_pool.manager import exp_manager +from metagpt.exp_pool.decorator import exp_cache + +__all__ = ["exp_manager", "exp_cache"] diff --git a/metagpt/exp_pool/context_builders/__init__.py b/metagpt/exp_pool/context_builders/__init__.py new file mode 100644 index 000000000..047558be0 --- /dev/null +++ b/metagpt/exp_pool/context_builders/__init__.py @@ -0,0 +1,7 @@ +"""Context builders init.""" + +from metagpt.exp_pool.context_builders.base import BaseContextBuilder +from metagpt.exp_pool.context_builders.simple import SimpleContextBuilder +from metagpt.exp_pool.context_builders.role_zero import RoleZeroContextBuilder + +__all__ = ["BaseContextBuilder", "SimpleContextBuilder", "RoleZeroContextBuilder"] diff --git a/metagpt/exp_pool/context_builders/action_node.py b/metagpt/exp_pool/context_builders/action_node.py new file mode 100644 index 000000000..891b898be --- /dev/null +++ b/metagpt/exp_pool/context_builders/action_node.py @@ -0,0 +1,30 @@ +"""Action Node context builder.""" + +from typing import Any + +from metagpt.exp_pool.context_builders.base import BaseContextBuilder + +ACTION_NODE_CONTEXT_TEMPLATE = """ +{req} + +### Experiences +----- +{exps} +----- + +## Instruction +Consider **Experiences** to generate a better answer. +""" + + +class ActionNodeContextBuilder(BaseContextBuilder): + async def build(self, req: Any) -> str: + """Builds the action node context string. + + If there are no experiences, returns the original `req`; + otherwise returns context with `req` and formatted experiences. + """ + + exps = self.format_exps() + + return ACTION_NODE_CONTEXT_TEMPLATE.format(req=req, exps=exps) if exps else req diff --git a/metagpt/exp_pool/context_builders/base.py b/metagpt/exp_pool/context_builders/base.py new file mode 100644 index 000000000..691d51c8c --- /dev/null +++ b/metagpt/exp_pool/context_builders/base.py @@ -0,0 +1,41 @@ +"""Base context builder.""" + +from abc import ABC, abstractmethod +from typing import Any + +from pydantic import BaseModel, ConfigDict + +from metagpt.exp_pool.schema import Experience + +EXP_TEMPLATE = """Given the request: {req}, We can get the response: {resp}, which scored: {score}.""" + + +class BaseContextBuilder(BaseModel, ABC): + model_config = ConfigDict(arbitrary_types_allowed=True) + + exps: list[Experience] = [] + + @abstractmethod + async def build(self, req: Any) -> Any: + """Build context from req. + + Do not modify `req`. If modification is necessary, use copy.deepcopy to create a copy first. + """ + + def format_exps(self) -> str: + """Format experiences into a numbered list of strings. + + Example: + 1. Given the request: req1, We can get the response: resp1, which scored: 8. + 2. Given the request: req2, We can get the response: resp2, which scored: 9. + + Returns: + str: The formatted experiences as a string. + """ + + result = [] + for i, exp in enumerate(self.exps, start=1): + score_val = exp.metric.score.val if exp.metric and exp.metric.score else "N/A" + result.append(f"{i}. " + EXP_TEMPLATE.format(req=exp.req, resp=exp.resp, score=score_val)) + + return "\n".join(result) diff --git a/metagpt/exp_pool/context_builders/role_zero.py b/metagpt/exp_pool/context_builders/role_zero.py new file mode 100644 index 000000000..924bd56aa --- /dev/null +++ b/metagpt/exp_pool/context_builders/role_zero.py @@ -0,0 +1,56 @@ +"""RoleZero context builder.""" + +import copy +import re +from typing import Any + +from metagpt.exp_pool.context_builders.base import BaseContextBuilder + + +class RoleZeroContextBuilder(BaseContextBuilder): + async def build(self, req: Any) -> list[dict]: + """Builds the role zero context string. + + Note: + 1. The expected format for `req`, e.g., [{...}, {"role": "user", "content": "context"}]. + 2. Returns the original `req` if it is empty. + 3. Creates a copy of req and replaces the example content in the copied req with actual experiences. + """ + + if not req: + return req + + exps = self.format_exps() + if not exps: + return req + + req_copy = copy.deepcopy(req) + + req_copy[-1]["content"] = self.replace_example_content(req_copy[-1].get("content", ""), exps) + + return req_copy + + def replace_example_content(self, text: str, new_example_content: str) -> str: + return self.replace_content_between_markers(text, "# Example", "# Instruction", new_example_content) + + @staticmethod + def replace_content_between_markers(text: str, start_marker: str, end_marker: str, new_content: str) -> str: + """Replace the content between `start_marker` and `end_marker` in the text with `new_content`. + + Args: + text (str): The original text. + new_content (str): The new content to replace the old content. + start_marker (str): The marker indicating the start of the content to be replaced, such as '# Example'. + end_marker (str): The marker indicating the end of the content to be replaced, such as '# Instruction'. + + Returns: + str: The text with the content replaced. + """ + + pattern = re.compile(f"({start_marker}\n)(.*?)(\n{end_marker})", re.DOTALL) + + def replacement(match): + return f"{match.group(1)}{new_content}\n{match.group(3)}" + + replaced_text = pattern.sub(replacement, text) + return replaced_text diff --git a/metagpt/exp_pool/context_builders/simple.py b/metagpt/exp_pool/context_builders/simple.py new file mode 100644 index 000000000..d7b8d0be9 --- /dev/null +++ b/metagpt/exp_pool/context_builders/simple.py @@ -0,0 +1,26 @@ +"""Simple context builder.""" + + +from typing import Any + +from metagpt.exp_pool.context_builders.base import BaseContextBuilder + +SIMPLE_CONTEXT_TEMPLATE = """ +## Context + +### Experiences +----- +{exps} +----- + +## User Requirement +{req} + +## Instruction +Consider **Experiences** to generate a better answer. +""" + + +class SimpleContextBuilder(BaseContextBuilder): + async def build(self, req: Any) -> str: + return SIMPLE_CONTEXT_TEMPLATE.format(req=req, exps=self.format_exps()) diff --git a/metagpt/exp_pool/decorator.py b/metagpt/exp_pool/decorator.py new file mode 100644 index 000000000..7a2f926c5 --- /dev/null +++ b/metagpt/exp_pool/decorator.py @@ -0,0 +1,211 @@ +"""Experience Decorator.""" + +import asyncio +import functools +from typing import Any, Callable, Optional, TypeVar + +from pydantic import BaseModel, ConfigDict, model_validator + +from metagpt.config2 import config +from metagpt.exp_pool.context_builders import BaseContextBuilder, SimpleContextBuilder +from metagpt.exp_pool.manager import ExperienceManager, exp_manager +from metagpt.exp_pool.perfect_judges import BasePerfectJudge, SimplePerfectJudge +from metagpt.exp_pool.schema import Experience, Metric, QueryType, Score +from metagpt.exp_pool.scorers import BaseScorer, SimpleScorer +from metagpt.exp_pool.serializers import BaseSerializer, SimpleSerializer +from metagpt.logs import logger +from metagpt.utils.async_helper import NestAsyncio +from metagpt.utils.exceptions import handle_exception + +ReturnType = TypeVar("ReturnType") + + +def exp_cache( + _func: Optional[Callable[..., ReturnType]] = None, + query_type: QueryType = QueryType.SEMANTIC, + manager: Optional[ExperienceManager] = None, + scorer: Optional[BaseScorer] = None, + perfect_judge: Optional[BasePerfectJudge] = None, + context_builder: Optional[BaseContextBuilder] = None, + serializer: Optional[BaseSerializer] = None, + tag: Optional[str] = None, +): + """Decorator to get a perfect experience, otherwise, it executes the function, and create a new experience. + + Note: + 1. This can be applied to both synchronous and asynchronous functions. + 2. The function must have a `req` parameter, and it must be provided as a keyword argument. + 3. If `config.exp_pool.enable_read` is False, the decorator will just directly execute the function. + 4. If `config.exp_pool.enable_write` is False, the decorator will skip evaluating and saving the experience. + + Args: + _func: Just to make the decorator more flexible, for example, it can be used directly with @exp_cache by default, without the need for @exp_cache(). + query_type: The type of query to be used when fetching experiences. + manager: How to fetch, evaluate and save experience, etc. Default to `exp_manager`. + scorer: Evaluate experience. Default to `SimpleScorer()`. + perfect_judge: Determines if an experience is perfect. Defaults to `SimplePerfectJudge()`. + context_builder: Build the context from exps and the function parameters. Default to `SimpleContextBuilder()`. + serializer: Serializes the request and the function's return value for storage, deserializes the stored response back to the function's return value. Defaults to `SimpleSerializer()`. + tag: An optional tag for the experience. Default to `ClassName.method_name` or `function_name`. + """ + + def decorator(func: Callable[..., ReturnType]) -> Callable[..., ReturnType]: + if not config.exp_pool.enable_read: + return func + + @functools.wraps(func) + async def get_or_create(args: Any, kwargs: Any) -> ReturnType: + handler = ExpCacheHandler( + func=func, + args=args, + kwargs=kwargs, + query_type=query_type, + exp_manager=manager, + exp_scorer=scorer, + exp_perfect_judge=perfect_judge, + context_builder=context_builder, + serializer=serializer, + tag=tag, + ) + + await handler.fetch_experiences() + + if exp := await handler.get_one_perfect_exp(): + return exp + + await handler.execute_function() + + if config.exp_pool.enable_write: + await handler.process_experience() + + return handler._raw_resp + + return ExpCacheHandler.choose_wrapper(func, get_or_create) + + return decorator(_func) if _func else decorator + + +class ExpCacheHandler(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + func: Callable + args: Any + kwargs: Any + query_type: QueryType = QueryType.SEMANTIC + exp_manager: Optional[ExperienceManager] = None + exp_scorer: Optional[BaseScorer] = None + exp_perfect_judge: Optional[BasePerfectJudge] = None + context_builder: Optional[BaseContextBuilder] = None + serializer: Optional[BaseSerializer] = None + tag: Optional[str] = None + + _exps: list[Experience] = None + _req: str = "" + _resp: str = "" + _raw_resp: Any = None + _score: Score = None + + @model_validator(mode="after") + def initialize(self): + """Initialize default values for optional parameters if they are None. + + This is necessary because the decorator might pass None, which would override the default values set by Field. + """ + + self._validate_params() + + self.exp_manager = self.exp_manager or exp_manager + self.exp_scorer = self.exp_scorer or SimpleScorer() + self.exp_perfect_judge = self.exp_perfect_judge or SimplePerfectJudge() + self.context_builder = self.context_builder or SimpleContextBuilder() + self.serializer = self.serializer or SimpleSerializer() + self.tag = self.tag or self._generate_tag() + + self._req = self.serializer.serialize_req(**self.kwargs) + + return self + + async def fetch_experiences(self): + """Fetch experiences by query_type.""" + + self._exps = await self.exp_manager.query_exps(self._req, query_type=self.query_type, tag=self.tag) + + async def get_one_perfect_exp(self) -> Optional[Any]: + """Get a potentially perfect experience, and resolve resp.""" + + for exp in self._exps: + if await self.exp_perfect_judge.is_perfect_exp(exp, self._req, *self.args, **self.kwargs): + logger.info(f"Get one perfect experience: {exp.req[:20]}...") + return self.serializer.deserialize_resp(exp.resp) + + return None + + async def execute_function(self): + """Execute the function, and save resp.""" + + self._raw_resp = await self._execute_function() + self._resp = self.serializer.serialize_resp(self._raw_resp) + + @handle_exception + async def process_experience(self): + """Process experience. + + Evaluates and saves experience. + Use `handle_exception` to ensure robustness, do not stop subsequent operations. + """ + + await self.evaluate_experience() + self.save_experience() + + async def evaluate_experience(self): + """Evaluate the experience, and save the score.""" + + self._score = await self.exp_scorer.evaluate(self._req, self._resp) + + def save_experience(self): + """Save the new experience.""" + + exp = Experience(req=self._req, resp=self._resp, tag=self.tag, metric=Metric(score=self._score)) + self.exp_manager.create_exp(exp) + + @staticmethod + def choose_wrapper(func, wrapped_func): + """Choose how to run wrapped_func based on whether the function is asynchronous.""" + + async def async_wrapper(*args, **kwargs): + return await wrapped_func(args, kwargs) + + def sync_wrapper(*args, **kwargs): + NestAsyncio.apply_once() + return asyncio.get_event_loop().run_until_complete(wrapped_func(args, kwargs)) + + return async_wrapper if asyncio.iscoroutinefunction(func) else sync_wrapper + + def _validate_params(self): + if "req" not in self.kwargs: + raise ValueError("`req` must be provided as a keyword argument.") + + def _generate_tag(self) -> str: + """Generates a tag for the self.func. + + "ClassName.method_name" if the first argument is a class instance, otherwise just "function_name". + """ + + if self.args and hasattr(self.args[0], "__class__"): + cls_name = type(self.args[0]).__name__ + return f"{cls_name}.{self.func.__name__}" + + return self.func.__name__ + + async def _build_context(self) -> str: + self.context_builder.exps = self._exps + + return await self.context_builder.build(self.kwargs["req"]) + + async def _execute_function(self): + self.kwargs["req"] = await self._build_context() + + if asyncio.iscoroutinefunction(self.func): + return await self.func(*self.args, **self.kwargs) + + return self.func(*self.args, **self.kwargs) diff --git a/metagpt/exp_pool/manager.py b/metagpt/exp_pool/manager.py new file mode 100644 index 000000000..9bf289038 --- /dev/null +++ b/metagpt/exp_pool/manager.py @@ -0,0 +1,116 @@ +"""Experience Manager.""" + +from typing import TYPE_CHECKING, Any + +from pydantic import BaseModel, ConfigDict + +from metagpt.config2 import Config, config +from metagpt.exp_pool.schema import ( + DEFAULT_COLLECTION_NAME, + DEFAULT_SIMILARITY_TOP_K, + Experience, + QueryType, +) +from metagpt.logs import logger +from metagpt.utils.exceptions import handle_exception + +if TYPE_CHECKING: + from llama_index.vector_stores.chroma import ChromaVectorStore + + +class ExperienceManager(BaseModel): + """ExperienceManager manages the lifecycle of experiences, including CRUD and optimization. + + Args: + config (Config): Configuration for managing experiences. + _storage (SimpleEngine): Engine to handle the storage and retrieval of experiences. + _vector_store (ChromaVectorStore): The actual place where vectors are stored. + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + config: Config = config + + _storage: Any = None + _vector_store: Any = None + + @property + def storage(self): + if self._storage is None: + try: + from metagpt.rag.engines import SimpleEngine + from metagpt.rag.schema import ChromaRetrieverConfig, LLMRankerConfig + except ImportError: + raise ImportError("To use the experience pool, you need to install the rag module.") + + retriever_configs = [ + ChromaRetrieverConfig( + persist_path=self.config.exp_pool.persist_path, + collection_name=DEFAULT_COLLECTION_NAME, + similarity_top_k=DEFAULT_SIMILARITY_TOP_K, + ) + ] + ranker_configs = [LLMRankerConfig(top_n=DEFAULT_SIMILARITY_TOP_K)] + + self._storage: SimpleEngine = SimpleEngine.from_objs( + retriever_configs=retriever_configs, ranker_configs=ranker_configs + ) + logger.debug(f"exp_pool config: {self.config.exp_pool}") + + return self._storage + + @property + def vector_store(self): + if not self._vector_store: + self._vector_store: ChromaVectorStore = self.storage._retriever._vector_store + + return self._vector_store + + @handle_exception + def create_exp(self, exp: Experience): + """Adds an experience to the storage if writing is enabled. + + Args: + exp (Experience): The experience to add. + """ + + if not self.config.exp_pool.enable_write: + return + + self.storage.add_objs([exp]) + + @handle_exception(default_return=[]) + async def query_exps(self, req: str, tag: str = "", query_type: QueryType = QueryType.SEMANTIC) -> list[Experience]: + """Retrieves and filters experiences. + + Args: + req (str): The query string to retrieve experiences. + tag (str): Optional tag to filter the experiences by. + query_type (QueryType): Default semantic to vector matching. exact to same matching. + + Returns: + list[Experience]: A list of experiences that match the args. + """ + + if not self.config.exp_pool.enable_read: + return [] + + nodes = await self.storage.aretrieve(req) + exps: list[Experience] = [node.metadata["obj"] for node in nodes] + + # TODO: filter by metadata + if tag: + exps = [exp for exp in exps if exp.tag == tag] + + if query_type == QueryType.EXACT: + exps = [exp for exp in exps if exp.req == req] + + return exps + + def get_exps_count(self) -> int: + """Get the total number of experiences.""" + + return self.vector_store._collection.count() + + +exp_manager = ExperienceManager() diff --git a/metagpt/exp_pool/perfect_judges/__init__.py b/metagpt/exp_pool/perfect_judges/__init__.py new file mode 100644 index 000000000..d8796c7c8 --- /dev/null +++ b/metagpt/exp_pool/perfect_judges/__init__.py @@ -0,0 +1,6 @@ +"""Perfect judges init.""" + +from metagpt.exp_pool.perfect_judges.base import BasePerfectJudge +from metagpt.exp_pool.perfect_judges.simple import SimplePerfectJudge + +__all__ = ["BasePerfectJudge", "SimplePerfectJudge"] diff --git a/metagpt/exp_pool/perfect_judges/base.py b/metagpt/exp_pool/perfect_judges/base.py new file mode 100644 index 000000000..293522993 --- /dev/null +++ b/metagpt/exp_pool/perfect_judges/base.py @@ -0,0 +1,20 @@ +"""Base perfect judge.""" + +from abc import ABC, abstractmethod + +from pydantic import BaseModel, ConfigDict + +from metagpt.exp_pool.schema import Experience + + +class BasePerfectJudge(BaseModel, ABC): + model_config = ConfigDict(arbitrary_types_allowed=True) + + @abstractmethod + async def is_perfect_exp(self, exp: Experience, serialized_req: str, *args, **kwargs) -> bool: + """Determine whether the experience is perfect. + + Args: + exp (Experience): The experience to evaluate. + serialized_req (str): The serialized request to compare against the experience's request. + """ diff --git a/metagpt/exp_pool/perfect_judges/simple.py b/metagpt/exp_pool/perfect_judges/simple.py new file mode 100644 index 000000000..37ede95c3 --- /dev/null +++ b/metagpt/exp_pool/perfect_judges/simple.py @@ -0,0 +1,27 @@ +"""Simple perfect judge.""" + + +from pydantic import ConfigDict + +from metagpt.exp_pool.perfect_judges.base import BasePerfectJudge +from metagpt.exp_pool.schema import MAX_SCORE, Experience + + +class SimplePerfectJudge(BasePerfectJudge): + model_config = ConfigDict(arbitrary_types_allowed=True) + + async def is_perfect_exp(self, exp: Experience, serialized_req: str, *args, **kwargs) -> bool: + """Determine whether the experience is perfect. + + Args: + exp (Experience): The experience to evaluate. + serialized_req (str): The serialized request to compare against the experience's request. + + Returns: + bool: True if the serialized request matches the experience's request and the experience's score is perfect, False otherwise. + """ + + if not exp.metric or not exp.metric.score: + return False + + return serialized_req == exp.req and exp.metric.score.val == MAX_SCORE diff --git a/metagpt/exp_pool/schema.py b/metagpt/exp_pool/schema.py new file mode 100644 index 000000000..627dcbb4e --- /dev/null +++ b/metagpt/exp_pool/schema.py @@ -0,0 +1,72 @@ +"""Experience schema.""" + +from enum import Enum +from typing import Optional + +from pydantic import BaseModel, Field + +MAX_SCORE = 10 + +DEFAULT_COLLECTION_NAME = "experience_pool" +DEFAULT_SIMILARITY_TOP_K = 2 + + +class QueryType(str, Enum): + """Type of query experiences.""" + + EXACT = "exact" + SEMANTIC = "semantic" + + +class ExperienceType(str, Enum): + """Experience Type.""" + + SUCCESS = "success" + FAILURE = "failure" + INSIGHT = "insight" + + +class EntryType(Enum): + """Experience Entry Type.""" + + AUTOMATIC = "Automatic" + MANUAL = "Manual" + + +class Score(BaseModel): + """Score in Metric.""" + + val: int = Field(default=1, description="Value of the score, Between 1 and 10, higher is better.") + reason: str = Field(default="", description="Reason for the value.") + + +class Metric(BaseModel): + """Experience Metric.""" + + time_cost: float = Field(default=0.000, description="Time cost, the unit is milliseconds.") + money_cost: float = Field(default=0.000, description="Money cost, the unit is US dollars.") + score: Score = Field(default=None, description="Score, with value and reason.") + + +class Trajectory(BaseModel): + """Experience Trajectory.""" + + plan: str = Field(default="", description="The plan.") + action: str = Field(default="", description="Action for the plan.") + observation: str = Field(default="", description="Output of the action.") + reward: int = Field(default=0, description="Measure the action.") + + +class Experience(BaseModel): + """Experience.""" + + req: str = Field(..., description="") + resp: str = Field(..., description="The type is string/json/code.") + metric: Optional[Metric] = Field(default=None, description="Metric.") + exp_type: ExperienceType = Field(default=ExperienceType.SUCCESS, description="The type of experience.") + entry_type: EntryType = Field(default=EntryType.AUTOMATIC, description="Type of entry: Manual or Automatic.") + tag: str = Field(default="", description="Tagging experience.") + traj: Optional[Trajectory] = Field(default=None, description="Trajectory.") + + def rag_key(self): + return self.req diff --git a/metagpt/exp_pool/scorers/__init__.py b/metagpt/exp_pool/scorers/__init__.py new file mode 100644 index 000000000..caa845b14 --- /dev/null +++ b/metagpt/exp_pool/scorers/__init__.py @@ -0,0 +1,6 @@ +"""Scorers init.""" + +from metagpt.exp_pool.scorers.base import BaseScorer +from metagpt.exp_pool.scorers.simple import SimpleScorer + +__all__ = ["BaseScorer", "SimpleScorer"] diff --git a/metagpt/exp_pool/scorers/base.py b/metagpt/exp_pool/scorers/base.py new file mode 100644 index 000000000..97cac4992 --- /dev/null +++ b/metagpt/exp_pool/scorers/base.py @@ -0,0 +1,15 @@ +"""Base scorer.""" + +from abc import ABC, abstractmethod + +from pydantic import BaseModel, ConfigDict + +from metagpt.exp_pool.schema import Score + + +class BaseScorer(BaseModel, ABC): + model_config = ConfigDict(arbitrary_types_allowed=True) + + @abstractmethod + async def evaluate(self, req: str, resp: str) -> Score: + """Evaluates the quality of a response relative to a given request.""" diff --git a/metagpt/exp_pool/scorers/simple.py b/metagpt/exp_pool/scorers/simple.py new file mode 100644 index 000000000..4b060aac4 --- /dev/null +++ b/metagpt/exp_pool/scorers/simple.py @@ -0,0 +1,65 @@ +"""Simple scorer.""" + +import json + +from pydantic import Field + +from metagpt.exp_pool.schema import Score +from metagpt.exp_pool.scorers.base import BaseScorer +from metagpt.llm import LLM +from metagpt.provider.base_llm import BaseLLM +from metagpt.utils.common import CodeParser + +SIMPLE_SCORER_TEMPLATE = """ +Role: You are a highly efficient assistant, tasked with evaluating a response to a given request. The response is generated by a large language model (LLM). + +I will provide you with a request and a corresponding response. Your task is to assess this response and provide a score from a human perspective. + +## Context +### Request +{req} + +### Response +{resp} + +## Format Example +```json +{{ + "val": "the value of the score, int from 1 to 10, higher is better.", + "reason": "an explanation supporting the score." +}} +``` + +## Instructions +- Understand the request and response given by the user. +- Evaluate the response based on its quality relative to the given request. +- Provide a score from 1 to 10, where 10 is the best. +- Provide a reason supporting your score. + +## Constraint +Format: Just print the result in json format like **Format Example**. + +## Action +Follow instructions, generate output and make sure it follows the **Constraint**. +""" + + +class SimpleScorer(BaseScorer): + llm: BaseLLM = Field(default_factory=LLM) + + async def evaluate(self, req: str, resp: str) -> Score: + """Evaluates the quality of a response relative to a given request, as scored by an LLM. + + Args: + req (str): The request. + resp (str): The response. + + Returns: + Score: An object containing the score (1-10) and the reasoning. + """ + + prompt = SIMPLE_SCORER_TEMPLATE.format(req=req, resp=resp) + resp = await self.llm.aask(prompt) + resp_json = json.loads(CodeParser.parse_code(resp, lang="json")) + + return Score(**resp_json) diff --git a/metagpt/exp_pool/serializers/__init__.py b/metagpt/exp_pool/serializers/__init__.py new file mode 100644 index 000000000..8e1045588 --- /dev/null +++ b/metagpt/exp_pool/serializers/__init__.py @@ -0,0 +1,9 @@ +"""Serializers init.""" + +from metagpt.exp_pool.serializers.base import BaseSerializer +from metagpt.exp_pool.serializers.simple import SimpleSerializer +from metagpt.exp_pool.serializers.action_node import ActionNodeSerializer +from metagpt.exp_pool.serializers.role_zero import RoleZeroSerializer + + +__all__ = ["BaseSerializer", "SimpleSerializer", "ActionNodeSerializer", "RoleZeroSerializer"] diff --git a/metagpt/exp_pool/serializers/action_node.py b/metagpt/exp_pool/serializers/action_node.py new file mode 100644 index 000000000..7746d6be4 --- /dev/null +++ b/metagpt/exp_pool/serializers/action_node.py @@ -0,0 +1,36 @@ +"""ActionNode Serializer.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Type + +# Import ActionNode only for type checking to avoid circular imports +if TYPE_CHECKING: + from metagpt.actions.action_node import ActionNode + +from metagpt.exp_pool.serializers.simple import SimpleSerializer + + +class ActionNodeSerializer(SimpleSerializer): + def serialize_resp(self, resp: ActionNode) -> str: + return resp.instruct_content.model_dump_json() + + def deserialize_resp(self, resp: str) -> ActionNode: + """Customized deserialization, it will be triggered when a perfect experience is found. + + ActionNode cannot be serialized, it throws an error 'cannot pickle 'SSLContext' object'. + """ + + class InstructContent: + def __init__(self, json_data): + self.json_data = json_data + + def model_dump_json(self): + return self.json_data + + from metagpt.actions.action_node import ActionNode + + action_node = ActionNode(key="", expected_type=Type[str], instruction="", example="") + action_node.instruct_content = InstructContent(resp) + + return action_node diff --git a/metagpt/exp_pool/serializers/base.py b/metagpt/exp_pool/serializers/base.py new file mode 100644 index 000000000..c09488e12 --- /dev/null +++ b/metagpt/exp_pool/serializers/base.py @@ -0,0 +1,29 @@ +"""Base serializer.""" + +from abc import ABC, abstractmethod +from typing import Any + +from pydantic import BaseModel, ConfigDict + + +class BaseSerializer(BaseModel, ABC): + model_config = ConfigDict(arbitrary_types_allowed=True) + + @abstractmethod + def serialize_req(self, **kwargs) -> str: + """Serializes the request for storage. + + Do not modify kwargs. If modification is necessary, use copy.deepcopy to create a copy first. + Note that copy.deepcopy may raise errors, such as TypeError: cannot pickle '_thread.RLock' object. + """ + + @abstractmethod + def serialize_resp(self, resp: Any) -> str: + """Serializes the function's return value for storage. + + Do not modify resp. The rest is the same as `serialize_req`. + """ + + @abstractmethod + def deserialize_resp(self, resp: str) -> Any: + """Deserializes the stored response back to the function's return value""" diff --git a/metagpt/exp_pool/serializers/role_zero.py b/metagpt/exp_pool/serializers/role_zero.py new file mode 100644 index 000000000..89dd73f39 --- /dev/null +++ b/metagpt/exp_pool/serializers/role_zero.py @@ -0,0 +1,58 @@ +"""RoleZero Serializer.""" + +import copy +import json + +from metagpt.exp_pool.serializers.simple import SimpleSerializer + + +class RoleZeroSerializer(SimpleSerializer): + def serialize_req(self, **kwargs) -> str: + """Serialize the request for database storage, ensuring it is a string. + + Only extracts the necessary content from `req` because `req` may be very lengthy and could cause embedding errors. + + Args: + req (list[dict]): The request to be serialized. Example: + [ + {"role": "user", "content": "..."}, + {"role": "assistant", "content": "..."}, + {"role": "user", "content": "context"}, + ] + + Returns: + str: The serialized request as a JSON string. + """ + req = kwargs.get("req", []) + + if not req: + return "" + + filtered_req = self._filter_req(req) + + if state_data := kwargs.get("state_data"): + filtered_req.append({"role": "user", "content": state_data}) + + return json.dumps(filtered_req) + + def _filter_req(self, req: list[dict]) -> list[dict]: + """Filter the `req` to include only necessary items. + + Args: + req (list[dict]): The original request. + + Returns: + list[dict]: The filtered request. + """ + + filtered_req = [copy.deepcopy(item) for item in req if self._is_useful_content(item["content"])] + + return filtered_req + + def _is_useful_content(self, content: str) -> bool: + """Currently, only the content of the file is considered, and more judgments can be added later.""" + + if "Command Editor.read executed: file_path" in content: + return True + + return False diff --git a/metagpt/exp_pool/serializers/simple.py b/metagpt/exp_pool/serializers/simple.py new file mode 100644 index 000000000..ebd06e0e0 --- /dev/null +++ b/metagpt/exp_pool/serializers/simple.py @@ -0,0 +1,22 @@ +"""Simple Serializer.""" + +from typing import Any + +from metagpt.exp_pool.serializers.base import BaseSerializer + + +class SimpleSerializer(BaseSerializer): + def serialize_req(self, **kwargs) -> str: + """Just use `str` to convert the request object into a string.""" + + return str(kwargs.get("req", "")) + + def serialize_resp(self, resp: Any) -> str: + """Just use `str` to convert the response object into a string.""" + + return str(resp) + + def deserialize_resp(self, resp: str) -> Any: + """Just return the string response as it is.""" + + return resp diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py index 243f33604..890c1d562 100644 --- a/metagpt/prompts/di/role_zero.py +++ b/metagpt/prompts/di/role_zero.py @@ -8,7 +8,7 @@ Note: 2. Carefully review your progress at the current task, if your actions so far has not fulfilled the task instruction, you should continue with current task. Otherwise, finish current task by Plan.finish_current_task explicitly. 3. Each time you finish a task, use RoleZero.reply_to_human to report your progress. """ - +# To ensure compatibility with hard-coded experience, do not add any other content between "# Example" and "# Available Commands". CMD_PROMPT = """ # Data Structure class Task(BaseModel): @@ -17,7 +17,7 @@ class Task(BaseModel): instruction: str = "" task_type: str = "" assignee: str = "" - + # Available Task Types {task_type_desc} diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index 5f27c79ef..ce305707e 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -10,6 +10,9 @@ from pydantic import model_validator from metagpt.actions import Action, UserRequirement from metagpt.actions.di.run_command import RunCommand +from metagpt.exp_pool import exp_cache +from metagpt.exp_pool.context_builders import RoleZeroContextBuilder +from metagpt.exp_pool.serializers import RoleZeroSerializer from metagpt.logs import logger from metagpt.prompts.di.role_zero import ( CMD_PROMPT, @@ -143,25 +146,42 @@ class RoleZero(Role): tool_info = json.dumps({tool.name: tool.schemas for tool in tools}) ### Make Decision Dynamically ### + instruction = self.instruction.strip() prompt = self.cmd_prompt.format( - plan_status=plan_status, - current_task=current_task, example=example, available_commands=tool_info, - instruction=self.instruction.strip(), task_type_desc=self.task_type_desc, + plan_status=plan_status, + current_task=current_task, + instruction=instruction, ) memory = self.rc.memory.get(self.memory_k) memory = await self.parse_browser_actions(memory) - context = self.llm.format_msg(memory + [UserMessage(content=prompt)]) - # print(*context, sep="\n" + "*" * 5 + "\n") + + req = self.llm.format_msg(memory + [UserMessage(content=prompt)]) async with ThoughtReporter(enable_llm_stream=True) as reporter: await reporter.async_report({"type": "react"}) - self.command_rsp = await self.llm.aask(context, system_msgs=self.system_msg) + state_data = dict( + plan_status=plan_status, + current_task=current_task, + instruction=instruction, + ) + self.command_rsp = await self.llm_cached_aask(req=req, system_msgs=self.system_msg, state_data=state_data) + self.rc.memory.add(AIMessage(content=self.command_rsp)) return True + @exp_cache(context_builder=RoleZeroContextBuilder(), serializer=RoleZeroSerializer()) + async def llm_cached_aask(self, *, req: list[dict], system_msgs: list[str], **kwargs) -> str: + """Use `exp_cache` to automatically manage experiences. + + The `RoleZeroContextBuilder` attempts to add experiences to `req`. + The `RoleZeroSerializer` extracts essential parts of `req` for the experience pool, trimming lengthy entries to retain only necessary parts. + """ + + return await self.llm.aask(req, system_msgs=system_msgs) + async def parse_browser_actions(self, memory: List[Message]) -> List[Message]: if not self.browser.is_empty_page: pattern = re.compile(r"Command Browser\.(\w+) executed") diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index 668b44c40..ce5a8480e 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -791,13 +791,13 @@ Explanation: I will first need to read the system design document and the projec { "command_name": "Editor.read", "args": { - "path": "/tmp/docs/project_schedule.json" + "path": "/tmp/project_schedule.json" } }, { "command_name": "Editor.read", "args": { - "path": "/tmp/docs/system_design.json" + "path": "/tmp/system_design.json" } } ] diff --git a/metagpt/strategy/solver.py b/metagpt/strategy/solver.py index e532f736b..4aedb42aa 100644 --- a/metagpt/strategy/solver.py +++ b/metagpt/strategy/solver.py @@ -39,7 +39,7 @@ class NaiveSolver(BaseSolver): self.graph.topological_sort() for key in self.graph.execution_order: op = self.graph.nodes[key] - await op.fill(self.context, self.llm, mode="root") + await op.fill(req=self.context, llm=self.llm, mode="root") class TOTSolver(BaseSolver): diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index f6440b6ef..1019a17af 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -154,6 +154,7 @@ TOKEN_MAX = { "gpt-4-1106-preview": 128000, "gpt-4-vision-preview": 128000, "gpt-4-1106-vision-preview": 128000, + "gpt-4-turbo": 128000, "gpt-4": 8192, "gpt-4-0613": 8192, "gpt-4-32k": 32768, diff --git a/tests/metagpt/actions/test_action_node.py b/tests/metagpt/actions/test_action_node.py index bc85925a8..23779c984 100644 --- a/tests/metagpt/actions/test_action_node.py +++ b/tests/metagpt/actions/test_action_node.py @@ -91,10 +91,10 @@ async def test_action_node_two_layer(): assert node_b in root.children.values() # FIXME: ADD MARKDOWN SUPPORT. NEED TO TUNE MARKDOWN SYMBOL FIRST. - answer1 = await root.fill(context="what's the answer to 123+456?", schema="json", strgy="simple", llm=LLM()) + answer1 = await root.fill(req="what's the answer to 123+456?", schema="json", strgy="simple", llm=LLM()) assert "579" in answer1.content - answer2 = await root.fill(context="what's the answer to 123+456?", schema="json", strgy="complex", llm=LLM()) + answer2 = await root.fill(req="what's the answer to 123+456?", schema="json", strgy="complex", llm=LLM()) assert "579" in answer2.content @@ -112,7 +112,7 @@ async def test_action_node_review(): with pytest.raises(RuntimeError): _ = await node_a.review() - _ = await node_a.fill(context=None, llm=LLM()) + _ = await node_a.fill(req=None, llm=LLM()) setattr(node_a.instruct_content, key, "game snake") # wrong content to review review_comments = await node_a.review(review_mode=ReviewMode.AUTO) @@ -126,7 +126,7 @@ async def test_action_node_review(): with pytest.raises(RuntimeError): _ = await node.review() - _ = await node.fill(context=None, llm=LLM()) + _ = await node.fill(req=None, llm=LLM()) review_comments = await node.review(review_mode=ReviewMode.AUTO) assert len(review_comments) == 1 @@ -151,7 +151,7 @@ async def test_action_node_revise(): with pytest.raises(RuntimeError): _ = await node_a.review() - _ = await node_a.fill(context=None, llm=LLM()) + _ = await node_a.fill(req=None, llm=LLM()) setattr(node_a.instruct_content, key, "game snake") # wrong content to revise revise_contents = await node_a.revise(revise_mode=ReviseMode.AUTO) assert len(revise_contents) == 1 @@ -164,7 +164,7 @@ async def test_action_node_revise(): with pytest.raises(RuntimeError): _ = await node.revise() - _ = await node.fill(context=None, llm=LLM()) + _ = await node.fill(req=None, llm=LLM()) setattr(node.instruct_content, key, "game snake") revise_contents = await node.revise(revise_mode=ReviseMode.AUTO) assert len(revise_contents) == 1 @@ -257,7 +257,7 @@ async def test_action_node_with_image(mocker): invoice_path = Path(__file__).parent.joinpath("..", "..", "data", "invoices", "invoice-2.png") img_base64 = encode_image(invoice_path) mocker.patch("metagpt.provider.openai_api.OpenAILLM._cons_kwargs", _cons_kwargs) - node = await invoice.fill(context="", llm=LLM(), images=[img_base64]) + node = await invoice.fill(req="", llm=LLM(), images=[img_base64]) assert node.instruct_content.invoice diff --git a/tests/metagpt/actions/test_design_api_an.py b/tests/metagpt/actions/test_design_api_an.py index 3d11f200d..4ed3cb362 100644 --- a/tests/metagpt/actions/test_design_api_an.py +++ b/tests/metagpt/actions/test_design_api_an.py @@ -38,7 +38,7 @@ async def test_write_design_an(mocker): mocker.patch("metagpt.actions.design_api_an.REFINED_DESIGN_NODE.fill", return_value=root) prompt = NEW_REQ_TEMPLATE.format(old_design=DESIGN_SAMPLE, context=dict_to_markdown(REFINED_PRD_JSON)) - node = await REFINED_DESIGN_NODE.fill(prompt, llm) + node = await REFINED_DESIGN_NODE.fill(req=prompt, llm=llm) assert "Refined Implementation Approach" in node.instruct_content.model_dump() assert "Refined File list" in node.instruct_content.model_dump() diff --git a/tests/metagpt/actions/test_project_management_an.py b/tests/metagpt/actions/test_project_management_an.py index 5a65e50c9..6d41109c9 100644 --- a/tests/metagpt/actions/test_project_management_an.py +++ b/tests/metagpt/actions/test_project_management_an.py @@ -42,7 +42,7 @@ async def test_project_management_an(mocker): root.instruct_content.model_dump = mock_task_json mocker.patch("metagpt.actions.project_management_an.PM_NODE.fill", return_value=root) - node = await PM_NODE.fill(dict_to_markdown(REFINED_DESIGN_JSON), llm) + node = await PM_NODE.fill(req=dict_to_markdown(REFINED_DESIGN_JSON), llm=llm) assert "Logic Analysis" in node.instruct_content.model_dump() assert "Task list" in node.instruct_content.model_dump() @@ -59,7 +59,7 @@ async def test_project_management_an_inc(mocker): mocker.patch("metagpt.actions.project_management_an.REFINED_PM_NODE.fill", return_value=root) prompt = NEW_REQ_TEMPLATE.format(old_task=TASK_SAMPLE, context=dict_to_markdown(REFINED_DESIGN_JSON)) - node = await REFINED_PM_NODE.fill(prompt, llm) + node = await REFINED_PM_NODE.fill(req=prompt, llm=llm) assert "Refined Logic Analysis" in node.instruct_content.model_dump() assert "Refined Task list" in node.instruct_content.model_dump() diff --git a/tests/metagpt/actions/test_write_prd_an.py b/tests/metagpt/actions/test_write_prd_an.py index 378ce42c3..b6e92d3d6 100644 --- a/tests/metagpt/actions/test_write_prd_an.py +++ b/tests/metagpt/actions/test_write_prd_an.py @@ -39,7 +39,7 @@ async def test_write_prd_an(mocker): requirements=NEW_REQUIREMENT_SAMPLE, old_prd=PRD_SAMPLE, ) - node = await REFINED_PRD_NODE.fill(prompt, llm) + node = await REFINED_PRD_NODE.fill(req=prompt, llm=llm) assert "Refined Requirements" in node.instruct_content.model_dump() assert "Refined Product Goals" in node.instruct_content.model_dump() diff --git a/tests/metagpt/exp_pool/test_context_builders/test_base_context_builder.py b/tests/metagpt/exp_pool/test_context_builders/test_base_context_builder.py new file mode 100644 index 000000000..0a160fb42 --- /dev/null +++ b/tests/metagpt/exp_pool/test_context_builders/test_base_context_builder.py @@ -0,0 +1,32 @@ +import pytest + +from metagpt.exp_pool.context_builders.base import ( + EXP_TEMPLATE, + BaseContextBuilder, + Experience, +) +from metagpt.exp_pool.schema import Metric, Score + + +class TestBaseContextBuilder: + class ConcreteContextBuilder(BaseContextBuilder): + async def build(self, *args, **kwargs): + pass + + @pytest.fixture + def context_builder(self): + return self.ConcreteContextBuilder() + + def test_format_exps(self, context_builder): + exp1 = Experience(req="req1", resp="resp1", metric=Metric(score=Score(val=8))) + exp2 = Experience(req="req2", resp="resp2", metric=Metric(score=Score(val=9))) + context_builder.exps = [exp1, exp2] + + result = context_builder.format_exps() + expected = "\n".join( + [ + f"1. {EXP_TEMPLATE.format(req='req1', resp='resp1', score=8)}", + f"2. {EXP_TEMPLATE.format(req='req2', resp='resp2', score=9)}", + ] + ) + assert result == expected diff --git a/tests/metagpt/exp_pool/test_context_builders/test_rolezero_context_builder.py b/tests/metagpt/exp_pool/test_context_builders/test_rolezero_context_builder.py new file mode 100644 index 000000000..b7182602d --- /dev/null +++ b/tests/metagpt/exp_pool/test_context_builders/test_rolezero_context_builder.py @@ -0,0 +1,51 @@ +import pytest + +from metagpt.exp_pool.context_builders.base import BaseContextBuilder +from metagpt.exp_pool.context_builders.role_zero import RoleZeroContextBuilder + + +class TestRoleZeroContextBuilder: + @pytest.fixture + def context_builder(self): + return RoleZeroContextBuilder() + + @pytest.mark.asyncio + async def test_build_empty_req(self, context_builder): + result = await context_builder.build(req=[]) + assert result == [] + + @pytest.mark.asyncio + async def test_build_no_experiences(self, context_builder, mocker): + mocker.patch.object(BaseContextBuilder, "format_exps", return_value="") + req = [{"content": "Original content"}] + result = await context_builder.build(req=req) + assert result == req + + @pytest.mark.asyncio + async def test_build_with_experiences(self, context_builder, mocker): + mocker.patch.object(BaseContextBuilder, "format_exps", return_value="Formatted experiences") + mocker.patch.object(RoleZeroContextBuilder, "replace_example_content", return_value="Updated content") + req = [{"content": "Original content 1"}] + result = await context_builder.build(req=req) + assert result == [{"content": "Updated content"}] + + def test_replace_example_content(self, context_builder, mocker): + mocker.patch.object(RoleZeroContextBuilder, "replace_content_between_markers", return_value="Replaced content") + result = context_builder.replace_example_content("Original text", "New example content") + assert result == "Replaced content" + context_builder.replace_content_between_markers.assert_called_once_with( + "Original text", "# Example", "# Instruction", "New example content" + ) + + def test_replace_content_between_markers(self): + text = "Start\n# Example\nOld content\n# Instruction\nEnd" + new_content = "New content" + result = RoleZeroContextBuilder.replace_content_between_markers(text, "# Example", "# Instruction", new_content) + expected = "Start\n# Example\nNew content\n\n# Instruction\nEnd" + assert result == expected + + def test_replace_content_between_markers_no_match(self): + text = "Start\nNo markers\nEnd" + new_content = "New content" + result = RoleZeroContextBuilder.replace_content_between_markers(text, "# Example", "# Instruction", new_content) + assert result == text diff --git a/tests/metagpt/exp_pool/test_context_builders/test_simple_context_builder.py b/tests/metagpt/exp_pool/test_context_builders/test_simple_context_builder.py new file mode 100644 index 000000000..cf1a42f27 --- /dev/null +++ b/tests/metagpt/exp_pool/test_context_builders/test_simple_context_builder.py @@ -0,0 +1,47 @@ +import pytest + +from metagpt.exp_pool.context_builders.base import BaseContextBuilder +from metagpt.exp_pool.context_builders.simple import ( + SIMPLE_CONTEXT_TEMPLATE, + SimpleContextBuilder, +) + + +class TestSimpleContextBuilder: + @pytest.fixture + def context_builder(self): + return SimpleContextBuilder() + + @pytest.mark.asyncio + async def test_build_with_experiences(self, mocker, context_builder: SimpleContextBuilder): + # Mock the format_exps method + mock_exps = "Mocked experiences" + mocker.patch.object(BaseContextBuilder, "format_exps", return_value=mock_exps) + + req = "Test request" + result = await context_builder.build(req=req) + + expected = SIMPLE_CONTEXT_TEMPLATE.format(req=req, exps=mock_exps) + assert result == expected + + @pytest.mark.asyncio + async def test_build_without_experiences(self, mocker, context_builder: SimpleContextBuilder): + # Mock the format_exps method to return an empty string + mocker.patch.object(BaseContextBuilder, "format_exps", return_value="") + + req = "Test request" + result = await context_builder.build(req=req) + + expected = SIMPLE_CONTEXT_TEMPLATE.format(req=req, exps="") + assert result == expected + + @pytest.mark.asyncio + async def test_build_without_req(self, mocker, context_builder: SimpleContextBuilder): + # Mock the format_exps method + mock_exps = "Mocked experiences" + mocker.patch.object(BaseContextBuilder, "format_exps", return_value=mock_exps) + + result = await context_builder.build(req="") + + expected = SIMPLE_CONTEXT_TEMPLATE.format(req="", exps=mock_exps) + assert result == expected diff --git a/tests/metagpt/exp_pool/test_decorator.py b/tests/metagpt/exp_pool/test_decorator.py new file mode 100644 index 000000000..0c02dcdfc --- /dev/null +++ b/tests/metagpt/exp_pool/test_decorator.py @@ -0,0 +1,200 @@ +import asyncio + +import pytest + +from metagpt.config2 import Config +from metagpt.configs.exp_pool_config import ExperiencePoolConfig +from metagpt.exp_pool.context_builders import SimpleContextBuilder +from metagpt.exp_pool.decorator import ExpCacheHandler, exp_cache +from metagpt.exp_pool.manager import ExperienceManager +from metagpt.exp_pool.perfect_judges import SimplePerfectJudge +from metagpt.exp_pool.schema import Experience, QueryType, Score +from metagpt.exp_pool.scorers import SimpleScorer +from metagpt.rag.engines import SimpleEngine + + +class TestExpCacheHandler: + @pytest.fixture + def mock_func(self, mocker): + return mocker.AsyncMock() + + @pytest.fixture + def mock_exp_manager(self, mocker): + manager = mocker.MagicMock(spec=ExperienceManager) + manager.storage = mocker.MagicMock(spec=SimpleEngine) + manager.config = mocker.MagicMock(spec=Config) + manager.config.exp_pool = ExperiencePoolConfig() + manager.query_exps = mocker.AsyncMock() + manager.create_exp = mocker.MagicMock() + return manager + + @pytest.fixture + def mock_scorer(self, mocker): + scorer = mocker.MagicMock(spec=SimpleScorer) + scorer.evaluate = mocker.AsyncMock() + return scorer + + @pytest.fixture + def mock_perfect_judge(self, mocker): + return mocker.MagicMock(spec=SimplePerfectJudge) + + @pytest.fixture + def mock_context_builder(self, mocker): + return mocker.MagicMock(spec=SimpleContextBuilder) + + @pytest.fixture + def exp_cache_handler(self, mock_func, mock_exp_manager, mock_scorer, mock_perfect_judge, mock_context_builder): + return ExpCacheHandler( + func=mock_func, + args=(), + kwargs={"req": "test_req"}, + exp_manager=mock_exp_manager, + exp_scorer=mock_scorer, + exp_perfect_judge=mock_perfect_judge, + context_builder=mock_context_builder, + ) + + @pytest.mark.asyncio + async def test_fetch_experiences(self, exp_cache_handler, mock_exp_manager): + mock_exp_manager.query_exps.return_value = [Experience(req="test_req", resp="test_resp")] + await exp_cache_handler.fetch_experiences() + mock_exp_manager.query_exps.assert_called_once_with( + "test_req", query_type=QueryType.SEMANTIC, tag=exp_cache_handler.tag + ) + assert len(exp_cache_handler._exps) == 1 + + @pytest.mark.asyncio + async def test_get_one_perfect_exp(self, exp_cache_handler, mock_perfect_judge): + exp = Experience(req="test_req", resp="perfect_resp") + exp_cache_handler._exps = [exp] + mock_perfect_judge.is_perfect_exp.return_value = True + result = await exp_cache_handler.get_one_perfect_exp() + assert result == "perfect_resp" + + @pytest.mark.asyncio + async def test_execute_function(self, exp_cache_handler, mock_func, mock_context_builder): + mock_context_builder.build.return_value = "built_context" + mock_func.return_value = "function_result" + await exp_cache_handler.execute_function() + mock_context_builder.build.assert_called_once() + mock_func.assert_called_once_with(req="built_context") + assert exp_cache_handler._raw_resp == "function_result" + assert exp_cache_handler._resp == "function_result" + + @pytest.mark.asyncio + async def test_process_experience(self, exp_cache_handler, mock_scorer, mock_exp_manager): + exp_cache_handler._resp = "test_resp" + mock_scorer.evaluate.return_value = Score(val=8) + await exp_cache_handler.process_experience() + mock_scorer.evaluate.assert_called_once() + mock_exp_manager.create_exp.assert_called_once() + + @pytest.mark.asyncio + async def test_evaluate_experience(self, exp_cache_handler, mock_scorer): + exp_cache_handler._resp = "test_resp" + mock_scorer.evaluate.return_value = Score(val=9) + await exp_cache_handler.evaluate_experience() + assert exp_cache_handler._score.val == 9 + + def test_save_experience(self, exp_cache_handler, mock_exp_manager): + exp_cache_handler._req = "test_req" + exp_cache_handler._resp = "test_resp" + exp_cache_handler._score = Score(val=7) + exp_cache_handler.save_experience() + mock_exp_manager.create_exp.assert_called_once() + + def test_choose_wrapper_async(self, mocker): + async def async_func(): + pass + + wrapper = ExpCacheHandler.choose_wrapper(async_func, mocker.AsyncMock()) + assert asyncio.iscoroutinefunction(wrapper) + + def test_choose_wrapper_sync(self, mocker): + def sync_func(): + pass + + wrapper = ExpCacheHandler.choose_wrapper(sync_func, mocker.AsyncMock()) + assert not asyncio.iscoroutinefunction(wrapper) + + def test_validate_params(self): + with pytest.raises(ValueError): + ExpCacheHandler(func=lambda x: x, args=(), kwargs={}) + + def test_generate_tag(self): + class TestClass: + def test_method(self): + pass + + handler = ExpCacheHandler(func=TestClass().test_method, args=(TestClass(),), kwargs={"req": "test"}) + assert handler._generate_tag() == "TestClass.test_method" + + handler = ExpCacheHandler(func=lambda x: x, args=(), kwargs={"req": "test"}) + assert handler._generate_tag() == "" + + +class TestExpCache: + @pytest.fixture + def mock_exp_manager(self, mocker, mock_config): + manager = mocker.MagicMock(spec=ExperienceManager) + manager.storage = mocker.MagicMock(spec=SimpleEngine) + manager.config = mock_config + manager.query_exps = mocker.AsyncMock() + manager.create_exp = mocker.MagicMock() + return manager + + @pytest.fixture + def mock_scorer(self, mocker): + scorer = mocker.MagicMock(spec=SimpleScorer) + scorer.evaluate = mocker.AsyncMock(return_value=Score()) + return scorer + + @pytest.fixture + def mock_perfect_judge(self, mocker): + return mocker.MagicMock(spec=SimplePerfectJudge) + + @pytest.fixture + def mock_config(self, mocker): + return mocker.patch("metagpt.exp_pool.decorator.config") + + @pytest.mark.asyncio + async def test_exp_cache_disabled(self, mock_config, mock_exp_manager): + mock_config.exp_pool.enable_read = False + + @exp_cache(manager=mock_exp_manager) + async def test_func(req): + return "result" + + result = await test_func(req="test") + assert result == "result" + mock_exp_manager.query_exps.assert_not_called() + + @pytest.mark.asyncio + async def test_exp_cache_enabled_no_perfect_exp(self, mock_config, mock_exp_manager, mock_scorer): + mock_config.exp_pool.enable_read = True + mock_exp_manager.query_exps.return_value = [] + + @exp_cache(manager=mock_exp_manager, scorer=mock_scorer) + async def test_func(req): + return "computed_result" + + result = await test_func(req="test") + assert result == "computed_result" + mock_exp_manager.query_exps.assert_called() + mock_exp_manager.create_exp.assert_called() + + @pytest.mark.asyncio + async def test_exp_cache_enabled_with_perfect_exp(self, mock_config, mock_exp_manager, mock_perfect_judge): + mock_config.exp_pool.enable_read = True + perfect_exp = Experience(req="test", resp="perfect_result") + mock_exp_manager.query_exps.return_value = [perfect_exp] + mock_perfect_judge.is_perfect_exp.return_value = True + + @exp_cache(manager=mock_exp_manager, perfect_judge=mock_perfect_judge) + async def test_func(req): + return "should_not_be_called" + + result = await test_func(req="test") + assert result == "perfect_result" + mock_exp_manager.query_exps.assert_called_once() + mock_exp_manager.create_exp.assert_not_called() diff --git a/tests/metagpt/exp_pool/test_manager.py b/tests/metagpt/exp_pool/test_manager.py new file mode 100644 index 000000000..3fb2aaa6d --- /dev/null +++ b/tests/metagpt/exp_pool/test_manager.py @@ -0,0 +1,78 @@ +import pytest + +from metagpt.config2 import Config +from metagpt.configs.exp_pool_config import ExperiencePoolConfig +from metagpt.configs.llm_config import LLMConfig +from metagpt.exp_pool.manager import Experience, ExperienceManager +from metagpt.exp_pool.schema import QueryType + + +class TestExperienceManager: + @pytest.fixture + def mock_config(self): + return Config(llm=LLMConfig(), exp_pool=ExperiencePoolConfig(enable_write=True, enable_read=True)) + + @pytest.fixture + def mock_storage(self, mocker): + engine = mocker.MagicMock() + engine.add_objs = mocker.MagicMock() + engine.aretrieve = mocker.AsyncMock(return_value=[]) + engine._retriever = mocker.MagicMock() + engine._retriever._vector_store = mocker.MagicMock() + engine._retriever._vector_store._collection = mocker.MagicMock() + engine._retriever._vector_store._collection.count = mocker.MagicMock(return_value=10) + return engine + + @pytest.fixture + def exp_manager(self, mock_config, mock_storage): + manager = ExperienceManager(config=mock_config) + manager._storage = mock_storage + return manager + + def test_vector_store_property(self, exp_manager): + assert exp_manager.vector_store == exp_manager.storage._retriever._vector_store + + @pytest.mark.asyncio + async def test_query_exps_with_exact_match(self, exp_manager, mocker): + req = "exact query" + exp1 = Experience(req=req, resp="response1") + exp2 = Experience(req="different query", resp="response2") + + mock_node1 = mocker.MagicMock(metadata={"obj": exp1}) + mock_node2 = mocker.MagicMock(metadata={"obj": exp2}) + + exp_manager.storage.aretrieve.return_value = [mock_node1, mock_node2] + + result = await exp_manager.query_exps(req, query_type=QueryType.EXACT) + assert len(result) == 1 + assert result[0].req == req + + @pytest.mark.asyncio + async def test_query_exps_with_tag_filter(self, exp_manager, mocker): + tag = "test_tag" + exp1 = Experience(req="query1", resp="response1", tag=tag) + exp2 = Experience(req="query2", resp="response2", tag="other_tag") + + mock_node1 = mocker.MagicMock(metadata={"obj": exp1}) + mock_node2 = mocker.MagicMock(metadata={"obj": exp2}) + + exp_manager.storage.aretrieve.return_value = [mock_node1, mock_node2] + + result = await exp_manager.query_exps("query", tag=tag) + assert len(result) == 1 + assert result[0].tag == tag + + def test_get_exps_count(self, exp_manager): + assert exp_manager.get_exps_count() == 10 + + def test_create_exp_write_disabled(self, exp_manager, mock_config): + mock_config.exp_pool.enable_write = False + exp = Experience(req="test", resp="response") + exp_manager.create_exp(exp) + exp_manager.storage.add_objs.assert_not_called() + + @pytest.mark.asyncio + async def test_query_exps_read_disabled(self, exp_manager, mock_config): + mock_config.exp_pool.enable_read = False + result = await exp_manager.query_exps("query") + assert result == [] diff --git a/tests/metagpt/exp_pool/test_perfect_judges/test_simple_perfect_judge.py b/tests/metagpt/exp_pool/test_perfect_judges/test_simple_perfect_judge.py new file mode 100644 index 000000000..5abd04f0d --- /dev/null +++ b/tests/metagpt/exp_pool/test_perfect_judges/test_simple_perfect_judge.py @@ -0,0 +1,40 @@ +import pytest + +from metagpt.exp_pool.perfect_judges import SimplePerfectJudge +from metagpt.exp_pool.schema import MAX_SCORE, Experience, Metric, Score + + +class TestSimplePerfectJudge: + @pytest.fixture + def simple_perfect_judge(self): + return SimplePerfectJudge() + + @pytest.mark.asyncio + async def test_is_perfect_exp_perfect_match(self, simple_perfect_judge): + exp = Experience(req="test_request", resp="resp", metric=Metric(score=Score(val=MAX_SCORE))) + result = await simple_perfect_judge.is_perfect_exp(exp, "test_request") + assert result is True + + @pytest.mark.asyncio + async def test_is_perfect_exp_imperfect_score(self, simple_perfect_judge): + exp = Experience(req="test_request", resp="resp", metric=Metric(score=Score(val=MAX_SCORE - 1))) + result = await simple_perfect_judge.is_perfect_exp(exp, "test_request") + assert result is False + + @pytest.mark.asyncio + async def test_is_perfect_exp_mismatched_request(self, simple_perfect_judge): + exp = Experience(req="test_request", resp="resp", metric=Metric(score=Score(val=MAX_SCORE))) + result = await simple_perfect_judge.is_perfect_exp(exp, "different_request") + assert result is False + + @pytest.mark.asyncio + async def test_is_perfect_exp_no_metric(self, simple_perfect_judge): + exp = Experience(req="test_request", resp="resp") + result = await simple_perfect_judge.is_perfect_exp(exp, "test_request") + assert result is False + + @pytest.mark.asyncio + async def test_is_perfect_exp_no_score(self, simple_perfect_judge): + exp = Experience(req="test_request", resp="resp", metric=Metric()) + result = await simple_perfect_judge.is_perfect_exp(exp, "test_request") + assert result is False diff --git a/tests/metagpt/exp_pool/test_scorers/test_simple_scorer.py b/tests/metagpt/exp_pool/test_scorers/test_simple_scorer.py new file mode 100644 index 000000000..e17edfca8 --- /dev/null +++ b/tests/metagpt/exp_pool/test_scorers/test_simple_scorer.py @@ -0,0 +1,64 @@ +import json + +import pytest + +from metagpt.exp_pool.schema import Score +from metagpt.exp_pool.scorers.simple import SIMPLE_SCORER_TEMPLATE, SimpleScorer +from metagpt.llm import BaseLLM + + +class TestSimpleScorer: + @pytest.fixture + def mock_llm(self, mocker): + mock_llm = mocker.MagicMock(spec=BaseLLM) + return mock_llm + + @pytest.fixture + def simple_scorer(self, mock_llm): + return SimpleScorer(llm=mock_llm) + + def test_init(self, mock_llm): + scorer = SimpleScorer(llm=mock_llm) + assert isinstance(scorer.llm, BaseLLM) + + @pytest.mark.asyncio + async def test_evaluate(self, simple_scorer, mock_llm, mocker): + # Mock request and response + req = "What is the capital of France?" + resp = "The capital of France is Paris." + + # Mock LLM response + mock_llm_response = '{"val": 9, "reason": "Accurate and concise answer"}' + mock_llm.aask.return_value = f"```json\n{mock_llm_response}\n```" + + # Mock CodeParser.parse_code + mocker.patch("metagpt.utils.common.CodeParser.parse_code", return_value=mock_llm_response) + + # Test evaluate method + result = await simple_scorer.evaluate(req, resp) + + # Assert LLM was called with correct prompt + expected_prompt = SIMPLE_SCORER_TEMPLATE.format(req=req, resp=resp) + mock_llm.aask.assert_called_once_with(expected_prompt) + + # Assert the result is correct + assert isinstance(result, Score) + assert result.val == 9 + assert result.reason == "Accurate and concise answer" + + @pytest.mark.asyncio + async def test_evaluate_invalid_response(self, simple_scorer, mock_llm, mocker): + # Mock request and response + req = "What is the capital of France?" + resp = "The capital of France is Paris." + + # Mock LLM response with invalid JSON + mock_llm_response = "Invalid JSON" + mock_llm.aask.return_value = f"```json\n{mock_llm_response}\n```" + + # Mock CodeParser.parse_code + mocker.patch("metagpt.utils.common.CodeParser.parse_code", return_value=mock_llm_response) + + # Test evaluate method with invalid response + with pytest.raises(json.JSONDecodeError): + await simple_scorer.evaluate(req, resp) diff --git a/tests/metagpt/exp_pool/test_serializers/test_action_node.py b/tests/metagpt/exp_pool/test_serializers/test_action_node.py new file mode 100644 index 000000000..e4ab4684d --- /dev/null +++ b/tests/metagpt/exp_pool/test_serializers/test_action_node.py @@ -0,0 +1,35 @@ +from typing import Type + +import pytest + +from metagpt.actions.action_node import ActionNode +from metagpt.exp_pool.serializers.action_node import ActionNodeSerializer + + +class TestActionNodeSerializer: + @pytest.fixture + def serializer(self): + return ActionNodeSerializer() + + @pytest.fixture + def action_node(self): + class InstructContent: + def __init__(self, json_data): + self.json_data = json_data + + def model_dump_json(self): + return self.json_data + + action_node = ActionNode(key="", expected_type=Type[str], instruction="", example="") + action_node.instruct_content = InstructContent('{"key": "value"}') + + return action_node + + def test_serialize_resp(self, serializer: ActionNodeSerializer, action_node: ActionNode): + serialized = serializer.serialize_resp(action_node) + assert serialized == '{"key": "value"}' + + def test_deserialize_resp(self, serializer: ActionNodeSerializer): + deserialized = serializer.deserialize_resp('{"key": "value"}') + assert isinstance(deserialized, ActionNode) + assert deserialized.instruct_content.model_dump_json() == '{"key": "value"}' diff --git a/tests/metagpt/exp_pool/test_serializers/test_role_zero.py b/tests/metagpt/exp_pool/test_serializers/test_role_zero.py new file mode 100644 index 000000000..964443f29 --- /dev/null +++ b/tests/metagpt/exp_pool/test_serializers/test_role_zero.py @@ -0,0 +1,46 @@ +import json + +import pytest + +from metagpt.exp_pool.serializers import RoleZeroSerializer + + +class TestRoleZeroSerializer: + @pytest.fixture + def serializer(self) -> RoleZeroSerializer: + return RoleZeroSerializer() + + @pytest.fixture + def last_item(self) -> dict: + return { + "role": "user", + "content": "# Current Plan\nsome plan\n# Current Plan\nsome plan\n# Instruction\nsome instruction", + } + + @pytest.fixture + def sample_req(self): + return [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}] + + def test_serialize_req_empty_input(self, serializer: RoleZeroSerializer): + assert serializer.serialize_req(req=[]) == "" + + def test_serialize_req_with_content(self, serializer: RoleZeroSerializer, last_item: dict): + req = [ + {"role": "user", "content": "Command Editor.read executed: file_path=test.py"}, + {"role": "assistant", "content": "Some other content"}, + last_item, + ] + expected_output = json.dumps([{"role": "user", "content": "Command Editor.read executed: file_path=test.py"}]) + assert serializer.serialize_req(req=req) == expected_output + + def test_filter_req(self, serializer: RoleZeroSerializer): + req = [ + {"role": "user", "content": "Command Editor.read executed: file_path=test1.py"}, + {"role": "assistant", "content": "Some other content"}, + {"role": "user", "content": "Command Editor.read executed: file_path=test2.py"}, + {"role": "assistant", "content": "Final content"}, + ] + filtered_req = serializer._filter_req(req) + assert len(filtered_req) == 2 + assert filtered_req[0]["content"] == "Command Editor.read executed: file_path=test1.py" + assert filtered_req[1]["content"] == "Command Editor.read executed: file_path=test2.py" diff --git a/tests/metagpt/exp_pool/test_serializers/test_simple.py b/tests/metagpt/exp_pool/test_serializers/test_simple.py new file mode 100644 index 000000000..2a6bf96e3 --- /dev/null +++ b/tests/metagpt/exp_pool/test_serializers/test_simple.py @@ -0,0 +1,44 @@ +import pytest + +from metagpt.exp_pool.serializers.simple import SimpleSerializer + + +class TestSimpleSerializer: + @pytest.fixture + def serializer(self): + return SimpleSerializer() + + def test_serialize_req(self, serializer: SimpleSerializer): + # Test with different types of input + assert serializer.serialize_req(req=123) == "123" + assert serializer.serialize_req(req="test") == "test" + assert serializer.serialize_req(req=[1, 2, 3]) == "[1, 2, 3]" + assert serializer.serialize_req(req={"a": 1}) == "{'a': 1}" + + def test_serialize_resp(self, serializer: SimpleSerializer): + # Test with different types of input + assert serializer.serialize_resp(456) == "456" + assert serializer.serialize_resp("response") == "response" + assert serializer.serialize_resp([4, 5, 6]) == "[4, 5, 6]" + assert serializer.serialize_resp({"b": 2}) == "{'b': 2}" + + def test_deserialize_resp(self, serializer: SimpleSerializer): + # Test with different types of input + assert serializer.deserialize_resp("789") == "789" + assert serializer.deserialize_resp("test_response") == "test_response" + assert serializer.deserialize_resp("[7, 8, 9]") == "[7, 8, 9]" + assert serializer.deserialize_resp("{'c': 3}") == "{'c': 3}" + + def test_roundtrip(self, serializer: SimpleSerializer): + # Test serialization and deserialization roundtrip + original = "test_roundtrip" + serialized = serializer.serialize_resp(original) + deserialized = serializer.deserialize_resp(serialized) + assert deserialized == original + + @pytest.mark.parametrize("input_value", [123, "test", [1, 2, 3], {"a": 1}, None]) + def test_serialize_req_types(self, serializer: SimpleSerializer, input_value): + # Test serialize_req with various input types + result = serializer.serialize_req(req=input_value) + assert isinstance(result, str) + assert result == str(input_value)