diff --git a/examples/exp_pool/init_exp_pool.py b/examples/exp_pool/init_exp_pool.py index 1601abe0b..62747b8d8 100644 --- a/examples/exp_pool/init_exp_pool.py +++ b/examples/exp_pool/init_exp_pool.py @@ -8,7 +8,7 @@ import json from pathlib import Path from metagpt.const import EXAMPLE_DATA_PATH -from metagpt.exp_pool import exp_manager +from metagpt.exp_pool import get_exp_manager from metagpt.exp_pool.schema import EntryType, Experience, Metric, Score from metagpt.logs import logger from metagpt.utils.common import aread @@ -45,7 +45,8 @@ async def add_exp(req: str, resp: str, tag: str, metric: Metric = None): tag=tag, metric=metric or Metric(score=Score(val=10, reason="Manual")), ) - + exp_manager = get_exp_manager() + exp_manager.config.exp_pool.enabled = True exp_manager.config.exp_pool.enable_write = True exp_manager.create_exp(exp) logger.info(f"New experience created for the request `{req[:10]}`.") @@ -59,8 +60,10 @@ async def add_exps(exps: list, tag: str): tag: A tag for categorizing the experiences. """ - - tasks = [add_exp(req=json.dumps(exp["req"]), resp=exp["resp"], tag=tag) for exp in exps] + tasks = [ + add_exp(req=exp["req"] if isinstance(exp["req"], str) else json.dumps(exp["req"]), resp=exp["resp"], tag=tag) + for exp in exps + ] await asyncio.gather(*tasks) @@ -79,7 +82,7 @@ async def add_exps_from_file(tag: str, filepath: Path): def query_exps_count(): """Queries and logs the total count of experiences in the pool.""" - + exp_manager = get_exp_manager() count = exp_manager.get_exps_count() logger.info(f"Experiences Count: {count}") diff --git a/examples/exp_pool/manager.py b/examples/exp_pool/manager.py index ae998214a..c9ec46da5 100644 --- a/examples/exp_pool/manager.py +++ b/examples/exp_pool/manager.py @@ -6,7 +6,7 @@ This script creates a new experience, logs its creation, and then queries for ex import asyncio -from metagpt.exp_pool import exp_manager +from metagpt.exp_pool import get_exp_manager from metagpt.exp_pool.schema import EntryType, Experience from metagpt.logs import logger @@ -18,6 +18,7 @@ async def main(): # Add the new experience exp = Experience(req=req, resp=resp, entry_type=EntryType.MANUAL) + exp_manager = get_exp_manager() exp_manager.create_exp(exp) logger.info(f"New experience created for the request `{req}`.") diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 6ae21ebd9..4476e7c0a 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -32,6 +32,7 @@ from metagpt.tools.tool_registry import register_tool from metagpt.utils.common import ( aread, awrite, + rectify_pathname, save_json_to_markdown, to_markdown_code_block, ) @@ -262,10 +263,10 @@ class WriteDesign(Action): ) if not output_pathname: - output_pathname = Path(output_pathname) / "docs" / "sytem_design.json" + output_pathname = Path(output_pathname) / "docs" / "system_design.json" elif not Path(output_pathname).is_absolute(): output_pathname = self.config.workspace.path / output_pathname - output_pathname = Path(output_pathname) + output_pathname = rectify_pathname(path=output_pathname, default_filename="system_design.json") await awrite(filename=output_pathname, data=design.content) output_filename = output_pathname.parent / f"{output_pathname.stem}-class-diagram" await self._save_data_api_design(design_doc=design, output_filename=output_filename) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index b9d3bc3ba..2d54ffe08 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -26,6 +26,7 @@ from metagpt.tools.tool_registry import register_tool from metagpt.utils.common import ( aread, awrite, + rectify_pathname, save_json_to_markdown, to_markdown_code_block, ) @@ -191,7 +192,7 @@ class WriteTasks(Action): output_pathname = Path(output_pathname) / "docs" / "project_schedule.json" elif not Path(output_pathname).is_absolute(): output_pathname = self.config.workspace.path / output_pathname - output_pathname = Path(output_pathname) + output_pathname = rectify_pathname(path=output_pathname, default_filename="project_schedule.json") await awrite(filename=output_pathname, data=file_content) md_output_filename = output_pathname.with_suffix(".md") await save_json_to_markdown(content=file_content, output_filename=md_output_filename) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index e10e7333a..4b2015145 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -43,6 +43,7 @@ from metagpt.utils.common import ( CodeParser, aread, awrite, + rectify_pathname, save_json_to_markdown, to_markdown_code_block, ) @@ -314,7 +315,7 @@ class WritePRD(Action): output_pathname = self.config.workspace.path / "docs" / "prd.json" elif not Path(output_pathname).is_absolute(): output_pathname = self.config.workspace.path / output_pathname - output_pathname = Path(output_pathname) + output_pathname = rectify_pathname(path=output_pathname, default_filename="prd.json") await awrite(filename=output_pathname, data=new_prd.content) competitive_analysis_filename = output_pathname.parent / f"{output_pathname.stem}-competitive-analysis" await self._save_competitive_analysis(prd_doc=new_prd, output_filename=Path(competitive_analysis_filename)) diff --git a/metagpt/exp_pool/decorator.py b/metagpt/exp_pool/decorator.py index ed5f5e068..bb3d434c0 100644 --- a/metagpt/exp_pool/decorator.py +++ b/metagpt/exp_pool/decorator.py @@ -35,8 +35,10 @@ def exp_cache( Note: 1. This can be applied to both synchronous and asynchronous functions. 2. The function must have a `req` parameter, and it must be provided as a keyword argument. - 3. If `config.exp_pool.enable_read` is False, the decorator will just directly execute the function. + 3. If `config.exp_pool.enabled` is False, the decorator will just directly execute the function. 4. If `config.exp_pool.enable_write` is False, the decorator will skip evaluating and saving the experience. + 5. If `config.exp_pool.enable_read` is False, the decorator will skip reading from the experience pool. + Args: _func: Just to make the decorator more flexible, for example, it can be used directly with @exp_cache by default, without the need for @exp_cache(). diff --git a/metagpt/prompts/di/data_analyst.py b/metagpt/prompts/di/data_analyst.py index 9d7f3ec50..8e5b888d3 100644 --- a/metagpt/prompts/di/data_analyst.py +++ b/metagpt/prompts/di/data_analyst.py @@ -8,8 +8,8 @@ EXTRA_INSTRUCTION = """ - For information searching requirement, you should use the Browser tool instead of web scraping. - When no link is provided, you should use the Browser tool to search for the information. 7. When you are making plan. It is highly recommend to plan and append all the tasks in first response once time, except for 7.1. -7.1. When the requirement is given with a file, read the file first through either Editor.read (write code instead for excel) WITHOUT a plan. After reading the file content, use RoleZero.reply_to_human if the requirement can be answered straightaway, otherwise, make a plan if further calculation is needed. -8. Don't finish_current_task multiple times for the same task. +7.1. When the requirement is inquiring about a pdf, docx, md, or txt document, read the document first through either Editor.read WITHOUT a plan. After reading the document, use RoleZero.reply_to_human if the requirement can be answered straightaway, otherwise, make a plan if further calculation is needed. +8. Don't finish_current_task multiple times for the same task. 9. Finish current task timely, such as when the code is written and executed successfully. 10. When using the command 'end', add the command 'finish_current_task' before it. """ diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py index 24ca29af5..4c407fb97 100644 --- a/metagpt/prompts/di/role_zero.py +++ b/metagpt/prompts/di/role_zero.py @@ -22,6 +22,8 @@ Note: ########################### SYSTEM_PROMPT = """ +# Basic Info +{role_info} # Data Structure class Task(BaseModel): diff --git a/metagpt/rag/factories/embedding.py b/metagpt/rag/factories/embedding.py index 8a9d4bc95..d647883bd 100644 --- a/metagpt/rag/factories/embedding.py +++ b/metagpt/rag/factories/embedding.py @@ -29,7 +29,7 @@ class RAGEmbeddingFactory(GenericFactory): LLMType.AZURE: self._create_azure, } super().__init__(creators) - self.config = config if self.config else Config.default() + self.config = config if config else Config.default() def get_rag_embedding(self, key: EmbeddingType = None) -> BaseEmbedding: """Key is EmbeddingType.""" diff --git a/metagpt/rag/factories/llm.py b/metagpt/rag/factories/llm.py index 5d27cde3a..59f6db4d9 100644 --- a/metagpt/rag/factories/llm.py +++ b/metagpt/rag/factories/llm.py @@ -10,7 +10,7 @@ from llama_index.core.llms import ( LLMMetadata, ) from llama_index.core.llms.callbacks import llm_completion_callback -from pydantic import Field, model_validator +from pydantic import Field from metagpt.config2 import Config from metagpt.llm import LLM @@ -30,19 +30,30 @@ class RAGLLM(CustomLLM): num_output: int = -1 model_name: str = "" - @model_validator(mode="after") - def update_from_config(self): + def __init__( + self, + model_infer: BaseLLM, + context_window: int = -1, + num_output: int = -1, + model_name: str = "", + *args, + **kwargs + ): + super().__init__(*args, **kwargs) config = Config.default() - if self.context_window < 0: - self.context_window = TOKEN_MAX.get(config.llm.model, DEFAULT_CONTEXT_WINDOW) + if context_window < 0: + context_window = TOKEN_MAX.get(config.llm.model, DEFAULT_CONTEXT_WINDOW) - if self.num_output < 0: - self.num_output = config.llm.max_token + if num_output < 0: + num_output = config.llm.max_token - if not self.model_name: - self.model_name = config.llm.model + if not model_name: + model_name = config.llm.model - return self + self.model_infer = model_infer + self.context_window = context_window + self.num_output = num_output + self.model_name = model_name @property def metadata(self) -> LLMMetadata: diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index 32508910c..4c4f063f2 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -166,7 +166,11 @@ class RoleZero(Role): ### Role Instruction ### instruction = self.instruction.strip() system_prompt = self.system_prompt.format( - task_type_desc=self.task_type_desc, available_commands=tool_info, example=example, instruction=instruction + role_info=self._get_prefix(), + task_type_desc=self.task_type_desc, + available_commands=tool_info, + example=example, + instruction=instruction, ) ### Make Decision Dynamically ### @@ -284,7 +288,9 @@ class RoleZero(Role): # routing memory = self.get_memories(k=4) # FIXME: A magic number for two rounds of Q&A context = self.llm.format_msg(memory + [UserMessage(content=QUICK_THINK_PROMPT)]) - intent_result = await self.llm.aask(context, system_msgs=[self.format_quick_system_prompt()]) + async with ThoughtReporter() as reporter: + await reporter.async_report({"type": "classify"}) + intent_result = await self.llm.aask(context, system_msgs=[self.format_quick_system_prompt()]) if "QUICK" in intent_result or "AMBIGUOUS" in intent_result: # llm call with the original context async with ThoughtReporter(enable_llm_stream=True) as reporter: diff --git a/metagpt/roles/di/team_leader.py b/metagpt/roles/di/team_leader.py index 4a39193a2..97100f295 100644 --- a/metagpt/roles/di/team_leader.py +++ b/metagpt/roles/di/team_leader.py @@ -47,12 +47,12 @@ class TeamLeader(RoleZero): # continue team_info += f"{role.name}: {role.profile}, {role.goal}\n" return team_info - + def _get_prefix(self) -> str: role_info = super()._get_prefix() team_info = self._get_team_info() return TL_INFO.format(role_info=role_info, team_info=team_info) - + async def _think(self) -> bool: self.instruction = TL_INSTRUCTION.format(team_info=self._get_team_info()) return await super()._think() diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index d8023715b..542993516 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -26,7 +26,7 @@ class FileBlock(BaseModel): class Editor(BaseModel): """ A tool for reading, understanding, writing, and editing files. - Support local file including text-based files (txt, md, json, py, html, js, css, etc.), pdf, docx, excluding images, excel, or online links + Support local file including text-based files (txt, md, json, py, html, js, css, etc.), pdf, docx, excluding images, csv, excel, or online links """ model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index fa2f3cbbc..def127fc9 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -1118,3 +1118,25 @@ def log_time(method): return result return timeit_wrapper_async if iscoroutinefunction(method) else timeit_wrapper + + +def rectify_pathname(path: Union[str, Path], default_filename: str) -> Path: + """ + Rectifies the given path to ensure a valid output file path. + + If the given `path` is a directory, it creates the directory (if it doesn't exist) and appends the `default_filename` to it. If the `path` is a file path, it creates the parent directory (if it doesn't exist) and returns the `path`. + + Args: + path (Union[str, Path]): The input path, which can be a string or a `Path` object. + default_filename (str): The default filename to use if the `path` is a directory. + + Returns: + Path: The rectified output path. + """ + output_pathname = Path(path) + if output_pathname.is_dir(): + output_pathname.mkdir(parents=True, exist_ok=True) + output_pathname = output_pathname / default_filename + else: + output_pathname.parent.mkdir(parents=True, exist_ok=True) + return output_pathname diff --git a/tests/metagpt/exp_pool/test_decorator.py b/tests/metagpt/exp_pool/test_decorator.py index 0ca4c6ce1..9d104fca4 100644 --- a/tests/metagpt/exp_pool/test_decorator.py +++ b/tests/metagpt/exp_pool/test_decorator.py @@ -155,7 +155,10 @@ class TestExpCache: @pytest.fixture def mock_config(self, mocker): - return mocker.patch("metagpt.exp_pool.decorator.config") + config = Config.default().model_copy(deep=True) + default = mocker.patch("metagpt.config2.Config.default") + default.return_value = config + return config @pytest.mark.asyncio async def test_exp_cache_disabled(self, mock_config, mock_exp_manager): @@ -171,7 +174,9 @@ class TestExpCache: @pytest.mark.asyncio async def test_exp_cache_enabled_no_perfect_exp(self, mock_config, mock_exp_manager, mock_scorer): + mock_config.exp_pool.enabled = True mock_config.exp_pool.enable_read = True + mock_config.exp_pool.enable_write = True mock_exp_manager.query_exps.return_value = [] @exp_cache(manager=mock_exp_manager, scorer=mock_scorer) @@ -185,6 +190,7 @@ class TestExpCache: @pytest.mark.asyncio async def test_exp_cache_enabled_with_perfect_exp(self, mock_config, mock_exp_manager, mock_perfect_judge): + mock_config.exp_pool.enabled = True mock_config.exp_pool.enable_read = True perfect_exp = Experience(req="test", resp="perfect_result") mock_exp_manager.query_exps.return_value = [perfect_exp]