diff --git a/config/config2.example.yaml b/config/config2.example.yaml index f1158775b..c5ca6e767 100644 --- a/config/config2.example.yaml +++ b/config/config2.example.yaml @@ -74,6 +74,9 @@ s3: secure: false bucket: "test" +experience_pool: + enable_read: false + enable_write: false azure_tts_subscription_key: "YOUR_SUBSCRIPTION_KEY" azure_tts_region: "eastus" diff --git a/examples/exp_pool/manager.py b/examples/exp_pool/manager.py new file mode 100644 index 000000000..f5766f9a5 --- /dev/null +++ b/examples/exp_pool/manager.py @@ -0,0 +1,21 @@ +from metagpt.exp_pool.manager import ExperiencePoolManager +from metagpt.exp_pool.schema import Experience +from pprint import pprint +import asyncio +# import logging +# logging.basicConfig(level=logging.DEBUG) + +async def main(): + req = "2048 game" + exp = Experience(req=req, resp="python code") + + manager = ExperiencePoolManager() + + # pprint(manager.storage.get()) + # manager.create_exp(exp) + result = await manager.query_exp(req) + print(result) + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/metagpt/config2.py b/metagpt/config2.py index 717fe63a9..6f5a1add6 100644 --- a/metagpt/config2.py +++ b/metagpt/config2.py @@ -22,6 +22,7 @@ from metagpt.configs.search_config import SearchConfig from metagpt.configs.workspace_config import WorkspaceConfig from metagpt.const import CONFIG_ROOT, METAGPT_ROOT from metagpt.utils.yaml_model import YamlModel +from metagpt.configs.exp_pool_config import ExperiencePoolConfig class CLIParams(BaseModel): @@ -71,6 +72,9 @@ class Config(CLIParams, YamlModel): enable_longterm_memory: bool = False code_review_k_times: int = 2 + # Experience Pool Parameters + experience_pool: Optional[ExperiencePoolConfig] = None + # Will be removed in the future metagpt_tti_url: str = "" language: str = "English" diff --git a/metagpt/configs/exp_pool_config.py b/metagpt/configs/exp_pool_config.py new file mode 100644 index 000000000..f7312d2de --- /dev/null +++ b/metagpt/configs/exp_pool_config.py @@ -0,0 +1,6 @@ +from metagpt.utils.yaml_model import YamlModel + + +class ExperiencePoolConfig(YamlModel): + enable_read: bool = False + enable_write: bool = False diff --git a/metagpt/exp_pool/__init__.py b/metagpt/exp_pool/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/metagpt/exp_pool/decorator.py b/metagpt/exp_pool/decorator.py new file mode 100644 index 000000000..6629e8377 --- /dev/null +++ b/metagpt/exp_pool/decorator.py @@ -0,0 +1,4 @@ + + +def exp_cache(func): + pass diff --git a/metagpt/exp_pool/manager.py b/metagpt/exp_pool/manager.py new file mode 100644 index 000000000..c32073a9f --- /dev/null +++ b/metagpt/exp_pool/manager.py @@ -0,0 +1,32 @@ +from pydantic import BaseModel, ConfigDict +from metagpt.exp_pool.schema import Experience +import uuid +import chromadb +from chromadb import Collection, QueryResult +from typing import Optional +from metagpt.rag.engines import SimpleEngine +from metagpt.rag.schema import ChromaRetrieverConfig + + +class ExperiencePoolManager(BaseModel): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._storage = None + + @property + def storage(self) -> SimpleEngine: + if self._storage is None: + self._storage = SimpleEngine.from_objs(retriever_configs=[ChromaRetrieverConfig(collection_name="experience_pool", persist_path="./chroma_data")]) + return self._storage + + def create_exp(self, exp: Experience): + self.storage.add_objs([exp]) + + async def query_exp(self, req: str) -> list[Experience]: + nodes = await self.storage.aretrieve(req) + exps = [node.metadata["obj"] for node in nodes] + + return exps + + + diff --git a/metagpt/exp_pool/schema.py b/metagpt/exp_pool/schema.py new file mode 100644 index 000000000..359268612 --- /dev/null +++ b/metagpt/exp_pool/schema.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel, Field +from llama_index.core.schema import TextNode + + +class Experience(BaseModel): + req: str = Field(..., description="") + resp: str = Field(..., description="") + + def rag_key(self): + return self.req + + +class ExperienceNodeMetadata(BaseModel): + """Metadata of ExperienceNode.""" + + resp: str = Field(..., description="") + + +class ExperienceNode(TextNode): + """ExperienceNode for RAG.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.excluded_llm_metadata_keys = list(ExperienceNodeMetadata.model_fields.keys()) + self.excluded_embed_metadata_keys = self.excluded_llm_metadata_keys diff --git a/metagpt/utils/reflection.py b/metagpt/utils/reflection.py index 8b8237ae7..688831f06 100644 --- a/metagpt/utils/reflection.py +++ b/metagpt/utils/reflection.py @@ -1,4 +1,5 @@ """class tools, including method inspection, class attributes, inheritance relationships, etc.""" +import inspect def check_methods(C, *methods): @@ -16,3 +17,11 @@ def check_methods(C, *methods): else: return NotImplemented return True + + +def get_func_full_name(func, *args) -> str: + if inspect.ismethod(func) or (inspect.isfunction(func) and "self" in inspect.signature(func).parameters): + cls_name = args[0].__class__.__name__ + return f"{func.__module__}.{cls_name}.{func.__name__}" + + return f"{func.__module__}.{func.__name__}" diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index 0ba2daa89..496842a2d 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -150,6 +150,8 @@ TOKEN_MAX = { "gpt-4-1106-preview": 128000, "gpt-4-vision-preview": 128000, "gpt-4-1106-vision-preview": 128000, + "gpt-4-turbo": 128000, + "gpt-4o": 128000, "gpt-4": 8192, "gpt-4-0613": 8192, "gpt-4-32k": 32768,