From 5b15584480702eabd57d00c03233702e101e89c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 1 Jul 2024 20:23:44 +0800 Subject: [PATCH 1/6] feat: +runtime multi-llm support --- config/config2.example.yaml | 18 ++++ metagpt/actions/action.py | 14 ++- metagpt/configs/llm_config.py | 11 +- metagpt/configs/models_config.py | 112 ++++++++++++++++++++ metagpt/utils/token_counter.py | 1 - tests/data/config/config2.yaml | 27 +++++ tests/data/requirements/full.txt | 0 tests/metagpt/configs/__init__.py | 0 tests/metagpt/configs/test_models_config.py | 34 ++++++ 9 files changed, 210 insertions(+), 7 deletions(-) create mode 100644 metagpt/configs/models_config.py create mode 100644 tests/data/config/config2.yaml create mode 100644 tests/data/requirements/full.txt create mode 100644 tests/metagpt/configs/__init__.py create mode 100644 tests/metagpt/configs/test_models_config.py diff --git a/config/config2.example.yaml b/config/config2.example.yaml index 64cce630f..6986e9acb 100644 --- a/config/config2.example.yaml +++ b/config/config2.example.yaml @@ -59,3 +59,21 @@ iflytek_api_key: "YOUR_API_KEY" iflytek_api_secret: "YOUR_API_SECRET" metagpt_tti_url: "YOUR_MODEL_URL" + +models: +# "YOUR_MODEL_NAME_1": # model: "gpt-4-turbo" # or gpt-3.5-turbo +# api_type: "openai" # or azure / ollama / groq etc. +# base_url: "YOUR_BASE_URL" +# api_key: "YOUR_API_KEY" +# proxy: "YOUR_PROXY" # for LLM API requests +# # timeout: 600 # Optional. If set to 0, default value is 300. +# # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ +# pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's +# "YOUR_API_TYPE": # api_type: "openai" # or azure / ollama / groq etc. +# api_type: "openai" # or azure / ollama / groq etc. +# base_url: "YOUR_BASE_URL" +# api_key: "YOUR_API_KEY" +# proxy: "YOUR_PROXY" # for LLM API requests +# # timeout: 600 # Optional. If set to 0, default value is 300. +# # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ +# pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's \ No newline at end of file diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 1b93213f7..a1ab5c2bc 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -8,11 +8,12 @@ from __future__ import annotations -from typing import Optional, Union +from typing import Any, Optional, Union from pydantic import BaseModel, ConfigDict, Field, model_validator from metagpt.actions.action_node import ActionNode +from metagpt.configs.models_config import ModelsConfig from metagpt.context_mixin import ContextMixin from metagpt.schema import ( CodePlanAndChangeContext, @@ -35,6 +36,17 @@ class Action(SerializationMixin, ContextMixin, BaseModel): prefix: str = "" # aask*时会加上prefix，作为system_message desc: str = "" # for skill manager node: ActionNode = Field(default=None, exclude=True) + # The model name or API type of LLM of the `models` in the `config2.yaml`; + # Using `None` to use the `llm` configuration in the `config2.yaml`. + llm_name_or_type: Optional[str] = None + + @model_validator(mode="after") + @classmethod + def _update_private_llm(cls, data: Any) -> Any: + config = ModelsConfig.default().get(data.llm_name_or_type) + if config: + data.llm.config = config + return data @property def repo(self) -> ProjectRepo: diff --git a/metagpt/configs/llm_config.py b/metagpt/configs/llm_config.py index 0284c8993..67fb6afdb 100644 --- a/metagpt/configs/llm_config.py +++ b/metagpt/configs/llm_config.py @@ -10,9 +10,9 @@ from typing import Optional from pydantic import field_validator -from metagpt.const import LLM_API_TIMEOUT +from metagpt.const import CONFIG_ROOT, LLM_API_TIMEOUT, METAGPT_ROOT from metagpt.utils.yaml_model import YamlModel -from metagpt.const import METAGPT_ROOT, CONFIG_ROOT + class LLMType(Enum): OPENAI = "openai" @@ -97,12 +97,13 @@ class LLMConfig(YamlModel): repo_config_path = METAGPT_ROOT / "config/config2.yaml" root_config_path = CONFIG_ROOT / "config2.yaml" if root_config_path.exists(): - raise ValueError( - f"Please set your API key in {root_config_path}. If you also set your config in {repo_config_path}, \nthe former will overwrite the latter. This may cause unexpected result.\n") + raise ValueError( + f"Please set your API key in {root_config_path}. If you also set your config in {repo_config_path}, \nthe former will overwrite the latter. This may cause unexpected result.\n" + ) elif repo_config_path.exists(): raise ValueError(f"Please set your API key in {repo_config_path}") else: - raise ValueError(f"Please set your API key in config2.yaml") + raise ValueError("Please set your API key in config2.yaml") return v @field_validator("timeout") diff --git a/metagpt/configs/models_config.py b/metagpt/configs/models_config.py new file mode 100644 index 000000000..9aaa79702 --- /dev/null +++ b/metagpt/configs/models_config.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +models_config.py + +This module defines the ModelsConfig class for handling configuration of LLM models. + +Attributes: + CONFIG_ROOT (Path): Root path for configuration files. + METAGPT_ROOT (Path): Root path for MetaGPT files. + +Classes: + ModelsConfig (YamlModel): Configuration class for LLM models. +""" +from pathlib import Path +from typing import Dict, List, Optional + +from pydantic import Field, field_validator + +from metagpt.config2 import merge_dict +from metagpt.configs.llm_config import LLMConfig +from metagpt.const import CONFIG_ROOT, METAGPT_ROOT +from metagpt.utils.yaml_model import YamlModel + + +class ModelsConfig(YamlModel): + """ + Configuration class for LLM models. + + Attributes: + models (Dict[str, LLMConfig]): Dictionary mapping model names to LLMConfig objects. + + Methods: + update_llm_model(cls, value): Validates and updates LLM model configurations. + from_home(cls, path): Loads configuration from ~/.metagpt/config2.yaml. + default(cls): Loads default configuration from predefined paths. + get(self, name_or_type: str) -> Optional[LLMConfig]: Retrieves LLMConfig by name or API type. + """ + + models: Dict[str, LLMConfig] = Field(default_factory=dict) + + @field_validator("models", mode="before") + @classmethod + def update_llm_model(cls, value): + """ + Validates and updates LLM model configurations. + + Args: + value (Dict[str, Union[LLMConfig, dict]]): Dictionary of LLM configurations. + + Returns: + Dict[str, Union[LLMConfig, dict]]: Updated dictionary of LLM configurations. + """ + for key, config in value.items(): + if isinstance(config, LLMConfig): + config.model = config.model or key + elif isinstance(config, dict): + config["model"] = config.get("model") or key + return value + + @classmethod + def from_home(cls, path): + """ + Loads configuration from ~/.metagpt/config2.yaml. + + Args: + path (str): Relative path to configuration file. + + Returns: + Optional[ModelsConfig]: Loaded ModelsConfig object or None if file doesn't exist. + """ + pathname = CONFIG_ROOT / path + if not pathname.exists(): + return None + return ModelsConfig.from_yaml_file(pathname) + + @classmethod + def default(cls): + """ + Loads default configuration from predefined paths. + + Returns: + ModelsConfig: Default ModelsConfig object. + """ + default_config_paths: List[Path] = [ + METAGPT_ROOT / "config/config2.yaml", + CONFIG_ROOT / "config2.yaml", + ] + + dicts = [ModelsConfig.read_yaml(path) for path in default_config_paths] + final = merge_dict(dicts) + return ModelsConfig(**final) + + def get(self, name_or_type: str) -> Optional[LLMConfig]: + """ + Retrieves LLMConfig object by name or API type. + + Args: + name_or_type (str): Name or API type of the LLM model. + + Returns: + Optional[LLMConfig]: LLMConfig object if found, otherwise None. + """ + if not name_or_type: + return None + model = self.models.get(name_or_type) + if model: + return model + for m in self.models.values(): + if m.api_type == name_or_type: + return m + return None diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index b3391a7e2..973263d55 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -259,7 +259,6 @@ TOKEN_MAX = { "qwen-7b-chat": 32000, "qwen-1.8b-longcontext-chat": 32000, "qwen-1.8b-chat": 8000, - } # For Amazon Bedrock US region diff --git a/tests/data/config/config2.yaml b/tests/data/config/config2.yaml new file mode 100644 index 000000000..e0cefd8ff --- /dev/null +++ b/tests/data/config/config2.yaml @@ -0,0 +1,27 @@ +llm: + api_type: "openai" # or azure / ollama / groq etc. + base_url: "YOUR_gpt-3.5-turbo_BASE_URL" + api_key: "YOUR_gpt-3.5-turbo_API_KEY" + model: "gpt-3.5-turbo" # or gpt-3.5-turbo + proxy: "YOUR_gpt-3.5-turbo_PROXY" # for LLM API requests + # timeout: 600 # Optional. If set to 0, default value is 300. + # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ + pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's + +models: + "YOUR_MODEL_NAME_1": # model: "gpt-4-turbo" # or gpt-3.5-turbo + api_type: "openai" # or azure / ollama / groq etc. + base_url: "YOUR_MODEL_1_BASE_URL" + api_key: "YOUR_MODEL_1_API_KEY" + proxy: "YOUR_MODEL_1_PROXY" # for LLM API requests + # timeout: 600 # Optional. If set to 0, default value is 300. + # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ + pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's + "YOUR_MODEL_NAME_2": # model: "gpt-4-turbo" # or gpt-3.5-turbo + api_type: "openai" # or azure / ollama / groq etc. + base_url: "YOUR_MODEL_2_BASE_URL" + api_key: "YOUR_MODEL_2_API_KEY" + proxy: "YOUR_MODEL_2_PROXY" # for LLM API requests + # timeout: 600 # Optional. If set to 0, default value is 300. + # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ + pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's \ No newline at end of file diff --git a/tests/data/requirements/full.txt b/tests/data/requirements/full.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/metagpt/configs/__init__.py b/tests/metagpt/configs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/metagpt/configs/test_models_config.py b/tests/metagpt/configs/test_models_config.py new file mode 100644 index 000000000..cfbf1f96b --- /dev/null +++ b/tests/metagpt/configs/test_models_config.py @@ -0,0 +1,34 @@ +import pytest + +from metagpt.actions.talk_action import TalkAction +from metagpt.configs.models_config import ModelsConfig +from metagpt.const import METAGPT_ROOT, TEST_DATA_PATH +from metagpt.utils.common import aread, awrite + + +@pytest.mark.asyncio +async def test_models_configs(context): + default_model = ModelsConfig.default() + assert default_model is not None + + models = ModelsConfig.from_yaml_file(TEST_DATA_PATH / "config/config2.yaml") + assert models + + default_models = ModelsConfig.default() + backup = "" + if not default_models.models: + backup = await aread(filename=METAGPT_ROOT / "config/config2.yaml") + test_data = await aread(filename=TEST_DATA_PATH / "config/config2.yaml") + await awrite(filename=METAGPT_ROOT / "config/config2.yaml", data=test_data) + + try: + action = TalkAction(context=context, i_context="who are you?", llm_name_or_type="YOUR_MODEL_NAME_1") + assert action.private_llm.config.model == "YOUR_MODEL_NAME_1" + assert context.config.llm.model != "YOUR_MODEL_NAME_1" + finally: + if backup: + await awrite(filename=METAGPT_ROOT / "config/config2.yaml", data=backup) + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From d7381a745574abdabb319cca069b5863397347a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 1 Jul 2024 20:35:03 +0800 Subject: [PATCH 2/6] refactor: comments --- metagpt/configs/models_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/configs/models_config.py b/metagpt/configs/models_config.py index 9aaa79702..bc4897fec 100644 --- a/metagpt/configs/models_config.py +++ b/metagpt/configs/models_config.py @@ -25,10 +25,10 @@ from metagpt.utils.yaml_model import YamlModel class ModelsConfig(YamlModel): """ - Configuration class for LLM models. + Configuration class for `models` in `config2.yaml`. Attributes: - models (Dict[str, LLMConfig]): Dictionary mapping model names to LLMConfig objects. + models (Dict[str, LLMConfig]): Dictionary mapping model names or types to LLMConfig objects. Methods: update_llm_model(cls, value): Validates and updates LLM model configurations. From a69715bd4ffad6c66f771d61b7e34a59dd9206c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 16 Jul 2024 14:43:23 +0800 Subject: [PATCH 3/6] refactor: comments --- config/config2.example.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config2.example.yaml b/config/config2.example.yaml index 6986e9acb..0fe11df4e 100644 --- a/config/config2.example.yaml +++ b/config/config2.example.yaml @@ -61,7 +61,7 @@ iflytek_api_secret: "YOUR_API_SECRET" metagpt_tti_url: "YOUR_MODEL_URL" models: -# "YOUR_MODEL_NAME_1": # model: "gpt-4-turbo" # or gpt-3.5-turbo +# "YOUR_MODEL_NAME_1 or YOUR_API_TYPE_1": # model: "gpt-4-turbo" # or gpt-3.5-turbo # api_type: "openai" # or azure / ollama / groq etc. # base_url: "YOUR_BASE_URL" # api_key: "YOUR_API_KEY" @@ -69,7 +69,7 @@ models: # # timeout: 600 # Optional. If set to 0, default value is 300. # # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ # pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's -# "YOUR_API_TYPE": # api_type: "openai" # or azure / ollama / groq etc. +# "YOUR_MODEL_NAME_2 or YOUR_API_TYPE_2": # api_type: "openai" # or azure / ollama / groq etc. # api_type: "openai" # or azure / ollama / groq etc. # base_url: "YOUR_BASE_URL" # api_key: "YOUR_API_KEY" From a3ba8aa661b7817b7275b87bcdfb1542512ae3e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 16 Jul 2024 17:24:36 +0800 Subject: [PATCH 4/6] fixbug: #1329 --- metagpt/utils/redis.py | 2 +- requirements.txt | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index 7a640563a..9f5ef8a92 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -10,7 +10,7 @@ from __future__ import annotations import traceback from datetime import timedelta -import aioredis # https://aioredis.readthedocs.io/en/latest/getting-started/ +import redis.asyncio as aioredis from metagpt.configs.redis_config import RedisConfig from metagpt.logs import logger diff --git a/requirements.txt b/requirements.txt index dc8a86ae2..4d8d7f32e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,9 @@ wrapt==1.15.0 #aiohttp_jinja2 # azure-cognitiveservices-speech~=1.31.0 # Used by metagpt/tools/azure_tts.py #aioboto3~=12.4.0 # Used by metagpt/utils/s3.py -aioredis~=2.0.1 # Used by metagpt/utils/redis.py +redis~=5.0.0 # Used by metagpt/utils/redis.py +curl-cffi~=0.7.0 +httplib2~=0.22.0 websocket-client~=1.8.0 aiofiles==23.2.1 gitpython==3.1.40 From bc63e0aa896cb09b9b4fcdabfc80e823657bf171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 17 Jul 2024 10:11:37 +0800 Subject: [PATCH 5/6] feat: delete useless file --- tests/data/requirements/full.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/data/requirements/full.txt diff --git a/tests/data/requirements/full.txt b/tests/data/requirements/full.txt deleted file mode 100644 index e69de29bb..000000000 From 9a96d443b6c8340d53af3db5154181b6184b71ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 17 Jul 2024 12:30:13 +0800 Subject: [PATCH 6/6] fixbug: llm client error --- metagpt/actions/action.py | 5 ++++- tests/data/config/config2.yaml | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index a1ab5c2bc..20c052aa9 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from metagpt.actions.action_node import ActionNode from metagpt.configs.models_config import ModelsConfig from metagpt.context_mixin import ContextMixin +from metagpt.provider.llm_provider_registry import create_llm_instance from metagpt.schema import ( CodePlanAndChangeContext, CodeSummarizeContext, @@ -45,7 +46,9 @@ class Action(SerializationMixin, ContextMixin, BaseModel): def _update_private_llm(cls, data: Any) -> Any: config = ModelsConfig.default().get(data.llm_name_or_type) if config: - data.llm.config = config + llm = create_llm_instance(config) + llm.cost_manager = data.llm.cost_manager + data.llm = llm return data @property diff --git a/tests/data/config/config2.yaml b/tests/data/config/config2.yaml index e0cefd8ff..8c9fc0703 100644 --- a/tests/data/config/config2.yaml +++ b/tests/data/config/config2.yaml @@ -3,7 +3,7 @@ llm: base_url: "YOUR_gpt-3.5-turbo_BASE_URL" api_key: "YOUR_gpt-3.5-turbo_API_KEY" model: "gpt-3.5-turbo" # or gpt-3.5-turbo - proxy: "YOUR_gpt-3.5-turbo_PROXY" # for LLM API requests + # proxy: "YOUR_gpt-3.5-turbo_PROXY" # for LLM API requests # timeout: 600 # Optional. If set to 0, default value is 300. # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's @@ -13,7 +13,7 @@ models: api_type: "openai" # or azure / ollama / groq etc. base_url: "YOUR_MODEL_1_BASE_URL" api_key: "YOUR_MODEL_1_API_KEY" - proxy: "YOUR_MODEL_1_PROXY" # for LLM API requests + # proxy: "YOUR_MODEL_1_PROXY" # for LLM API requests # timeout: 600 # Optional. If set to 0, default value is 300. # Details: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ pricing_plan: "" # Optional. Use for Azure LLM when its model name is not the same as OpenAI's