diff --git a/config/config2.yaml b/config/config2.yaml index 2c4ca636f..8e5825b57 100644 --- a/config/config2.yaml +++ b/config/config2.yaml @@ -1,3 +1,7 @@ +# Full Example: https://github.com/geekan/MetaGPT/blob/main/config/config2.example.yaml +# Reflected Code: https://github.com/geekan/MetaGPT/blob/main/metagpt/config2.py llm: - api_key: "YOUR_API_KEY" - model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview \ No newline at end of file + api_type: "openai" # or azure / ollama / open_llm etc. Check LLMType for more options + model: "gpt-4-turbo-preview" # or gpt-3.5-turbo-1106 / gpt-4-1106-preview + base_url: "https://api.openai.com/v1" # or forward url / other llm url + api_key: "YOUR_API_KEY" \ No newline at end of file diff --git a/metagpt/actions/rebuild_class_view.py b/metagpt/actions/rebuild_class_view.py index 9b1a5b3d0..6dd5690b6 100644 --- a/metagpt/actions/rebuild_class_view.py +++ b/metagpt/actions/rebuild_class_view.py @@ -148,7 +148,7 @@ class RebuildClassView(Action): logger.debug(content) return content - async def _create_mermaid_relationship(self, ns_class_name: str) -> Tuple[str, Set]: + async def _create_mermaid_relationship(self, ns_class_name: str) -> Tuple[Optional[str], Optional[Set]]: """Generates a Mermaid class relationship diagram for a specific class using data from the `graph_db` graph repository. Args: @@ -194,6 +194,13 @@ class RebuildClassView(Action): Returns: Tuple[str, str]: A tuple containing the representation of the difference ("+", "-", "=") and the path detail of the differing part. + + Example: + >>> _diff_path(path_root=Path("/Users/x/github/MetaGPT"), package_root=Path("/Users/x/github/MetaGPT/metagpt")) + "-", "metagpt" + + >>> _diff_path(path_root=Path("/Users/x/github/MetaGPT/metagpt"), package_root=Path("/Users/x/github/MetaGPT/metagpt")) + "=", "." """ if len(str(path_root)) > len(str(package_root)): return "+", str(path_root.relative_to(package_root)) @@ -212,6 +219,13 @@ class RebuildClassView(Action): Returns: str: The aligned path. + + Example: + >>> _align_root(path="metagpt/software_company.py", direction="+", diff_path="MetaGPT") + "MetaGPT/metagpt/software_company.py" + + >>> _align_root(path="metagpt/software_company.py", direction="-", diff_path="metagpt") + "software_company.py" """ if direction == "=": return path diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 41d5a5ad1..daf77e816 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -31,7 +31,7 @@ from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA from metagpt.provider.llm_provider_registry import register_provider from metagpt.schema import Message from metagpt.utils.common import CodeParser, decode_image -from metagpt.utils.cost_manager import CostManager, TokenCostManager +from metagpt.utils.cost_manager import CostManager from metagpt.utils.exceptions import handle_exception from metagpt.utils.token_counter import ( count_message_tokens, @@ -271,7 +271,6 @@ class OpenAILLM(BaseLLM): if not self.config.calc_usage: return usage - self.model if not isinstance(self.cost_manager, TokenCostManager) else "open-llm-model" try: usage.prompt_tokens = count_message_tokens(messages, self.pricing_plan) usage.completion_tokens = count_string_tokens(rsp, self.pricing_plan) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 558d60639..7bf64f05c 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -23,10 +23,9 @@ import platform import re import sys import traceback -import typing from io import BytesIO from pathlib import Path -from typing import Any, Callable, List, Tuple, Union +from typing import Any, Callable, List, Literal, Tuple, Union from urllib.parse import quote, unquote import aiofiles @@ -434,63 +433,40 @@ def is_send_to(message: "Message", addresses: set): def any_to_name(val): """ Convert a value to its name by extracting the last part of the dotted path. - - :param val: The value to convert. - - :return: The name of the value. """ return any_to_str(val).split(".")[-1] -def concat_namespace(*args) -> str: +def concat_namespace(*args, delimiter: str = ":") -> str: """Concatenate fields to create a unique namespace prefix. - Args: - *args: Variable number of arguments representing the fields to be concatenated. - - Returns: - str: A string containing the concatenated fields separated by colons. - Example: - >>> concat_namespace('prefix', 'field1', 'field2') + >>> concat_namespace('prefix', 'field1', 'field2', delimiter=":") 'prefix:field1:field2' """ - return ":".join(str(value) for value in args) + return delimiter.join(str(value) for value in args) -def split_namespace(ns_class_name: str, maxsplit=1) -> List[str]: +def split_namespace(ns_class_name: str, delimiter: str = ":", maxsplit: int = 1) -> List[str]: """Split a namespace-prefixed name into its namespace-prefix and name parts. - Args: - ns_class_name (str): The namespace-prefixed name to be split. - maxsplit (int, optional): The maximum number of splits to perform. Defaults to 1. - - Returns: - List[str]: A list containing the namespace-prefix part and the name part. - Example: >>> split_namespace('prefix:classname') ['prefix', 'classname'] - >>> split_namespace('prefix:module:class', maxsplit=2) + >>> split_namespace('prefix:module:class', delimiter=":", maxsplit=2) ['prefix', 'module', 'class'] """ - return ns_class_name.split(":", maxsplit=maxsplit) + return ns_class_name.split(delimiter, maxsplit=maxsplit) -def auto_namespace(name: str) -> str: +def auto_namespace(name: str, delimiter: str = ":") -> str: """Automatically handle namespace-prefixed names. If the input name is empty, returns a default namespace prefix and name. If the input name is not namespace-prefixed, adds a default namespace prefix. Otherwise, returns the input name unchanged. - Args: - name (str): The input name to be processed. - - Returns: - str: The processed name. - Example: >>> auto_namespace('classname') '?:classname' @@ -505,24 +481,16 @@ def auto_namespace(name: str) -> str: '?:custom' """ if not name: - return "?:?" - v = split_namespace(name) + return f"?{delimiter}?" + v = split_namespace(name, delimiter=delimiter) if len(v) < 2: - return f"?:{name}" + return f"?{delimiter}{name}" return name -def add_affix(text, affix="brace"): +def add_affix(text: str, affix: Literal["brace", "url", "none"] = "brace"): """Add affix to encapsulate data. - Args: - text (str): The input text to be encapsulated. - affix (str, optional): The type of affix to use. Defaults to "brace". - Supported affix types: "brace" for curly braces, "url" for URL encoding within curly braces. - - Returns: - str: The text encapsulated with the specified affix. - Example: >>> add_affix("data", affix="brace") '{data}' @@ -530,7 +498,7 @@ def add_affix(text, affix="brace"): >>> add_affix("example.com", affix="url") '%7Bexample.com%7D' - >>> add_affix("text", affix="unknown") + >>> add_affix("text", affix="none") 'text' """ mappings = { @@ -541,7 +509,7 @@ def add_affix(text, affix="brace"): return encoder(text) -def remove_affix(text, affix="brace"): +def remove_affix(text, affix: Literal["brace", "url", "none"] = "brace"): """Remove affix to extract encapsulated data. Args: @@ -559,7 +527,7 @@ def remove_affix(text, affix="brace"): >>> remove_affix('%7Bexample.com%7D', affix="url") 'example.com' - >>> remove_affix('text', affix="unknown") + >>> remove_affix('text', affix="none") 'text' """ mappings = {"brace": lambda x: x[1:-1], "url": lambda x: unquote(x)[1:-1]} @@ -567,7 +535,7 @@ def remove_affix(text, affix="brace"): return decoder(text) -def general_after_log(i: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: +def general_after_log(i: "loguru.Logger", sec_format: str = "%0.3f") -> Callable[["RetryCallState"], None]: """ Generates a logging function to be used after a call is retried. @@ -760,15 +728,6 @@ def parse_json_code_block(markdown_text: str) -> List[str]: def remove_white_spaces(v: str) -> str: - """ - Removes white spaces from the provided string, excluding spaces within quotes. - - Args: - v (str): The input string containing white spaces. - - Returns: - str: The input string with white spaces removed, excluding spaces within quotes. - """ return re.sub(r"(? List[SPO]: """Retrieve triples from the directed graph repository based on specified criteria. - This method queries the directed graph repository and retrieves triples that match the specified criteria. - Args: subject (str, optional): The subject of the triple to filter by. predicate (str, optional): The predicate describing the relationship to filter by. @@ -91,16 +52,9 @@ class DiGraphRepository(GraphRepository): Returns: List[SPO]: A list of SPO objects representing the selected triples. - Raises: - SomeException: Describe any exceptions that might be raised during the selection process. - Example: selected_triples = await my_di_graph_repo.select(subject="Node1", predicate="connects_to") # Retrieves directed relationships where Node1 is the subject and the predicate is 'connects_to'. - - Note: - Implementations should handle the selection of triples from the directed graph. - """ result = [] for s, o, p in self._repo.edges(data="predicate"): @@ -116,8 +70,6 @@ class DiGraphRepository(GraphRepository): async def delete(self, subject: str = None, predicate: str = None, object_: str = None) -> int: """Delete triples from the directed graph repository based on specified criteria. - This method removes triples from the directed graph repository that match the specified criteria. - Args: subject (str, optional): The subject of the triple to filter by. predicate (str, optional): The predicate describing the relationship to filter by. @@ -126,16 +78,9 @@ class DiGraphRepository(GraphRepository): Returns: int: The number of triples deleted from the repository. - Raises: - SomeException: Describe any exceptions that might be raised during the deletion process. - Example: deleted_count = await my_di_graph_repo.delete(subject="Node1", predicate="connects_to") # Deletes directed relationships where Node1 is the subject and the predicate is 'connects_to'. - - Note: - Implementations should handle the deletion of triples from the directed graph. - """ rows = await self.select(subject=subject, predicate=predicate, object_=object_) if not rows: @@ -145,22 +90,7 @@ class DiGraphRepository(GraphRepository): return len(rows) def json(self) -> str: - """Convert the directed graph repository to a JSON-formatted string. - - This method converts the underlying directed graph repository to a JSON-formatted string using the node-link data - format. - - Returns: - str: A JSON-formatted string representing the directed graph repository. - - Example: - json_data = my_di_graph_repo.json() - # Retrieves a JSON-formatted string representing the directed graph repository. - - Note: - The resulting JSON string can be used for serialization or data interchange. - - """ + """Convert the directed graph repository to a JSON-formatted string.""" m = networkx.node_link_data(self._repo) data = json.dumps(m) return data @@ -168,23 +98,9 @@ class DiGraphRepository(GraphRepository): async def save(self, path: str | Path = None): """Save the directed graph repository to a JSON file. - This method converts the underlying directed graph repository to a JSON-formatted string and saves it to a file. - The file is saved with the name of the graph repository and a ".json" extension. - Args: path (Union[str, Path], optional): The directory path where the JSON file will be saved. If not provided, the default path is taken from the 'root' key in the keyword arguments. - - Returns: - None - - Example: - await my_di_graph_repo.save(path="/path/to/save") - # Saves the directed graph repository to a JSON file at the specified path. - - Note: - The saved JSON file contains the node-link data representing the directed graph. - """ data = self.json() path = path or self._kwargs.get("root") @@ -194,25 +110,7 @@ class DiGraphRepository(GraphRepository): await awrite(filename=pathname.with_suffix(".json"), data=data, encoding="utf-8") async def load(self, pathname: str | Path): - """Load a directed graph repository from a JSON file. - - This method reads a JSON file containing node-link data representing a directed graph and loads it into the - directed graph repository. - - Args: - pathname (Union[str, Path]): The path to the JSON file to be loaded. - - Returns: - None - - Example: - await my_di_graph_repo.load(pathname="/path/to/load/my_graph.json") - # Loads a directed graph repository from the specified JSON file. - - Note: - The JSON file should contain node-link data compatible with the format produced by the 'json' method. - - """ + """Load a directed graph repository from a JSON file.""" data = await aread(filename=pathname, encoding="utf-8") m = json.loads(data) self._repo = networkx.node_link_graph(m) @@ -221,22 +119,11 @@ class DiGraphRepository(GraphRepository): async def load_from(pathname: str | Path) -> GraphRepository: """Create and load a directed graph repository from a JSON file. - This class method creates a new instance of a graph repository and loads it from a JSON file containing node-link - data representing a directed graph. - Args: pathname (Union[str, Path]): The path to the JSON file to be loaded. Returns: GraphRepository: A new instance of the graph repository loaded from the specified JSON file. - - Example: - loaded_repo = await GraphRepository.load_from(pathname="/path/to/load/my_graph.json") - # Creates and loads a directed graph repository from the specified JSON file. - - Note: - The JSON file should contain node-link data compatible with the format produced by the 'json' method. - """ pathname = Path(pathname) name = pathname.with_suffix("").name @@ -248,52 +135,16 @@ class DiGraphRepository(GraphRepository): @property def root(self) -> str: - """Return the root directory path for the graph repository files. - - Returns: - str: The root directory path. - - Example: - root_path = my_graph_repo.root - # Retrieves the root directory path for the graph repository files. - - Note: - This property provides the directory path where graph repository files are saved or loaded. - - """ + """Return the root directory path for the graph repository files.""" return self._kwargs.get("root") @property def pathname(self) -> Path: - """Return the path and filename to the graph repository file. - - Returns: - Path: The path and filename to the graph repository file. - - Example: - file_path = my_graph_repo.pathname - # Retrieves the path and filename to the graph repository file. - - Note: - This property provides the full path, including the filename, to the graph repository file. - - """ + """Return the path and filename to the graph repository file.""" p = Path(self.root) / self.name return p.with_suffix(".json") @property def repo(self): - """Get the underlying directed graph repository. - - Returns: - networkx.DiGraph: The directed graph repository. - - Example: - my_di_graph = my_graph_repo.repo - # Retrieves the underlying directed graph repository. - - Note: - This property provides direct access to the networkx.DiGraph instance used by the graph repository. - - """ + """Get the underlying directed graph repository.""" return self._repo diff --git a/metagpt/utils/graph_repository.py b/metagpt/utils/graph_repository.py index 17343114f..eb1fc5e12 100644 --- a/metagpt/utils/graph_repository.py +++ b/metagpt/utils/graph_repository.py @@ -67,10 +67,6 @@ class SPO(BaseModel): Example: spo_record = SPO(subject="Node1", predicate="connects_to", object_="Node2") # Represents a triple: Node1 connects_to Node2 - - Note: - This class is a Pydantic BaseModel, allowing easy validation and serialization of graph records. - """ subject: str @@ -84,28 +80,6 @@ class GraphRepository(ABC): This class defines the interface for a graph repository, providing methods for inserting, selecting, deleting, and saving graph data. Concrete implementations of this class must provide functionality for these operations. - - Attributes: - _repo_name (str): The name of the graph repository. - _kwargs (dict): Additional keyword arguments for customization. - - Methods: - insert: Insert a new triple into the graph repository. - select: Retrieve triples from the graph repository based on specified criteria. - delete: Delete triples from the graph repository based on specified criteria. - save: Save any changes made to the graph repository. - name: Get the name of the graph repository. - - Example: - class MyGraphRepository(GraphRepository): - # Concrete implementation of the GraphRepository interface goes here... - - my_repo = MyGraphRepository(name="MyRepo") - my_repo.insert(subject="Node1", predicate="connects_to", object_="Node2") - - Note: - This class is meant to be subclassed to create specific implementations of graph repositories. - """ def __init__(self, name: str, **kwargs): @@ -121,19 +95,9 @@ class GraphRepository(ABC): predicate (str): The predicate describing the relationship. object_ (str): The object of the triple. - Returns: - None - - Raises: - SomeException: Describe any exceptions that might be raised during the insertion process. - Example: await my_repository.insert(subject="Node1", predicate="connects_to", object_="Node2") # Inserts a triple: Node1 connects_to Node2 into the graph repository. - - Note: - Implementations of this method should handle the insertion of triples into the underlying graph storage. - """ pass @@ -149,16 +113,9 @@ class GraphRepository(ABC): Returns: List[SPO]: A list of SPO objects representing the selected triples. - Raises: - SomeException: Describe any exceptions that might be raised during the selection process. - Example: selected_triples = await my_repository.select(subject="Node1", predicate="connects_to") # Retrieves triples where Node1 is the subject and the predicate is 'connects_to'. - - Note: - Implementations of this method should handle the selection of triples from the underlying graph storage. - """ pass @@ -174,16 +131,9 @@ class GraphRepository(ABC): Returns: int: The number of triples deleted from the repository. - Raises: - SomeException: Describe any exceptions that might be raised during the deletion process. - Example: deleted_count = await my_repository.delete(subject="Node1", predicate="connects_to") # Deletes triples where Node1 is the subject and the predicate is 'connects_to'. - - Note: - Implementations of this method should handle the deletion of triples from the underlying graph storage. - """ pass @@ -191,44 +141,15 @@ class GraphRepository(ABC): async def save(self): """Save any changes made to the graph repository. - This method is responsible for persisting any changes made to the graph repository, such as inserts, updates, or - deletions. Implementations should ensure that the changes are properly committed and reflected in the underlying - graph storage. - - Args: - None - - Returns: - None - - Raises: - SomeException: Describe any exceptions that might be raised during the saving process. - Example: await my_repository.save() # Persists any changes made to the graph repository. - - Note: - Implementations of this method should handle the persistence of changes in the underlying graph storage. - """ pass @property def name(self) -> str: - """Get the name of the graph repository. - - Returns: - str: The name of the graph repository. - - Example: - repository_name = my_repository.name - # Retrieves the name of the graph repository. - - Note: - The name serves as a unique identifier for the graph repository. - - """ + """Get the name of the graph repository.""" return self._repo_name @staticmethod @@ -253,16 +174,9 @@ class GraphRepository(ABC): graph_db (GraphRepository): The graph repository object to be updated. file_info (RepoFileInfo): The RepoFileInfo object containing information to be inserted. - Returns: - None - Example: await update_graph_db_with_file_info(my_graph_repo, my_file_info) # Updates 'my_graph_repo' with information from 'my_file_info'. - - Note: - The function is designed to handle the insertion of specific triple patterns into the graph repository. - """ await graph_db.insert(subject=file_info.file, predicate=GraphKeyword.IS, object_=GraphKeyword.SOURCE_CODE) file_types = {".py": "python", ".js": "javascript"} @@ -347,16 +261,10 @@ class GraphRepository(ABC): graph_db (GraphRepository): The graph repository object to be updated. class_views (List[DotClassInfo]): List of DotClassInfo objects containing class information to be inserted. - Returns: - None Example: await update_graph_db_with_class_views(my_graph_repo, [class_info1, class_info2]) # Updates 'my_graph_repo' with class information from the provided list of DotClassInfo objects. - - Note: - The function is designed to handle the insertion of specific triple patterns into the graph repository. - """ for c in class_views: filename, _ = c.package.split(":", 1) @@ -435,16 +343,10 @@ class GraphRepository(ABC): relationship_views (List[DotClassRelationship]): List of DotClassRelationship objects containing class relationship information to be inserted. - Returns: - None - Example: await update_graph_db_with_class_relationship_views(my_graph_repo, [relationship1, relationship2]) # Updates 'my_graph_repo' with class relationship information from the provided list of DotClassRelationship objects. - Note: - The function is designed to handle the insertion of specific triple patterns into the graph repository. - """ for r in relationship_views: await graph_db.insert( @@ -468,17 +370,6 @@ class GraphRepository(ABC): Args: graph_db (GraphRepository): The graph repository object to be updated. - - Returns: - None - - Example: - await append_namespace_to_relationship_spo_objects(my_graph_repo) - # Appends namespace-prefixed information to relationship SPO objects in 'my_graph_repo'. - - Note: - The function is designed to modify existing relationship SPO objects in the graph repository. - """ classes = await graph_db.select(predicate=GraphKeyword.IS, object_=GraphKeyword.CLASS) mapping = defaultdict(list) diff --git a/metagpt/utils/visual_graph_repo.py b/metagpt/utils/visual_graph_repo.py index 49edee39e..86b50df21 100644 --- a/metagpt/utils/visual_graph_repo.py +++ b/metagpt/utils/visual_graph_repo.py @@ -73,11 +73,6 @@ class VisualGraphRepo(ABC): graph_db: GraphRepository def __init__(self, graph_db): - """Initializes a VisualGraphRepo instance with a specified graph database. - - Args: - graph_db (GraphRepository): The graph repository used by the VisualGraphRepo. - """ self.graph_db = graph_db @@ -89,14 +84,7 @@ class VisualDiGraphRepo(VisualGraphRepo): @classmethod async def load_from(cls, filename: str | Path): - """Load a VisualDiGraphRepo instance from a file. - - Args: - filename (Union[str, Path]): The path to the file containing the graph data. - - Returns: - VisualDiGraphRepo: An instance of VisualDiGraphRepo loaded from the specified file. - """ + """Load a VisualDiGraphRepo instance from a file.""" graph_db = await DiGraphRepository.load_from(str(filename)) return cls(graph_db=graph_db) @@ -112,14 +100,7 @@ class VisualDiGraphRepo(VisualGraphRepo): return mermaid_txt async def _get_class_view(self, ns_class_name: str) -> _VisualClassView: - """Returns the Markdown Mermaid class diagram code block object for the specified class. - - Args: - ns_class_name (str): The namespace-prefixed class name. - - Returns: - _VisualClassView: An instance of _VisualClassView representing the class diagram. - """ + """Returns the Markdown Mermaid class diagram code block object for the specified class.""" rows = await self.graph_db.select(subject=ns_class_name) class_view = _VisualClassView(package=ns_class_name) for r in rows: @@ -143,12 +124,7 @@ class VisualDiGraphRepo(VisualGraphRepo): return class_view async def get_mermaid_sequence_views(self) -> List[(str, str)]: - """Returns all Markdown sequence diagrams with their corresponding graph repository keys. - - Returns: - List[Tuple[str, str]]: A list of tuples containing Markdown sequence diagrams and their graph repository - keys. - """ + """Returns all Markdown sequence diagrams with their corresponding graph repository keys.""" sequence_views = [] rows = await self.graph_db.select(predicate=GraphKeyword.HAS_SEQUENCE_VIEW) for r in rows: @@ -156,14 +132,18 @@ class VisualDiGraphRepo(VisualGraphRepo): return sequence_views @staticmethod - def _refine_name(name) -> str: + def _refine_name(name: str) -> str: """Removes impurity content from the given name. - Args: - name: The name to be refined. + Example: + >>> _refine_name("int") + "" - Returns: - str: The refined name. + >>> _refine_name('"Class1"') + 'Class1' + + >>> _refine_name("pkg.Class1") + "Class1" """ name = re.sub(r'^[\'"\\\(\)]+|[\'"\\\(\)]+$', "", name) if name in ["int", "float", "bool", "str", "list", "tuple", "set", "dict", "None"]: @@ -174,12 +154,7 @@ class VisualDiGraphRepo(VisualGraphRepo): return name async def get_mermaid_sequence_view_versions(self) -> List[(str, str)]: - """Returns all versioned Markdown sequence diagrams with their corresponding graph repository keys. - - Returns: - List[Tuple[str, str]]: A list of tuples containing versioned Markdown sequence diagrams and their graph - repository keys. - """ + """Returns all versioned Markdown sequence diagrams with their corresponding graph repository keys.""" sequence_views = [] rows = await self.graph_db.select(predicate=GraphKeyword.HAS_SEQUENCE_VIEW_VER) for r in rows: