apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-06-05 14:55:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: comments

Browse source
This commit is contained in:
莘权 马 2024-03-05 11:05:27 +08:00
parent e22a28215d
commit 52ac7e847b
7 changed files with 58 additions and 365 deletions
Download patch file Download diff file Expand all files Collapse all files

73
metagpt/utils/common.py
View file

@ -23,10 +23,9 @@ import platform
import re
import sys
import traceback
import typing
from io import BytesIO
from pathlib import Path
from typing import Any, Callable, List, Tuple, Union
from typing import Any, Callable, List, Literal, Tuple, Union
from urllib.parse import quote, unquote
import aiofiles
@ -434,63 +433,40 @@ def is_send_to(message: "Message", addresses: set):
def any_to_name(val):
"""
Convert a value to its name by extracting the last part of the dotted path.
:param val: The value to convert.
:return: The name of the value.
"""
return any_to_str(val).split(".")[-1]
def concat_namespace(*args) -> str:
def concat_namespace(*args, delimiter: str = ":") -> str:
"""Concatenate fields to create a unique namespace prefix.
Args:
*args: Variable number of arguments representing the fields to be concatenated.
Returns:
str: A string containing the concatenated fields separated by colons.
Example:
>>> concat_namespace('prefix', 'field1', 'field2')
>>> concat_namespace('prefix', 'field1', 'field2', delimiter=":")
'prefix:field1:field2'
"""
return ":".join(str(value) for value in args)
return delimiter.join(str(value) for value in args)
def split_namespace(ns_class_name: str, maxsplit=1) -> List[str]:
def split_namespace(ns_class_name: str, delimiter: str = ":", maxsplit: int = 1) -> List[str]:
"""Split a namespace-prefixed name into its namespace-prefix and name parts.
Args:
ns_class_name (str): The namespace-prefixed name to be split.
maxsplit (int, optional): The maximum number of splits to perform. Defaults to 1.
Returns:
List[str]: A list containing the namespace-prefix part and the name part.
Example:
>>> split_namespace('prefix:classname')
['prefix', 'classname']
>>> split_namespace('prefix:module:class', maxsplit=2)
>>> split_namespace('prefix:module:class', delimiter=":", maxsplit=2)
['prefix', 'module', 'class']
"""
return ns_class_name.split(":", maxsplit=maxsplit)
return ns_class_name.split(delimiter, maxsplit=maxsplit)
def auto_namespace(name: str) -> str:
def auto_namespace(name: str, delimiter: str = ":") -> str:
"""Automatically handle namespace-prefixed names.
If the input name is empty, returns a default namespace prefix and name.
If the input name is not namespace-prefixed, adds a default namespace prefix.
Otherwise, returns the input name unchanged.
Args:
name (str): The input name to be processed.
Returns:
str: The processed name.
Example:
>>> auto_namespace('classname')
'?:classname'
@ -505,24 +481,16 @@ def auto_namespace(name: str) -> str:
'?:custom'
"""
if not name:
return "?:?"
v = split_namespace(name)
return f"?{delimiter}?"
v = split_namespace(name, delimiter=delimiter)
if len(v) < 2:
return f"?:{name}"
return f"?{delimiter}{name}"
return name
def add_affix(text, affix="brace"):
def add_affix(text: str, affix: Literal["brace", "url", "none"] = "brace"):
"""Add affix to encapsulate data.
Args:
text (str): The input text to be encapsulated.
affix (str, optional): The type of affix to use. Defaults to "brace".
Supported affix types: "brace" for curly braces, "url" for URL encoding within curly braces.
Returns:
str: The text encapsulated with the specified affix.
Example:
>>> add_affix("data", affix="brace")
'{data}'
@ -530,7 +498,7 @@ def add_affix(text, affix="brace"):
>>> add_affix("example.com", affix="url")
'%7Bexample.com%7D'
>>> add_affix("text", affix="unknown")
>>> add_affix("text", affix="none")
'text'
"""
mappings = {
@ -541,7 +509,7 @@ def add_affix(text, affix="brace"):
return encoder(text)
def remove_affix(text, affix="brace"):
def remove_affix(text, affix: Literal["brace", "url", "none"] = "brace"):
"""Remove affix to extract encapsulated data.
Args:
@ -559,7 +527,7 @@ def remove_affix(text, affix="brace"):
>>> remove_affix('%7Bexample.com%7D', affix="url")
'example.com'
>>> remove_affix('text', affix="unknown")
>>> remove_affix('text', affix="none")
'text'
"""
mappings = {"brace": lambda x: x[1:-1], "url": lambda x: unquote(x)[1:-1]}
@ -567,7 +535,7 @@ def remove_affix(text, affix="brace"):
return decoder(text)
def general_after_log(i: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]:
def general_after_log(i: "loguru.Logger", sec_format: str = "%0.3f") -> Callable[["RetryCallState"], None]:
"""
Generates a logging function to be used after a call is retried.
@ -760,15 +728,6 @@ def parse_json_code_block(markdown_text: str) -> List[str]:
def remove_white_spaces(v: str) -> str:
"""
Removes white spaces from the provided string, excluding spaces within quotes.
Args:
v (str): The input string containing white spaces.
Returns:
str: The input string with white spaces removed, excluding spaces within quotes.
"""
return re.sub(r"(?<!['\"])\s|(?<=['\"])\s", "", v)

161
metagpt/utils/di_graph_repository.py
View file

@ -21,31 +21,7 @@ from metagpt.utils.graph_repository import SPO, GraphRepository
class DiGraphRepository(GraphRepository):
"""Graph repository based on DiGraph.
This class represents a graph repository that utilizes a directed graph (DiGraph) to manage relationships
between entities. It inherits from the GraphRepository class, providing a common interface for graph repositories.
Attributes:
_repo (DiGraph): The underlying directed graph representing the repository.
Methods:
insert: Insert a new triple into the graph repository.
select: Retrieve triples from the graph repository based on specified criteria.
delete: Delete triples from the graph repository based on specified criteria.
save: Save any changes made to the graph repository.
name: Get the name of the graph repository.
Example:
di_graph_repo = DiGraphRepository(name="MyDiGraphRepo")
di_graph_repo.insert(subject="Node1", predicate="connects_to", object_="Node2")
# Represents a directed relationship: Node1 connects_to Node2
Note:
This class extends the GraphRepository class and is specifically designed for managing directed relationships
using a DiGraph.
"""
"""Graph repository based on DiGraph."""
def __init__(self, name: str, **kwargs):
super().__init__(name=name, **kwargs)
@ -54,35 +30,20 @@ class DiGraphRepository(GraphRepository):
async def insert(self, subject: str, predicate: str, object_: str):
"""Insert a new triple into the directed graph repository.
This method adds a new triple to the underlying directed graph. The triple consists of a subject, a predicate
describing the relationship, and an object.
Args:
subject (str): The subject of the triple.
predicate (str): The predicate describing the relationship.
object_ (str): The object of the triple.
Returns:
None
Raises:
SomeException: Describe any exceptions that might be raised during the insertion process.
Example:
await my_di_graph_repo.insert(subject="Node1", predicate="connects_to", object_="Node2")
# Adds a directed relationship: Node1 connects_to Node2
Note:
Implementations should handle the insertion of triples into the directed graph.
"""
self._repo.add_edge(subject, object_, predicate=predicate)
async def select(self, subject: str = None, predicate: str = None, object_: str = None) -> List[SPO]:
"""Retrieve triples from the directed graph repository based on specified criteria.
This method queries the directed graph repository and retrieves triples that match the specified criteria.
Args:
subject (str, optional): The subject of the triple to filter by.
predicate (str, optional): The predicate describing the relationship to filter by.
@ -91,16 +52,9 @@ class DiGraphRepository(GraphRepository):
Returns:
List[SPO]: A list of SPO objects representing the selected triples.
Raises:
SomeException: Describe any exceptions that might be raised during the selection process.
Example:
selected_triples = await my_di_graph_repo.select(subject="Node1", predicate="connects_to")
# Retrieves directed relationships where Node1 is the subject and the predicate is 'connects_to'.
Note:
Implementations should handle the selection of triples from the directed graph.
"""
result = []
for s, o, p in self._repo.edges(data="predicate"):
@ -116,8 +70,6 @@ class DiGraphRepository(GraphRepository):
async def delete(self, subject: str = None, predicate: str = None, object_: str = None) -> int:
"""Delete triples from the directed graph repository based on specified criteria.
This method removes triples from the directed graph repository that match the specified criteria.
Args:
subject (str, optional): The subject of the triple to filter by.
predicate (str, optional): The predicate describing the relationship to filter by.
@ -126,16 +78,9 @@ class DiGraphRepository(GraphRepository):
Returns:
int: The number of triples deleted from the repository.
Raises:
SomeException: Describe any exceptions that might be raised during the deletion process.
Example:
deleted_count = await my_di_graph_repo.delete(subject="Node1", predicate="connects_to")
# Deletes directed relationships where Node1 is the subject and the predicate is 'connects_to'.
Note:
Implementations should handle the deletion of triples from the directed graph.
"""
rows = await self.select(subject=subject, predicate=predicate, object_=object_)
if not rows:
@ -145,22 +90,7 @@ class DiGraphRepository(GraphRepository):
return len(rows)
def json(self) -> str:
"""Convert the directed graph repository to a JSON-formatted string.
This method converts the underlying directed graph repository to a JSON-formatted string using the node-link data
format.
Returns:
str: A JSON-formatted string representing the directed graph repository.
Example:
json_data = my_di_graph_repo.json()
# Retrieves a JSON-formatted string representing the directed graph repository.
Note:
The resulting JSON string can be used for serialization or data interchange.
"""
"""Convert the directed graph repository to a JSON-formatted string."""
m = networkx.node_link_data(self._repo)
data = json.dumps(m)
return data
@ -168,23 +98,9 @@ class DiGraphRepository(GraphRepository):
async def save(self, path: str | Path = None):
"""Save the directed graph repository to a JSON file.
This method converts the underlying directed graph repository to a JSON-formatted string and saves it to a file.
The file is saved with the name of the graph repository and a ".json" extension.
Args:
path (Union[str, Path], optional): The directory path where the JSON file will be saved.
If not provided, the default path is taken from the 'root' key in the keyword arguments.
Returns:
None
Example:
await my_di_graph_repo.save(path="/path/to/save")
# Saves the directed graph repository to a JSON file at the specified path.
Note:
The saved JSON file contains the node-link data representing the directed graph.
"""
data = self.json()
path = path or self._kwargs.get("root")
@ -194,25 +110,7 @@ class DiGraphRepository(GraphRepository):
await awrite(filename=pathname.with_suffix(".json"), data=data, encoding="utf-8")
async def load(self, pathname: str | Path):
"""Load a directed graph repository from a JSON file.
This method reads a JSON file containing node-link data representing a directed graph and loads it into the
directed graph repository.
Args:
pathname (Union[str, Path]): The path to the JSON file to be loaded.
Returns:
None
Example:
await my_di_graph_repo.load(pathname="/path/to/load/my_graph.json")
# Loads a directed graph repository from the specified JSON file.
Note:
The JSON file should contain node-link data compatible with the format produced by the 'json' method.
"""
"""Load a directed graph repository from a JSON file."""
data = await aread(filename=pathname, encoding="utf-8")
m = json.loads(data)
self._repo = networkx.node_link_graph(m)
@ -221,22 +119,11 @@ class DiGraphRepository(GraphRepository):
async def load_from(pathname: str | Path) -> GraphRepository:
"""Create and load a directed graph repository from a JSON file.
This class method creates a new instance of a graph repository and loads it from a JSON file containing node-link
data representing a directed graph.
Args:
pathname (Union[str, Path]): The path to the JSON file to be loaded.
Returns:
GraphRepository: A new instance of the graph repository loaded from the specified JSON file.
Example:
loaded_repo = await GraphRepository.load_from(pathname="/path/to/load/my_graph.json")
# Creates and loads a directed graph repository from the specified JSON file.
Note:
The JSON file should contain node-link data compatible with the format produced by the 'json' method.
"""
pathname = Path(pathname)
name = pathname.with_suffix("").name
@ -248,52 +135,16 @@ class DiGraphRepository(GraphRepository):
@property
def root(self) -> str:
"""Return the root directory path for the graph repository files.
Returns:
str: The root directory path.
Example:
root_path = my_graph_repo.root
# Retrieves the root directory path for the graph repository files.
Note:
This property provides the directory path where graph repository files are saved or loaded.
"""
"""Return the root directory path for the graph repository files."""
return self._kwargs.get("root")
@property
def pathname(self) -> Path:
"""Return the path and filename to the graph repository file.
Returns:
Path: The path and filename to the graph repository file.
Example:
file_path = my_graph_repo.pathname
# Retrieves the path and filename to the graph repository file.
Note:
This property provides the full path, including the filename, to the graph repository file.
"""
"""Return the path and filename to the graph repository file."""
p = Path(self.root) / self.name
return p.with_suffix(".json")
@property
def repo(self):
"""Get the underlying directed graph repository.
Returns:
networkx.DiGraph: The directed graph repository.
Example:
my_di_graph = my_graph_repo.repo
# Retrieves the underlying directed graph repository.
Note:
This property provides direct access to the networkx.DiGraph instance used by the graph repository.
"""
"""Get the underlying directed graph repository."""
return self._repo

111
metagpt/utils/graph_repository.py
View file

@ -67,10 +67,6 @@ class SPO(BaseModel):
Example:
spo_record = SPO(subject="Node1", predicate="connects_to", object_="Node2")
# Represents a triple: Node1 connects_to Node2
Note:
This class is a Pydantic BaseModel, allowing easy validation and serialization of graph records.
"""
subject: str
@ -84,28 +80,6 @@ class GraphRepository(ABC):
This class defines the interface for a graph repository, providing methods for inserting, selecting,
deleting, and saving graph data. Concrete implementations of this class must provide functionality
for these operations.
Attributes:
_repo_name (str): The name of the graph repository.
_kwargs (dict): Additional keyword arguments for customization.
Methods:
insert: Insert a new triple into the graph repository.
select: Retrieve triples from the graph repository based on specified criteria.
delete: Delete triples from the graph repository based on specified criteria.
save: Save any changes made to the graph repository.
name: Get the name of the graph repository.
Example:
class MyGraphRepository(GraphRepository):
# Concrete implementation of the GraphRepository interface goes here...
my_repo = MyGraphRepository(name="MyRepo")
my_repo.insert(subject="Node1", predicate="connects_to", object_="Node2")
Note:
This class is meant to be subclassed to create specific implementations of graph repositories.
"""
def __init__(self, name: str, **kwargs):
@ -121,19 +95,9 @@ class GraphRepository(ABC):
predicate (str): The predicate describing the relationship.
object_ (str): The object of the triple.
Returns:
None
Raises:
SomeException: Describe any exceptions that might be raised during the insertion process.
Example:
await my_repository.insert(subject="Node1", predicate="connects_to", object_="Node2")
# Inserts a triple: Node1 connects_to Node2 into the graph repository.
Note:
Implementations of this method should handle the insertion of triples into the underlying graph storage.
"""
pass
@ -149,16 +113,9 @@ class GraphRepository(ABC):
Returns:
List[SPO]: A list of SPO objects representing the selected triples.
Raises:
SomeException: Describe any exceptions that might be raised during the selection process.
Example:
selected_triples = await my_repository.select(subject="Node1", predicate="connects_to")
# Retrieves triples where Node1 is the subject and the predicate is 'connects_to'.
Note:
Implementations of this method should handle the selection of triples from the underlying graph storage.
"""
pass
@ -174,16 +131,9 @@ class GraphRepository(ABC):
Returns:
int: The number of triples deleted from the repository.
Raises:
SomeException: Describe any exceptions that might be raised during the deletion process.
Example:
deleted_count = await my_repository.delete(subject="Node1", predicate="connects_to")
# Deletes triples where Node1 is the subject and the predicate is 'connects_to'.
Note:
Implementations of this method should handle the deletion of triples from the underlying graph storage.
"""
pass
@ -191,44 +141,15 @@ class GraphRepository(ABC):
async def save(self):
"""Save any changes made to the graph repository.
This method is responsible for persisting any changes made to the graph repository, such as inserts, updates, or
deletions. Implementations should ensure that the changes are properly committed and reflected in the underlying
graph storage.
Args:
None
Returns:
None
Raises:
SomeException: Describe any exceptions that might be raised during the saving process.
Example:
await my_repository.save()
# Persists any changes made to the graph repository.
Note:
Implementations of this method should handle the persistence of changes in the underlying graph storage.
"""
pass
@property
def name(self) -> str:
"""Get the name of the graph repository.
Returns:
str: The name of the graph repository.
Example:
repository_name = my_repository.name
# Retrieves the name of the graph repository.
Note:
The name serves as a unique identifier for the graph repository.
"""
"""Get the name of the graph repository."""
return self._repo_name
@staticmethod
@ -253,16 +174,9 @@ class GraphRepository(ABC):
graph_db (GraphRepository): The graph repository object to be updated.
file_info (RepoFileInfo): The RepoFileInfo object containing information to be inserted.
Returns:
None
Example:
await update_graph_db_with_file_info(my_graph_repo, my_file_info)
# Updates 'my_graph_repo' with information from 'my_file_info'.
Note:
The function is designed to handle the insertion of specific triple patterns into the graph repository.
"""
await graph_db.insert(subject=file_info.file, predicate=GraphKeyword.IS, object_=GraphKeyword.SOURCE_CODE)
file_types = {".py": "python", ".js": "javascript"}
@ -347,16 +261,10 @@ class GraphRepository(ABC):
graph_db (GraphRepository): The graph repository object to be updated.
class_views (List[DotClassInfo]): List of DotClassInfo objects containing class information to be inserted.
Returns:
None
Example:
await update_graph_db_with_class_views(my_graph_repo, [class_info1, class_info2])
# Updates 'my_graph_repo' with class information from the provided list of DotClassInfo objects.
Note:
The function is designed to handle the insertion of specific triple patterns into the graph repository.
"""
for c in class_views:
filename, _ = c.package.split(":", 1)
@ -435,16 +343,10 @@ class GraphRepository(ABC):
relationship_views (List[DotClassRelationship]): List of DotClassRelationship objects containing
class relationship information to be inserted.
Returns:
None
Example:
await update_graph_db_with_class_relationship_views(my_graph_repo, [relationship1, relationship2])
# Updates 'my_graph_repo' with class relationship information from the provided list of DotClassRelationship objects.
Note:
The function is designed to handle the insertion of specific triple patterns into the graph repository.
"""
for r in relationship_views:
await graph_db.insert(
@ -468,17 +370,6 @@ class GraphRepository(ABC):
Args:
graph_db (GraphRepository): The graph repository object to be updated.
Returns:
None
Example:
await append_namespace_to_relationship_spo_objects(my_graph_repo)
# Appends namespace-prefixed information to relationship SPO objects in 'my_graph_repo'.
Note:
The function is designed to modify existing relationship SPO objects in the graph repository.
"""
classes = await graph_db.select(predicate=GraphKeyword.IS, object_=GraphKeyword.CLASS)
mapping = defaultdict(list)

51
metagpt/utils/visual_graph_repo.py
View file

@ -73,11 +73,6 @@ class VisualGraphRepo(ABC):
graph_db: GraphRepository
def __init__(self, graph_db):
"""Initializes a VisualGraphRepo instance with a specified graph database.
Args:
graph_db (GraphRepository): The graph repository used by the VisualGraphRepo.
"""
self.graph_db = graph_db
@ -89,14 +84,7 @@ class VisualDiGraphRepo(VisualGraphRepo):
@classmethod
async def load_from(cls, filename: str | Path):
"""Load a VisualDiGraphRepo instance from a file.
Args:
filename (Union[str, Path]): The path to the file containing the graph data.
Returns:
VisualDiGraphRepo: An instance of VisualDiGraphRepo loaded from the specified file.
"""
"""Load a VisualDiGraphRepo instance from a file."""
graph_db = await DiGraphRepository.load_from(str(filename))
return cls(graph_db=graph_db)
@ -112,14 +100,7 @@ class VisualDiGraphRepo(VisualGraphRepo):
return mermaid_txt
async def _get_class_view(self, ns_class_name: str) -> _VisualClassView:
"""Returns the Markdown Mermaid class diagram code block object for the specified class.
Args:
ns_class_name (str): The namespace-prefixed class name.
Returns:
_VisualClassView: An instance of _VisualClassView representing the class diagram.
"""
"""Returns the Markdown Mermaid class diagram code block object for the specified class."""
rows = await self.graph_db.select(subject=ns_class_name)
class_view = _VisualClassView(package=ns_class_name)
for r in rows:
@ -143,12 +124,7 @@ class VisualDiGraphRepo(VisualGraphRepo):
return class_view
async def get_mermaid_sequence_views(self) -> List[(str, str)]:
"""Returns all Markdown sequence diagrams with their corresponding graph repository keys.
Returns:
List[Tuple[str, str]]: A list of tuples containing Markdown sequence diagrams and their graph repository
keys.
"""
"""Returns all Markdown sequence diagrams with their corresponding graph repository keys."""
sequence_views = []
rows = await self.graph_db.select(predicate=GraphKeyword.HAS_SEQUENCE_VIEW)
for r in rows:
@ -156,14 +132,18 @@ class VisualDiGraphRepo(VisualGraphRepo):
return sequence_views
@staticmethod
def _refine_name(name) -> str:
def _refine_name(name: str) -> str:
"""Removes impurity content from the given name.
Args:
name: The name to be refined.
Example:
>>> _refine_name("int")
""
Returns:
str: The refined name.
>>> _refine_name('"Class1"')
'Class1'
>>> _refine_name("pkg.Class1")
"Class1"
"""
name = re.sub(r'^[\'"\\\(\)]+|[\'"\\\(\)]+$', "", name)
if name in ["int", "float", "bool", "str", "list", "tuple", "set", "dict", "None"]:
@ -174,12 +154,7 @@ class VisualDiGraphRepo(VisualGraphRepo):
return name
async def get_mermaid_sequence_view_versions(self) -> List[(str, str)]:
"""Returns all versioned Markdown sequence diagrams with their corresponding graph repository keys.
Returns:
List[Tuple[str, str]]: A list of tuples containing versioned Markdown sequence diagrams and their graph
repository keys.
"""
"""Returns all versioned Markdown sequence diagrams with their corresponding graph repository keys."""
sequence_views = []
rows = await self.graph_db.select(predicate=GraphKeyword.HAS_SEQUENCE_VIEW_VER)
for r in rows: