apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-05-24 14:15:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

新修改。

Browse source
This commit is contained in:
zhouzinimg 2023-10-18 19:40:41 +08:00
parent f3c7da32a0
commit 9af9461b4f
152 changed files with 7692 additions and 792 deletions
Download patch file Download diff file Expand all files Collapse all files

1
metagpt/utils/__init__.py
View file

@ -14,7 +14,6 @@ from metagpt.utils.token_counter import (
count_string_tokens,
)
__all__ = [
"read_docx",
"Singleton",

6
metagpt/utils/common.py
View file

@ -86,8 +86,8 @@ class OutputParser:
@staticmethod
def parse_python_code(text: str) -> str:
for pattern in (
r"(.*?```python.*?\s+)?(?P<code>.*)(```.*?)",
r"(.*?```python.*?\s+)?(?P<code>.*)",
r"(.*?```python.*?\s+)?(?P<code>.*)(```.*?)",
r"(.*?```python.*?\s+)?(?P<code>.*)",
):
match = re.search(pattern, text, re.DOTALL)
if not match:
@ -180,7 +180,7 @@ class OutputParser:
if start_index != -1 and end_index != -1:
# Extract the structure part
structure_text = text[start_index : end_index + 1]
structure_text = text[start_index: end_index + 1]
try:
# Attempt to convert the text to a Python data type using ast.literal_eval

42
metagpt/utils/custom_decoder.py
View file

@ -36,11 +36,11 @@ def py_make_scanner(context):
return parse_object((string, idx + 1), strict, _scan_once, object_hook, object_pairs_hook, memo)
elif nextchar == "[":
return parse_array((string, idx + 1), _scan_once)
elif nextchar == "n" and string[idx : idx + 4] == "null":
elif nextchar == "n" and string[idx: idx + 4] == "null":
return None, idx + 4
elif nextchar == "t" and string[idx : idx + 4] == "true":
elif nextchar == "t" and string[idx: idx + 4] == "true":
return True, idx + 4
elif nextchar == "f" and string[idx : idx + 5] == "false":
elif nextchar == "f" and string[idx: idx + 5] == "false":
return False, idx + 5
m = match_number(string, idx)
@ -51,11 +51,11 @@ def py_make_scanner(context):
else:
res = parse_int(integer)
return res, m.end()
elif nextchar == "N" and string[idx : idx + 3] == "NaN":
elif nextchar == "N" and string[idx: idx + 3] == "NaN":
return parse_constant("NaN"), idx + 3
elif nextchar == "I" and string[idx : idx + 8] == "Infinity":
elif nextchar == "I" and string[idx: idx + 8] == "Infinity":
return parse_constant("Infinity"), idx + 8
elif nextchar == "-" and string[idx : idx + 9] == "-Infinity":
elif nextchar == "-" and string[idx: idx + 9] == "-Infinity":
return parse_constant("-Infinity"), idx + 9
else:
raise StopIteration(idx)
@ -89,7 +89,7 @@ WHITESPACE_STR = " \t\n\r"
def JSONObject(
s_and_end, strict, scan_once, object_hook, object_pairs_hook, memo=None, _w=WHITESPACE.match, _ws=WHITESPACE_STR
s_and_end, strict, scan_once, object_hook, object_pairs_hook, memo=None, _w=WHITESPACE.match, _ws=WHITESPACE_STR
):
"""Parse a JSON object from a string and return the parsed object.
@ -118,12 +118,12 @@ def JSONObject(
memo_get = memo.setdefault
# Use a slice to prevent IndexError from being raised, the following
# check will raise a more specific ValueError if the string is empty
nextchar = s[end : end + 1]
nextchar = s[end: end + 1]
# Normally we expect nextchar == '"'
if nextchar != '"' and nextchar != "'":
if nextchar in _ws:
end = _w(s, end).end()
nextchar = s[end : end + 1]
nextchar = s[end: end + 1]
# Trivial empty object
if nextchar == "}":
if object_pairs_hook is not None:
@ -146,9 +146,9 @@ def JSONObject(
key = memo_get(key, key)
# To skip some function call overhead we optimize the fast paths where
# the JSON key separator is ": " or just ":".
if s[end : end + 1] != ":":
if s[end: end + 1] != ":":
end = _w(s, end).end()
if s[end : end + 1] != ":":
if s[end: end + 1] != ":":
raise JSONDecodeError("Expecting ':' delimiter", s, end)
end += 1
@ -179,7 +179,7 @@ def JSONObject(
elif nextchar != ",":
raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
end = _w(s, end).end()
nextchar = s[end : end + 1]
nextchar = s[end: end + 1]
end += 1
if nextchar != '"':
raise JSONDecodeError("Expecting property name enclosed in double quotes", s, end - 1)
@ -257,7 +257,7 @@ def py_scanstring(s, end, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match, delim
else:
uni = _decode_uXXXX(s, end)
end += 5
if 0xD800 <= uni <= 0xDBFF and s[end : end + 2] == "\\u":
if 0xD800 <= uni <= 0xDBFF and s[end: end + 2] == "\\u":
uni2 = _decode_uXXXX(s, end + 1)
if 0xDC00 <= uni2 <= 0xDFFF:
uni = 0x10000 + (((uni - 0xD800) << 10) | (uni2 - 0xDC00))
@ -272,14 +272,14 @@ scanstring = py_scanstring
class CustomDecoder(json.JSONDecoder):
def __init__(
self,
*,
object_hook=None,
parse_float=None,
parse_int=None,
parse_constant=None,
strict=True,
object_pairs_hook=None
self,
*,
object_hook=None,
parse_float=None,
parse_int=None,
parse_constant=None,
strict=True,
object_pairs_hook=None
):
super().__init__(
object_hook=object_hook,

4
metagpt/utils/file.py
View file

@ -6,9 +6,10 @@
@File : file.py
@Describe : General file operations.
"""
import aiofiles
from pathlib import Path
import aiofiles
from metagpt.logs import logger
@ -72,4 +73,3 @@ class File:
except Exception as e:
logger.error(f"Error reading file: {e}")
raise e

2
metagpt/utils/highlight.py
View file

@ -1,7 +1,7 @@
# 添加代码语法高亮显示
from pygments import highlight as highlight_
from pygments.lexers import PythonLexer, SqlLexer
from pygments.formatters import TerminalFormatter, HtmlFormatter
from pygments.lexers import PythonLexer, SqlLexer
def highlight(code: str, language: str = 'python', formatter: str = 'terminal'):

1
metagpt/utils/mermaid.py
View file

@ -135,7 +135,6 @@ MMC2 = """sequenceDiagram
S-->>SE: return summary
SE-->>M: return summary"""
if __name__ == "__main__":
loop = asyncio.new_event_loop()
result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1"))

4
metagpt/utils/mmdc_ink.py
View file

@ -6,9 +6,9 @@
@File : mermaid.py
"""
import base64
import os
from aiohttp import ClientSession,ClientError
from aiohttp import ClientSession, ClientError
from metagpt.logs import logger

18
metagpt/utils/mmdc_playwright.py
View file

@ -8,10 +8,13 @@
import os
from urllib.parse import urljoin
from playwright.async_api import async_playwright
from metagpt.logs import logger
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int:
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
"""
Converts the given Mermaid code to various output formats and saves them to files.
@ -24,20 +27,21 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
Returns:
int: Returns 1 if the conversion and saving were successful, -1 otherwise.
"""
suffixes=['png', 'svg', 'pdf']
suffixes = ['png', 'svg', 'pdf']
__dirname = os.path.dirname(os.path.abspath(__file__))
async with async_playwright() as p:
browser = await p.chromium.launch()
device_scale_factor = 1.0
context = await browser.new_context(
viewport={'width': width, 'height': height},
device_scale_factor=device_scale_factor,
)
viewport={'width': width, 'height': height},
device_scale_factor=device_scale_factor,
)
page = await context.new_page()
async def console_message(msg):
logger.info(msg.text)
page.on('console', console_message)
try:
@ -72,7 +76,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
}''', [mermaid_code, mermaid_config, my_css, background_color])
if 'svg' in suffixes :
if 'svg' in suffixes:
svg_xml = await page.evaluate('''() => {
const svg = document.querySelector('svg');
const xmlSerializer = new XMLSerializer();
@ -82,7 +86,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
with open(f'{output_file_without_suffix}.svg', 'wb') as f:
f.write(svg_xml.encode('utf-8'))
if 'png' in suffixes:
if 'png' in suffixes:
clip = await page.evaluate('''() => {
const svg = document.querySelector('svg');
const rect = svg.getBoundingClientRect();

29
metagpt/utils/mmdc_pyppeteer.py
View file

@ -7,11 +7,14 @@
"""
import os
from urllib.parse import urljoin
from pyppeteer import launch
from metagpt.logs import logger
from metagpt.config import CONFIG
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int:
from pyppeteer import launch
from metagpt.config import CONFIG
from metagpt.logs import logger
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
"""
Converts the given Mermaid code to various output formats and saves them to files.
@ -24,15 +27,14 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
Returns:
int: Returns 1 if the conversion and saving were successful, -1 otherwise.
"""
suffixes = ['png', 'svg', 'pdf']
suffixes = ['png', 'svg', 'pdf']
__dirname = os.path.dirname(os.path.abspath(__file__))
if CONFIG.pyppeteer_executable_path:
browser = await launch(headless=True,
executablePath=CONFIG.pyppeteer_executable_path,
args=['--disable-extensions',"--no-sandbox"]
)
executablePath=CONFIG.pyppeteer_executable_path,
args=['--disable-extensions', "--no-sandbox"]
)
else:
logger.error("Please set the environment variable:PYPPETEER_EXECUTABLE_PATH.")
return -1
@ -41,6 +43,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
async def console_message(msg):
logger.info(msg.text)
page.on('console', console_message)
try:
@ -73,7 +76,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
}
}''', [mermaid_code, mermaid_config, my_css, background_color])
if 'svg' in suffixes :
if 'svg' in suffixes:
svg_xml = await page.evaluate('''() => {
const svg = document.querySelector('svg');
const xmlSerializer = new XMLSerializer();
@ -83,7 +86,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
with open(f'{output_file_without_suffix}.svg', 'wb') as f:
f.write(svg_xml.encode('utf-8'))
if 'png' in suffixes:
if 'png' in suffixes:
clip = await page.evaluate('''() => {
const svg = document.querySelector('svg');
const rect = svg.getBoundingClientRect();
@ -94,7 +97,8 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
height: Math.ceil(rect.height)
};
}''')
await page.setViewport({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height'], 'deviceScaleFactor': device_scale_factor})
await page.setViewport({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height'],
'deviceScaleFactor': device_scale_factor})
screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device')
logger.info(f"Generating {output_file_without_suffix}.png..")
with open(f'{output_file_without_suffix}.png', 'wb') as f:
@ -110,4 +114,3 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
return -1
finally:
await browser.close()

4
metagpt/utils/parse_html.py
View file

@ -16,7 +16,7 @@ class WebPage(BaseModel):
class Config:
underscore_attrs_are_private = True
_soup : Optional[BeautifulSoup] = None
_soup: Optional[BeautifulSoup] = None
_title: Optional[str] = None
@property
@ -24,7 +24,7 @@ class WebPage(BaseModel):
if self._soup is None:
self._soup = BeautifulSoup(self.html, "html.parser")
return self._soup
@property
def title(self):
if self._title is None:

15
metagpt/utils/pycst.py
View file

@ -37,12 +37,12 @@ def get_docstring_statement(body: DocstringNode) -> cst.SimpleStatementLine:
if not isinstance(expr, cst.Expr):
return None
val = expr.value
if not isinstance(val, (cst.SimpleString, cst.ConcatenatedString)):
return None
evaluated_value = val.evaluated_value
evaluated_value = val.evaluated_value
if isinstance(evaluated_value, bytes):
return None
@ -56,6 +56,7 @@ class DocstringCollector(cst.CSTVisitor):
stack: A list to keep track of the current path in the CST.
docstrings: A dictionary mapping paths in the CST to their corresponding docstrings.
"""
def __init__(self):
self.stack: list[str] = []
self.docstrings: dict[tuple[str, ...], cst.SimpleStatementLine] = {}
@ -96,9 +97,10 @@ class DocstringTransformer(cst.CSTTransformer):
stack: A list to keep track of the current path in the CST.
docstrings: A dictionary mapping paths in the CST to their corresponding docstrings.
"""
def __init__(
self,
docstrings: dict[tuple[str, ...], cst.SimpleStatementLine],
self,
docstrings: dict[tuple[str, ...], cst.SimpleStatementLine],
):
self.stack: list[str] = []
self.docstrings = docstrings
@ -125,7 +127,8 @@ class DocstringTransformer(cst.CSTTransformer):
key = tuple(self.stack)
self.stack.pop()
if hasattr(updated_node, "decorators") and any((i.decorator.value == "overload") for i in updated_node.decorators):
if hasattr(updated_node, "decorators") and any(
(i.decorator.value == "overload") for i in updated_node.decorators):
return updated_node
statement = self.docstrings.get(key)

1
metagpt/utils/read_document.py
View file

@ -8,6 +8,7 @@
import docx
def read_docx(file_path: str) -> list:
"""Open a docx file"""
doc = docx.Document(file_path)

1
metagpt/utils/singleton.py
View file

@ -20,4 +20,3 @@ class Singleton(abc.ABCMeta, type):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]

2
metagpt/utils/special_tokens.py
View file

@ -1,4 +1,4 @@
# token to separate different code messages in a WriteCode Message content
MSG_SEP = "#*000*#"
MSG_SEP = "#*000*#"
# token to seperate file name and the actual code text in a code message
FILENAME_CODE_SEP = "#*001*#"

19
metagpt/utils/text.py
View file

@ -3,7 +3,8 @@ from typing import Generator, Sequence
from metagpt.utils.token_counter import TOKEN_MAX, count_string_tokens
def reduce_message_length(msgs: Generator[str, None, None], model_name: str, system_text: str, reserved: int = 0,) -> str:
def reduce_message_length(msgs: Generator[str, None, None], model_name: str, system_text: str,
reserved: int = 0, ) -> str:
"""Reduce the length of concatenated message segments to fit within the maximum token size.
Args:
@ -27,11 +28,11 @@ def reduce_message_length(msgs: Generator[str, None, None], model_name: str, sys
def generate_prompt_chunk(
text: str,
prompt_template: str,
model_name: str,
system_text: str,
reserved: int = 0,
text: str,
prompt_template: str,
model_name: str,
system_text: str,
reserved: int = 0,
) -> Generator[str, None, None]:
"""Split the text into chunks of a maximum token size.
@ -49,9 +50,9 @@ def generate_prompt_chunk(
current_token = 0
current_lines = []
reserved = reserved + count_string_tokens(prompt_template+system_text, model_name)
reserved = reserved + count_string_tokens(prompt_template + system_text, model_name)
# 100 is a magic number to ensure the maximum context length is not exceeded
max_token = TOKEN_MAX.get(model_name, 2048) - reserved - 100
max_token = TOKEN_MAX.get(model_name, 2048) - reserved - 100
while paragraphs:
paragraph = paragraphs.pop(0)
@ -103,7 +104,7 @@ def decode_unicode_escape(text: str) -> str:
return text.encode("utf-8").decode("unicode_escape", "ignore")
def _split_by_count(lst: Sequence , count: int):
def _split_by_count(lst: Sequence, count: int):
avg = len(lst) // count
remainder = len(lst) % count
start = 0

1
metagpt/utils/token_counter.py
View file

@ -24,7 +24,6 @@ TOKEN_COSTS = {
"text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0},
}
TOKEN_MAX = {
"gpt-3.5-turbo": 4096,
"gpt-3.5-turbo-0301": 4096,