apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-04-28 18:36:22 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'main' into add_open_interpreter

Browse source
This commit is contained in:
Ben 2023-09-21 10:38:57 +08:00 • committed by GitHub
commit 58a0ae0a35
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
82 changed files with 2056 additions and 132 deletions
Download patch file Download diff file Expand all files Collapse all files

3
metagpt/utils/common.py
View file

@ -191,7 +191,8 @@ class CodeParser:
else:
logger.error(f"{pattern} not match following text:")
logger.error(text)
raise Exception
# raise Exception
return ""
return code
@classmethod

42
metagpt/utils/file.py Normal file
View file

@ -0,0 +1,42 @@
#!/usr/bin/env python3
# _*_ coding: utf-8 _*_
"""
@Time : 2023/9/4 15:40:40
@Author : Stitch-z
@File : file.py
@Describe : General file operations.
"""
import aiofiles
from pathlib import Path
from metagpt.logs import logger
class File:
"""A general util for file operations."""
@classmethod
async def write(cls, root_path: Path, filename: str, content: bytes) -> Path:
"""Write the file content to the local specified path.
Args:
root_path: The root path of file, such as "/data".
filename: The name of file, such as "test.txt".
content: The binary content of file.
Returns:
The full filename of file, such as "/data/test.txt".
Raises:
Exception: If an unexpected error occurs during the file writing process.
"""
try:
root_path.mkdir(parents=True, exist_ok=True)
full_path = root_path / filename
async with aiofiles.open(full_path, mode="wb") as writer:
await writer.write(content)
logger.info(f"Successfully write file: {full_path}")
return full_path
except Exception as e:
logger.error(f"Error writing file: {e}")
raise e

1
metagpt/utils/index.html Normal file
View file

File diff suppressed because one or more lines are too long

34
metagpt/utils/make_sk_kernel.py Normal file
View file

@ -0,0 +1,34 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/9/13 12:29
@Author : femto Zheng
@File : make_sk_kernel.py
"""
import semantic_kernel as sk
from semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion import (
AzureChatCompletion,
)
from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import (
OpenAIChatCompletion,
)
from metagpt.config import CONFIG
def make_sk_kernel():
kernel = sk.Kernel()
if CONFIG.openai_api_type == "azure":
kernel.add_chat_service(
"chat_completion",
AzureChatCompletion(CONFIG.deployment_name, CONFIG.openai_api_base, CONFIG.openai_api_key),
)
else:
kernel.add_chat_service(
"chat_completion",
OpenAIChatCompletion(
CONFIG.openai_api_model, CONFIG.openai_api_key, org_id=None, endpoint=CONFIG.openai_api_base
),
)
return kernel

95
metagpt/utils/mermaid.py
View file

@ -2,9 +2,10 @@
# -*- coding: utf-8 -*-
"""
@Time : 2023/7/4 10:53
@Author : alexanderwu
@Author : alexanderwu alitrack
@File : mermaid.py
"""
import asyncio
import subprocess
from pathlib import Path
@ -12,48 +13,76 @@ from metagpt.config import CONFIG
from metagpt.const import PROJECT_ROOT
from metagpt.logs import logger
from metagpt.utils.common import check_cmd_exists
import os
import sys
def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
"""suffix: png/svg/pdf
:param mermaid_code: mermaid code
:param output_file_without_suffix: output filename
:param width:
:param height:
:return: 0 if succed, -1 if failed
:return: 0 if succeed, -1 if failed
"""
# Write the Mermaid code to a temporary file
dir_name = os.path.dirname(output_file_without_suffix)
if dir_name and not os.path.exists(dir_name):
os.makedirs(dir_name)
tmp = Path(f"{output_file_without_suffix}.mmd")
tmp.write_text(mermaid_code, encoding="utf-8")
engine = CONFIG.mermaid_engine.lower()
if engine == "nodejs":
if check_cmd_exists("mmdc") != 0:
logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
return -1
for suffix in ["pdf", "svg", "png"]:
output_file = f"{output_file_without_suffix}.{suffix}"
# Call the `mmdc` command to convert the Mermaid code to a PNG
logger.info(f"Generating {output_file}..")
if check_cmd_exists("mmdc") != 0:
logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
return -1
for suffix in ["pdf", "svg", "png"]:
output_file = f"{output_file_without_suffix}.{suffix}"
# Call the `mmdc` command to convert the Mermaid code to a PNG
logger.info(f"Generating {output_file}..")
if CONFIG.puppeteer_config:
subprocess.run(
[
CONFIG.mmdc,
"-p",
CONFIG.puppeteer_config,
"-i",
str(tmp),
"-o",
output_file,
"-w",
str(width),
"-H",
str(height),
]
if CONFIG.puppeteer_config:
commands =[
CONFIG.mmdc,
"-p",
CONFIG.puppeteer_config,
"-i",
str(tmp),
"-o",
output_file,
"-w",
str(width),
"-H",
str(height),
]
else:
commands =[CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]
process = await asyncio.create_subprocess_exec(
*commands,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await process.communicate()
if stdout:
logger.info(stdout.decode())
if stderr:
logger.error(stderr.decode())
else:
if engine =='playwright':
from metagpt.utils.mmdc_playwright import mermaid_to_file
return await mermaid_to_file(mermaid_code, output_file_without_suffix, width, height)
elif engine =='pyppeteer':
from metagpt.utils.mmdc_pyppeteer import mermaid_to_file
return await mermaid_to_file(mermaid_code, output_file_without_suffix, width, height)
elif engine =='ink':
from metagpt.utils.mmdc_ink import mermaid_to_file
return await mermaid_to_file(mermaid_code, output_file_without_suffix)
else:
subprocess.run([CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)])
logger.warning(f"Unsupported mermaid engine: {engine}")
return 0
@ -108,7 +137,9 @@ MMC2 = """sequenceDiagram
SE-->>M: return summary"""
if __name__ == "__main__":
# logger.info(print_members(print_members))
mermaid_to_file(MMC1, PROJECT_ROOT / "tmp/1.png")
mermaid_to_file(MMC2, PROJECT_ROOT / "tmp/2.png")
loop = asyncio.new_event_loop()
result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1"))
result = loop.run_until_complete(mermaid_to_file(MMC2, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1"))
loop.close()

41
metagpt/utils/mmdc_ink.py Normal file
View file

@ -0,0 +1,41 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/9/4 16:12
@Author : alitrack
@File : mermaid.py
"""
import base64
import os
from aiohttp import ClientSession,ClientError
from metagpt.logs import logger
async def mermaid_to_file(mermaid_code, output_file_without_suffix):
"""suffix: png/svg
:param mermaid_code: mermaid code
:param output_file_without_suffix: output filename without suffix
:return: 0 if succeed, -1 if failed
"""
encoded_string = base64.b64encode(mermaid_code.encode()).decode()
for suffix in ["svg", "png"]:
output_file = f"{output_file_without_suffix}.{suffix}"
path_type = "svg" if suffix == "svg" else "img"
url = f"https://mermaid.ink/{path_type}/{encoded_string}"
async with ClientSession() as session:
try:
async with session.get(url) as response:
if response.status == 200:
text = await response.content.read()
with open(output_file, 'wb') as f:
f.write(text)
logger.info(f"Generating {output_file}..")
else:
logger.error(f"Failed to generate {output_file}")
return -1
except ClientError as e:
logger.error(f"network error: {e}")
return -1
return 0

111
metagpt/utils/mmdc_playwright.py Normal file
View file

@ -0,0 +1,111 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/9/4 16:12
@Author : Steven Lee
@File : mmdc_playwright.py
"""
import os
from urllib.parse import urljoin
from playwright.async_api import async_playwright
from metagpt.logs import logger
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int:
"""
Converts the given Mermaid code to various output formats and saves them to files.
Args:
mermaid_code (str): The Mermaid code to convert.
output_file_without_suffix (str): The output file name without the file extension.
width (int, optional): The width of the output image in pixels. Defaults to 2048.
height (int, optional): The height of the output image in pixels. Defaults to 2048.
Returns:
int: Returns 1 if the conversion and saving were successful, -1 otherwise.
"""
suffixes=['png', 'svg', 'pdf']
__dirname = os.path.dirname(os.path.abspath(__file__))
async with async_playwright() as p:
browser = await p.chromium.launch()
device_scale_factor = 1.0
context = await browser.new_context(
viewport={'width': width, 'height': height},
device_scale_factor=device_scale_factor,
)
page = await context.new_page()
async def console_message(msg):
logger.info(msg.text)
page.on('console', console_message)
try:
await page.set_viewport_size({'width': width, 'height': height})
mermaid_html_path = os.path.abspath(
os.path.join(__dirname, 'index.html'))
mermaid_html_url = urljoin('file:', mermaid_html_path)
await page.goto(mermaid_html_url)
await page.wait_for_load_state("networkidle")
await page.wait_for_selector("div#container", state="attached")
mermaid_config = {}
background_color = "#ffffff"
my_css = ""
await page.evaluate(f'document.body.style.background = "{background_color}";')
metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => {
const { mermaid, zenuml } = globalThis;
await mermaid.registerExternalDiagrams([zenuml]);
mermaid.initialize({ startOnLoad: false, ...mermaidConfig });
const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container'));
document.getElementById('container').innerHTML = svg;
const svgElement = document.querySelector('svg');
svgElement.style.backgroundColor = backgroundColor;
if (myCSS) {
const style = document.createElementNS('http://www.w3.org/2000/svg', 'style');
style.appendChild(document.createTextNode(myCSS));
svgElement.appendChild(style);
}
}''', [mermaid_code, mermaid_config, my_css, background_color])
if 'svg' in suffixes :
svg_xml = await page.evaluate('''() => {
const svg = document.querySelector('svg');
const xmlSerializer = new XMLSerializer();
return xmlSerializer.serializeToString(svg);
}''')
logger.info(f"Generating {output_file_without_suffix}.svg..")
with open(f'{output_file_without_suffix}.svg', 'wb') as f:
f.write(svg_xml.encode('utf-8'))
if 'png' in suffixes:
clip = await page.evaluate('''() => {
const svg = document.querySelector('svg');
const rect = svg.getBoundingClientRect();
return {
x: Math.floor(rect.left),
y: Math.floor(rect.top),
width: Math.ceil(rect.width),
height: Math.ceil(rect.height)
};
}''')
await page.set_viewport_size({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height']})
screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device')
logger.info(f"Generating {output_file_without_suffix}.png..")
with open(f'{output_file_without_suffix}.png', 'wb') as f:
f.write(screenshot)
if 'pdf' in suffixes:
pdf_data = await page.pdf(scale=device_scale_factor)
logger.info(f"Generating {output_file_without_suffix}.pdf..")
with open(f'{output_file_without_suffix}.pdf', 'wb') as f:
f.write(pdf_data)
return 0
except Exception as e:
logger.error(e)
return -1
finally:
await browser.close()

113
metagpt/utils/mmdc_pyppeteer.py Normal file
View file

@ -0,0 +1,113 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/9/4 16:12
@Author : alitrack
@File : mmdc_pyppeteer.py
"""
import os
from urllib.parse import urljoin
from pyppeteer import launch
from metagpt.logs import logger
from metagpt.config import CONFIG
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int:
"""
Converts the given Mermaid code to various output formats and saves them to files.
Args:
mermaid_code (str): The Mermaid code to convert.
output_file_without_suffix (str): The output file name without the file extension.
width (int, optional): The width of the output image in pixels. Defaults to 2048.
height (int, optional): The height of the output image in pixels. Defaults to 2048.
Returns:
int: Returns 1 if the conversion and saving were successful, -1 otherwise.
"""
suffixes = ['png', 'svg', 'pdf']
__dirname = os.path.dirname(os.path.abspath(__file__))
if CONFIG.pyppeteer_executable_path:
browser = await launch(headless=True,
executablePath=CONFIG.pyppeteer_executable_path,
args=['--disable-extensions',"--no-sandbox"]
)
else:
logger.error("Please set the environment variable:PYPPETEER_EXECUTABLE_PATH.")
return -1
page = await browser.newPage()
device_scale_factor = 1.0
async def console_message(msg):
logger.info(msg.text)
page.on('console', console_message)
try:
await page.setViewport(viewport={'width': width, 'height': height, 'deviceScaleFactor': device_scale_factor})
mermaid_html_path = os.path.abspath(
os.path.join(__dirname, 'index.html'))
mermaid_html_url = urljoin('file:', mermaid_html_path)
await page.goto(mermaid_html_url)
await page.querySelector("div#container")
mermaid_config = {}
background_color = "#ffffff"
my_css = ""
await page.evaluate(f'document.body.style.background = "{background_color}";')
metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => {
const { mermaid, zenuml } = globalThis;
await mermaid.registerExternalDiagrams([zenuml]);
mermaid.initialize({ startOnLoad: false, ...mermaidConfig });
const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container'));
document.getElementById('container').innerHTML = svg;
const svgElement = document.querySelector('svg');
svgElement.style.backgroundColor = backgroundColor;
if (myCSS) {
const style = document.createElementNS('http://www.w3.org/2000/svg', 'style');
style.appendChild(document.createTextNode(myCSS));
svgElement.appendChild(style);
}
}''', [mermaid_code, mermaid_config, my_css, background_color])
if 'svg' in suffixes :
svg_xml = await page.evaluate('''() => {
const svg = document.querySelector('svg');
const xmlSerializer = new XMLSerializer();
return xmlSerializer.serializeToString(svg);
}''')
logger.info(f"Generating {output_file_without_suffix}.svg..")
with open(f'{output_file_without_suffix}.svg', 'wb') as f:
f.write(svg_xml.encode('utf-8'))
if 'png' in suffixes:
clip = await page.evaluate('''() => {
const svg = document.querySelector('svg');
const rect = svg.getBoundingClientRect();
return {
x: Math.floor(rect.left),
y: Math.floor(rect.top),
width: Math.ceil(rect.width),
height: Math.ceil(rect.height)
};
}''')
await page.setViewport({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height'], 'deviceScaleFactor': device_scale_factor})
screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device')
logger.info(f"Generating {output_file_without_suffix}.png..")
with open(f'{output_file_without_suffix}.png', 'wb') as f:
f.write(screenshot)
if 'pdf' in suffixes:
pdf_data = await page.pdf(scale=device_scale_factor)
logger.info(f"Generating {output_file_without_suffix}.pdf..")
with open(f'{output_file_without_suffix}.pdf', 'wb') as f:
f.write(pdf_data)
return 0
except Exception as e:
logger.error(e)
return -1
finally:
await browser.close()