diff --git a/README.md b/README.md index 864d56c53..a380ce1a8 100644 --- a/README.md +++ b/README.md @@ -81,29 +81,68 @@ # Step 3: Clone the repository to your local machine, and install it. - if `python setup.py install` fails with error `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`, try instead running `python setup.py install --user` -- To convert Mermaid charts to SVG, PNG, and PDF formats. In addition to the Node.js version of Mermaid-CLI, you now have the option to use Python version Playwright for this task. +- To convert Mermaid charts to SVG, PNG, and PDF formats. In addition to the Node.js version of Mermaid-CLI, you now have the option to use Python version Playwright, pyppeteer or mermaid.ink for this task. -- **Install Playwright** + - Playwright + - **Install Playwright** -```bash -pip install playwright -``` + ```bash + pip install playwright + ``` -- **Install the Required Browsers** + - **Install the Required Browsers** -to support PDF conversion, had better install Chrominum. + to support PDF conversion, had better install Chrominum. -```bash -playwright install --with-deps chromium -``` + ```bash + playwright install --with-deps chromium + ``` -- **modify `config.yaml`** + - **modify `config.yaml`** -uncomment MERMAID_ENGINE from config.yaml and change it to `playwright` + uncomment MERMAID_ENGINE from config.yaml and change it to `playwright` -```yaml -MERMAID_ENGINE: playwright -``` + ```yaml + MERMAID_ENGINE: playwright + ``` + + - pyppeteer + - **Install pyppeteer** + + ```bash + pip install pyppeteer + ``` + + - **Install the Required Browsers** + + ```bash + pyppeteer-install + ``` + + pyppeteer alow you use already installed browsers, if you do not want to run the above command, please set the following envirment + + ```bash + export PUPPETEER_EXECUTABLE_PATH = /path/to/your/chromium or edge or chrome + ``` + + - **modify `config.yaml`** + + uncomment MERMAID_ENGINE from config.yaml and change it to `pyppeteer` + + ```yaml + MERMAID_ENGINE: pyppeteer + ``` + + - mermaid.ink + - **modify `config.yaml`** + + uncomment MERMAID_ENGINE from config.yaml and change it to `ink` + + ```yaml + MERMAID_ENGINE: ink + ``` + + Note: this method does not support pdf export. ### Installation by Docker diff --git a/config/config.yaml b/config/config.yaml index 40d37451a..179985a6f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -78,5 +78,5 @@ MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### choose the engine for mermaid conversion, -# default is nodejs, you can change it to playwright +# default is nodejs, you can change it to playwright,pyppeteer or ink # MERMAID_ENGINE: nodejs \ No newline at end of file diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index f395b43b2..713f49601 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -12,6 +12,7 @@ from metagpt.config import CONFIG from metagpt.const import PROJECT_ROOT from metagpt.logs import logger from metagpt.utils.common import check_cmd_exists +import os @@ -31,34 +32,58 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height if check_cmd_exists("mmdc") != 0: logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") return -1 + engine = CONFIG.mermaid_engine.lower() - for suffix in ["pdf", "svg", "png"]: - output_file = f"{output_file_without_suffix}.{suffix}" - # Call the `mmdc` command to convert the Mermaid code to a PNG - logger.info(f"Generating {output_file}..") + if engine == "nodejs": + for suffix in ["pdf", "svg", "png"]: + output_file = f"{output_file_without_suffix}.{suffix}" + # Call the `mmdc` command to convert the Mermaid code to a PNG + logger.info(f"Generating {output_file}..") - if CONFIG.puppeteer_config: - subprocess.run( - [ - CONFIG.mmdc, - "-p", - CONFIG.puppeteer_config, - "-i", - str(tmp), - "-o", - output_file, - "-w", - str(width), - "-H", - str(height), - ] - ) - else: - subprocess.run([CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) + if CONFIG.puppeteer_config: + subprocess.run( + [ + CONFIG.mmdc, + "-p", + CONFIG.puppeteer_config, + "-i", + str(tmp), + "-o", + output_file, + "-w", + str(width), + "-H", + str(height), + ] + ) + else: + subprocess.run([CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) + else: + if engine not in ['playwright', 'puppeteer', 'ink']: + logger.warning(f"Unsupported mermaid engine: {engine}") + return -1 + __dirname = os.path.dirname(os.path.abspath(__file__)) + module_path = os.path.join(__dirname, f'mmdc_{engine}.py') + import sys + # 构建命令行参数 + command = [ + sys.executable, + module_path, + "-i",mermaid_code, + "-o",output_file_without_suffix + ] + + # 执行命令 + try: + result = subprocess.run(command, text=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + logger.info(result.stdout) + if result.stderr: + logger.error(result.stderr) + except subprocess.CalledProcessError as e: + logger.error(f"Command execution failed with return code {e.returncode}") + logger.error(e.output) return 0 -if CONFIG.mermaid_engine.lower() == "playwright": - from metagpt.utils.mermaid_playwright import mermaid_to_file MMC1 = """classDiagram class Main { diff --git a/metagpt/utils/mmdc_ink.py b/metagpt/utils/mmdc_ink.py new file mode 100644 index 000000000..ce50b11cd --- /dev/null +++ b/metagpt/utils/mmdc_ink.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/7/4 10:53 +@Author : alexanderwu, imjohndoe +@File : mermaid.py +""" + +import requests +import base64 +import os + +import click +@click.command() +@click.version_option() +@click.option("-i","--mermaid_code", type=str, help="mermaid code") +@click.option("-o","--output_file_without_suffix", type=str, help="output filename without suffix") +def mermaid_to_file(mermaid_code, output_file_without_suffix): + """suffix: png/svg + :param mermaid_code: mermaid code + :param output_file_without_suffix: output filename without suffix + :return: 0 if succeed, -1 if failed + """ + print('Starting mermaid_to_file command of mermaid.ink...') + + encoded_string = base64.b64encode(mermaid_code.encode()).decode() + + dir_name = os.path.dirname(output_file_without_suffix) + if dir_name and not os.path.exists(dir_name): + os.makedirs(dir_name) + with open(f"{output_file_without_suffix}.mmd", "w", encoding="utf-8") as f: + f.write(mermaid_code) + + for suffix in ["svg", "png"]: + output_file = f"{output_file_without_suffix}.{suffix}" + path_type = "svg" if suffix == "svg" else "img" + url = f"https://mermaid.ink/{path_type}/{encoded_string}" + response = requests.get(url) + + if response.status_code == 200: + with open(output_file, 'wb') as f: + f.write(response.content) + print(f"Generating {output_file}..") + else: + print(f"Failed to retrieve {suffix}") + return -1 + + return 0 + +if __name__ == "__main__": + mermaid_to_file() \ No newline at end of file diff --git a/metagpt/utils/mermaid_playwright.py b/metagpt/utils/mmdc_playwright.py similarity index 63% rename from metagpt/utils/mermaid_playwright.py rename to metagpt/utils/mmdc_playwright.py index aa04e70eb..d5d6b898e 100644 --- a/metagpt/utils/mermaid_playwright.py +++ b/metagpt/utils/mmdc_playwright.py @@ -3,22 +3,24 @@ """ @Time : 2023/9/4 16:12 @Author : Steven Lee -@File : mermaid_playwright.py +@File : mmdc_playwright.py """ import os import asyncio -from metagpt.config import CONFIG -from metagpt.const import PROJECT_ROOT -from metagpt.logs import logger - +import click from urllib.parse import urljoin + from playwright.async_api import async_playwright -import nest_asyncio -__dirname = os.path.dirname(os.path.abspath(__file__)) +@click.command() +@click.version_option() +@click.option("-i","--mermaid_code", type=str, help="mermaid code") +@click.option("-o","--output_file_without_suffix", type=str, help="output filename without suffix") +@click.option("--width",type=int,help="width",default=2048) +@click.option("--height",type=int,help="height",default=2048) +def mermaid_to_file(mermaid_code, output_file_without_suffix, width, height): - -def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048, output_formats=['png', 'svg', 'pdf']) -> int: + """ Converts the given Mermaid code to various output formats and saves them to files. @@ -27,18 +29,18 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height output_file_without_suffix (str): The output file name without the file extension. width (int, optional): The width of the output image in pixels. Defaults to 2048. height (int, optional): The height of the output image in pixels. Defaults to 2048. - output_formats (list[str], optional): The list of output formats to generate. Defaults to ['png', 'svg', 'pdf']. Returns: int: Returns 1 if the conversion and saving were successful, -1 otherwise. """ - async def mermaid_to_file0(mermaid_code, output_file_without_suffix, width=2048, height=2048, output_formats=['png', 'svg', 'pdf'])-> int: + __dirname = os.path.dirname(os.path.abspath(__file__)) + + async def mermaid_to_file0(mermaid_code, output_file_without_suffix, width=2048, height=2048, suffixes=['png', 'svg', 'pdf'])-> int: async with async_playwright() as p: browser = await p.chromium.launch() device_scale_factor = 1.0 - context = await browser.new_context( viewport={'width': width, 'height': height}, device_scale_factor=device_scale_factor, @@ -79,37 +81,19 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height svgElement.appendChild(style); } - let title = null; - let desc = null; - - if (svgElement.firstChild instanceof SVGTitleElement) { - title = svgElement.firstChild.textContent; - } - - for (const svgNode of svgElement.children) { - if (svgNode instanceof SVGDescElement) { - desc = svgNode.textContent; - break; - } - } - - return { - title, - desc - }; }''', [mermaid_code, mermaid_config, my_css, background_color]) - if 'svg' in output_formats : + if 'svg' in suffixes : svg_xml = await page.evaluate('''() => { const svg = document.querySelector('svg'); const xmlSerializer = new XMLSerializer(); return xmlSerializer.serializeToString(svg); }''') - # result[f'{output_file_without_suffix}.svg'] = svg_xml + print(f"Generating {output_file_without_suffix}.svg..") with open(f'{output_file_without_suffix}.svg', 'wb') as f: f.write(svg_xml.encode('utf-8')) - if 'png' in output_formats: + if 'png' in suffixes: clip = await page.evaluate('''() => { const svg = document.querySelector('svg'); const rect = svg.getBoundingClientRect(); @@ -122,78 +106,31 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height }''') await page.set_viewport_size({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height']}) screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device') + print(f"Generating {output_file_without_suffix}.png..") with open(f'{output_file_without_suffix}.png', 'wb') as f: f.write(screenshot) - if 'pdf' in output_formats: + if 'pdf' in suffixes: pdf_data = await page.pdf(scale=device_scale_factor) + print(f"Generating {output_file_without_suffix}.pdf..") with open(f'{output_file_without_suffix}.pdf', 'wb') as f: f.write(pdf_data) return 1 except Exception as e: - logger.error(e) + print(e) return -1 finally: await browser.close() + dir_name = os.path.dirname(output_file_without_suffix) + if dir_name and not os.path.exists(dir_name): + os.makedirs(dir_name) with open(f"{output_file_without_suffix}.mmd", "w", encoding="utf-8") as f: f.write(mermaid_code) - nest_asyncio.apply() + suffixes = ['png', 'svg', 'pdf'] loop = asyncio.new_event_loop() - result = loop.run_until_complete(mermaid_to_file0(mermaid_code, output_file_without_suffix, width, height, output_formats)) + result = loop.run_until_complete(mermaid_to_file0(mermaid_code, output_file_without_suffix, width, height, suffixes)) loop.close() return result -MMC1 = """classDiagram - class Main { - -SearchEngine search_engine - +main() str - } - class SearchEngine { - -Index index - -Ranking ranking - -Summary summary - +search(query: str) str - } - class Index { - -KnowledgeBase knowledge_base - +create_index(data: dict) - +query_index(query: str) list - } - class Ranking { - +rank_results(results: list) list - } - class Summary { - +summarize_results(results: list) str - } - class KnowledgeBase { - +update(data: dict) - +fetch_data(query: str) dict - } - Main --> SearchEngine - SearchEngine --> Index - SearchEngine --> Ranking - SearchEngine --> Summary - Index --> KnowledgeBase""" - -MMC2 = """sequenceDiagram - participant M as Main - participant SE as SearchEngine - participant I as Index - participant R as Ranking - participant S as Summary - participant KB as KnowledgeBase - M->>SE: search(query) - SE->>I: query_index(query) - I->>KB: fetch_data(query) - KB-->>I: return data - I-->>SE: return results - SE->>R: rank_results(results) - R-->>SE: return ranked_results - SE->>S: summarize_results(ranked_results) - S-->>SE: return summary - SE-->>M: return summary""" - - if __name__ == "__main__": - # logger.info(print_members(print_members)) - mermaid_to_file(MMC1, PROJECT_ROOT / "MMC1") - mermaid_to_file(MMC2, PROJECT_ROOT / "MMC2") + mermaid_to_file() + diff --git a/metagpt/utils/mmdc_pyppeteer.py b/metagpt/utils/mmdc_pyppeteer.py new file mode 100644 index 000000000..e6986bc76 --- /dev/null +++ b/metagpt/utils/mmdc_pyppeteer.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/4 16:12 +@Author : Steven Lee +@File : mmdc_pyppeteer.py +""" +import asyncio +import click +import os +from urllib.parse import urljoin +import sys +from pyppeteer import launch + +@click.command() +@click.version_option() +@click.option("-i","--mermaid_code", type=str, help="mermaid code") +@click.option("-o","--output_file_without_suffix", type=str, help="output filename without suffix") +@click.option("--width",type=int,help="width",default=2048) +@click.option("--height",type=int,help="height",default=2048) +def mermaid_to_file(mermaid_code, output_file_without_suffix, width, height): + """ + Converts the given Mermaid code to various output formats and saves them to files. + + Args: + mermaid_code (str): The Mermaid code to convert. + output_file_without_suffix (str): The output file name without the file extension. + width (int, optional): The width of the output image in pixels. Defaults to 2048. + height (int, optional): The height of the output image in pixels. Defaults to 2048. + + Returns: + int: Returns 1 if the conversion and saving were successful, -1 otherwise. + """ + + async def mermaid_to_file0(mermaid_code, output_file_without_suffix, width=2048, height=2048, suffixes=['png', 'svg', 'pdf'])-> int: + __dirname = os.path.dirname(os.path.abspath(__file__)) + browser = await launch(headless=True, + executablePath=os.getenv('PUPPETEER_EXECUTABLE_PATH',"/opt/homebrew/bin/chromium"), + args=['--disable-extensions',"--no-sandbox"] + ) + page = await browser.newPage() + device_scale_factor = 1.0 + + async def console_message(msg): + print(msg.text) + page.on('console', console_message) + + try: + await page.setViewport(viewport={'width': width, 'height': height, 'deviceScaleFactor': device_scale_factor}) + + mermaid_html_path = os.path.abspath( + os.path.join(__dirname, 'index.html')) + mermaid_html_url = urljoin('file:', mermaid_html_path) + await page.goto(mermaid_html_url) + + await page.querySelector("div#container") + mermaid_config = {} + background_color = "#ffffff" + my_css = "" + await page.evaluate(f'document.body.style.background = "{background_color}";') + + metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => { + const { mermaid, zenuml } = globalThis; + await mermaid.registerExternalDiagrams([zenuml]); + mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); + const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); + document.getElementById('container').innerHTML = svg; + const svgElement = document.querySelector('svg'); + svgElement.style.backgroundColor = backgroundColor; + + if (myCSS) { + const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); + style.appendChild(document.createTextNode(myCSS)); + svgElement.appendChild(style); + } + }''', [mermaid_code, mermaid_config, my_css, background_color]) + + if 'svg' in suffixes : + svg_xml = await page.evaluate('''() => { + const svg = document.querySelector('svg'); + const xmlSerializer = new XMLSerializer(); + return xmlSerializer.serializeToString(svg); + }''') + print(f"Generating {output_file_without_suffix}.svg..") + with open(f'{output_file_without_suffix}.svg', 'wb') as f: + f.write(svg_xml.encode('utf-8')) + + if 'png' in suffixes: + clip = await page.evaluate('''() => { + const svg = document.querySelector('svg'); + const rect = svg.getBoundingClientRect(); + return { + x: Math.floor(rect.left), + y: Math.floor(rect.top), + width: Math.ceil(rect.width), + height: Math.ceil(rect.height) + }; + }''') + await page.setViewport({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height'], 'deviceScaleFactor': device_scale_factor}) + screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device') + print(f"Generating {output_file_without_suffix}.png..") + with open(f'{output_file_without_suffix}.png', 'wb') as f: + f.write(screenshot) + if 'pdf' in suffixes: + pdf_data = await page.pdf(scale=device_scale_factor) + print(f"Generating {output_file_without_suffix}.pdf..") + with open(f'{output_file_without_suffix}.pdf', 'wb') as f: + f.write(pdf_data) + return 1 + except Exception as e: + print(e) + return -1 + finally: + await browser.close() + + + suffixes = ['png', 'svg', 'pdf'] + dir_name = os.path.dirname(output_file_without_suffix) + if dir_name and not os.path.exists(dir_name): + os.makedirs(dir_name) + with open(f"{output_file_without_suffix}.mmd", "w", encoding="utf-8") as f: + f.write(mermaid_code) + loop = asyncio.new_event_loop() + result = loop.run_until_complete(mermaid_to_file0(mermaid_code, output_file_without_suffix, width, height,suffixes)) + loop.close() + return result + +if __name__ == "__main__": + mermaid_to_file()