Merge branch 'minecraft' of github.com:geekan/MetaGPT into minecraft

metagpt/utils/common.py

@@ -11,26 +11,25 @@ import inspect
 import os
 import platform
 import re
-from typing import List, Tuple
+from typing import List, Tuple, Union
 
 from metagpt.logs import logger
 
 
 def check_cmd_exists(command) -> int:
-    """ 检查命令是否存在
+    """检查命令是否存在
     :param command: 待检查的命令
     :return: 如果命令存在，返回0，如果不存在，返回非0
     """
-    if platform.system().lower() == 'windows':
-        check_command = 'where ' + command
+    if platform.system().lower() == "windows":
+        check_command = "where " + command
     else:
-        check_command = 'command -v ' + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }'
+        check_command = "command -v " + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }'
     result = os.system(check_command)
     return result
 
 
 class OutputParser:
-
     @classmethod
     def parse_blocks(cls, text: str):
         # 首先根据"##"将文本分割成不同的block
@@ -54,7 +53,7 @@ class OutputParser:
 
     @classmethod
     def parse_code(cls, text: str, lang: str = "") -> str:
-        pattern = rf'```{lang}.*?\s+(.*?)```'
+        pattern = rf"```{lang}.*?\s+(.*?)```"
         match = re.search(pattern, text, re.DOTALL)
         if match:
             code = match.group(1)
@@ -65,13 +64,13 @@ class OutputParser:
     @classmethod
     def parse_str(cls, text: str):
         text = text.split("=")[-1]
-        text = text.strip().strip("'").strip("\"")
+        text = text.strip().strip("'").strip('"')
         return text
 
     @classmethod
     def parse_file_list(cls, text: str) -> list[str]:
         # Regular expression pattern to find the tasks list.
-        pattern = r'\s*(.*=.*)?(\[.*\])'
+        pattern = r"\s*(.*=.*)?(\[.*\])"
 
         # Extract tasks list string using regex.
         match = re.search(pattern, text, re.DOTALL)
@@ -83,12 +82,12 @@ class OutputParser:
         else:
             tasks = text.split("\n")
         return tasks
-    
+
     @staticmethod
     def parse_python_code(text: str) -> str:
         for pattern in (
-            r'(.*?```python.*?\s+)?(?P<code>.*)(```.*?)', 
-            r'(.*?```python.*?\s+)?(?P<code>.*)', 
+            r"(.*?```python.*?\s+)?(?P<code>.*)(```.*?)",
+            r"(.*?```python.*?\s+)?(?P<code>.*)",
         ):
             match = re.search(pattern, text, re.DOTALL)
             if not match:
@@ -135,7 +134,7 @@ class OutputParser:
                 typing = typing_define[0]
             else:
                 typing = typing_define
-            if typing == List[str] or typing == List[Tuple[str, str]]:
+            if typing == List[str] or typing == List[Tuple[str, str]] or typing == List[List[str]]:
                 # 尝试解析list
                 try:
                     content = cls.parse_file_list(text=content)
@@ -151,9 +150,55 @@ class OutputParser:
             parsed_data[block] = content
         return parsed_data
 
+    @classmethod
+    def extract_struct(cls, text: str, data_type: Union[type(list), type(dict)]) -> Union[list, dict]:
+        """Extracts and parses a specified type of structure (dictionary or list) from the given text.
+        The text only contains a list or dictionary, which may have nested structures.
+
+        Args:
+            text: The text containing the structure (dictionary or list).
+            data_type: The data type to extract, can be "list" or "dict".
+
+        Returns:
+            - If extraction and parsing are successful, it returns the corresponding data structure (list or dictionary).
+            - If extraction fails or parsing encounters an error, it throw an exception.
+
+        Examples:
+            >>> text = 'xxx [1, 2, ["a", "b", [3, 4]], {"x": 5, "y": [6, 7]}] xxx'
+            >>> result_list = OutputParser.extract_struct(text, "list")
+            >>> print(result_list)
+            >>> # Output: [1, 2, ["a", "b", [3, 4]], {"x": 5, "y": [6, 7]}]
+
+            >>> text = 'xxx {"x": 1, "y": {"a": 2, "b": {"c": 3}}} xxx'
+            >>> result_dict = OutputParser.extract_struct(text, "dict")
+            >>> print(result_dict)
+            >>> # Output: {"x": 1, "y": {"a": 2, "b": {"c": 3}}}
+        """
+        # Find the first "[" or "{" and the last "]" or "}"
+        start_index = text.find("[" if data_type is list else "{")
+        end_index = text.rfind("]" if data_type is list else "}")
+
+        if start_index != -1 and end_index != -1:
+            # Extract the structure part
+            structure_text = text[start_index:end_index + 1]
+
+            try:
+                # Attempt to convert the text to a Python data type using ast.literal_eval
+                result = ast.literal_eval(structure_text)
+
+                # Ensure the result matches the specified data type
+                if isinstance(result, list) or isinstance(result, dict):
+                    return result
+
+                raise ValueError(f"The extracted structure is not a {data_type}.")
+
+            except (ValueError, SyntaxError) as e:
+                raise Exception(f"Error while extracting and parsing the {data_type}: {e}")
+        else:
+            raise Exception(f"No {data_type} found in the text.")
+
 
 class CodeParser:
-
     @classmethod
     def parse_block(cls, block: str, text: str) -> str:
         blocks = cls.parse_blocks(text)
@@ -184,21 +229,22 @@ class CodeParser:
     def parse_code(cls, block: str, text: str, lang: str = "") -> str:
         if block:
             text = cls.parse_block(block, text)
-        pattern = rf'```{lang}.*?\s+(.*?)```'
+        pattern = rf"```{lang}.*?\s+(.*?)```"
         match = re.search(pattern, text, re.DOTALL)
         if match:
             code = match.group(1)
         else:
             logger.error(f"{pattern} not match following text:")
             logger.error(text)
-            raise Exception
+            # raise Exception
+            return ""
         return code
 
     @classmethod
     def parse_str(cls, block: str, text: str, lang: str = ""):
         code = cls.parse_code(block, text, lang)
         code = code.split("=")[-1]
-        code = code.strip().strip("'").strip("\"")
+        code = code.strip().strip("'").strip('"')
         return code
 
     @classmethod
@@ -206,7 +252,7 @@ class CodeParser:
         # Regular expression pattern to find the tasks list.
         code = cls.parse_code(block, text, lang)
         # print(code)
-        pattern = r'\s*(.*=.*)?(\[.*\])'
+        pattern = r"\s*(.*=.*)?(\[.*\])"
 
         # Extract tasks list string using regex.
         match = re.search(pattern, code, re.DOTALL)
@@ -229,7 +275,7 @@ class NoMoneyException(Exception):
         super().__init__(self.message)
 
     def __str__(self):
-        return f'{self.message} -> Amount required: {self.amount}'
+        return f"{self.message} -> Amount required: {self.amount}"
 
 
 def print_members(module, indent=0):
@@ -239,19 +285,19 @@ def print_members(module, indent=0):
     :param indent:
     :return:
     """
-    prefix = ' ' * indent
+    prefix = " " * indent
     for name, obj in inspect.getmembers(module):
         print(name, obj)
         if inspect.isclass(obj):
-            print(f'{prefix}Class: {name}')
+            print(f"{prefix}Class: {name}")
             # print the methods within the class
-            if name in ['__class__', '__base__']:
+            if name in ["__class__", "__base__"]:
                 continue
             print_members(obj, indent + 2)
         elif inspect.isfunction(obj):
-            print(f'{prefix}Function: {name}')
+            print(f"{prefix}Function: {name}")
         elif inspect.ismethod(obj):
-            print(f'{prefix}Method: {name}')
+            print(f"{prefix}Method: {name}")
 
 
 def parse_recipient(text):

metagpt/utils/custom_decoder.py

@@ -0,0 +1,297 @@
+import json
+import re
+from json import JSONDecodeError
+from json.decoder import _decode_uXXXX
+
+NUMBER_RE = re.compile(r"(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?", (re.VERBOSE | re.MULTILINE | re.DOTALL))
+
+
+def py_make_scanner(context):
+    parse_object = context.parse_object
+    parse_array = context.parse_array
+    parse_string = context.parse_string
+    match_number = NUMBER_RE.match
+    strict = context.strict
+    parse_float = context.parse_float
+    parse_int = context.parse_int
+    parse_constant = context.parse_constant
+    object_hook = context.object_hook
+    object_pairs_hook = context.object_pairs_hook
+    memo = context.memo
+
+    def _scan_once(string, idx):
+        try:
+            nextchar = string[idx]
+        except IndexError:
+            raise StopIteration(idx) from None
+
+        if nextchar == '"' or nextchar == "'":
+            if idx + 2 < len(string) and string[idx + 1] == nextchar and string[idx + 2] == nextchar:
+                # Handle the case where the next two characters are the same as nextchar
+                return parse_string(string, idx + 3, strict, delimiter=nextchar * 3)  # triple quote
+            else:
+                # Handle the case where the next two characters are not the same as nextchar
+                return parse_string(string, idx + 1, strict, delimiter=nextchar)
+        elif nextchar == "{":
+            return parse_object((string, idx + 1), strict, _scan_once, object_hook, object_pairs_hook, memo)
+        elif nextchar == "[":
+            return parse_array((string, idx + 1), _scan_once)
+        elif nextchar == "n" and string[idx : idx + 4] == "null":
+            return None, idx + 4
+        elif nextchar == "t" and string[idx : idx + 4] == "true":
+            return True, idx + 4
+        elif nextchar == "f" and string[idx : idx + 5] == "false":
+            return False, idx + 5
+
+        m = match_number(string, idx)
+        if m is not None:
+            integer, frac, exp = m.groups()
+            if frac or exp:
+                res = parse_float(integer + (frac or "") + (exp or ""))
+            else:
+                res = parse_int(integer)
+            return res, m.end()
+        elif nextchar == "N" and string[idx : idx + 3] == "NaN":
+            return parse_constant("NaN"), idx + 3
+        elif nextchar == "I" and string[idx : idx + 8] == "Infinity":
+            return parse_constant("Infinity"), idx + 8
+        elif nextchar == "-" and string[idx : idx + 9] == "-Infinity":
+            return parse_constant("-Infinity"), idx + 9
+        else:
+            raise StopIteration(idx)
+
+    def scan_once(string, idx):
+        try:
+            return _scan_once(string, idx)
+        finally:
+            memo.clear()
+
+    return scan_once
+
+
+FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
+STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
+STRINGCHUNK_SINGLEQUOTE = re.compile(r"(.*?)([\'\\\x00-\x1f])", FLAGS)
+STRINGCHUNK_TRIPLE_DOUBLE_QUOTE = re.compile(r"(.*?)(\"\"\"|[\\\x00-\x1f])", FLAGS)
+STRINGCHUNK_TRIPLE_SINGLEQUOTE = re.compile(r"(.*?)('''|[\\\x00-\x1f])", FLAGS)
+BACKSLASH = {
+    '"': '"',
+    "\\": "\\",
+    "/": "/",
+    "b": "\b",
+    "f": "\f",
+    "n": "\n",
+    "r": "\r",
+    "t": "\t",
+}
+WHITESPACE = re.compile(r"[ \t\n\r]*", FLAGS)
+WHITESPACE_STR = " \t\n\r"
+
+
+def JSONObject(
+    s_and_end, strict, scan_once, object_hook, object_pairs_hook, memo=None, _w=WHITESPACE.match, _ws=WHITESPACE_STR
+):
+    """Parse a JSON object from a string and return the parsed object.
+
+    Args:
+        s_and_end (tuple): A tuple containing the input string to parse and the current index within the string.
+        strict (bool): If `True`, enforces strict JSON string decoding rules.
+            If `False`, allows literal control characters in the string. Defaults to `True`.
+        scan_once (callable): A function to scan and parse JSON values from the input string.
+        object_hook (callable): A function that, if specified, will be called with the parsed object as a dictionary.
+        object_pairs_hook (callable): A function that, if specified, will be called with the parsed object as a list of pairs.
+        memo (dict, optional): A dictionary used to memoize string keys for optimization. Defaults to None.
+        _w (function): A regular expression matching function for whitespace. Defaults to WHITESPACE.match.
+        _ws (str): A string containing whitespace characters. Defaults to WHITESPACE_STR.
+
+    Returns:
+        tuple or dict: A tuple containing the parsed object and the index of the character in the input string
+        after the end of the object.
+    """
+
+    s, end = s_and_end
+    pairs = []
+    pairs_append = pairs.append
+    # Backwards compatibility
+    if memo is None:
+        memo = {}
+    memo_get = memo.setdefault
+    # Use a slice to prevent IndexError from being raised, the following
+    # check will raise a more specific ValueError if the string is empty
+    nextchar = s[end : end + 1]
+    # Normally we expect nextchar == '"'
+    if nextchar != '"' and nextchar != "'":
+        if nextchar in _ws:
+            end = _w(s, end).end()
+            nextchar = s[end : end + 1]
+        # Trivial empty object
+        if nextchar == "}":
+            if object_pairs_hook is not None:
+                result = object_pairs_hook(pairs)
+                return result, end + 1
+            pairs = {}
+            if object_hook is not None:
+                pairs = object_hook(pairs)
+            return pairs, end + 1
+        elif nextchar != '"':
+            raise JSONDecodeError("Expecting property name enclosed in double quotes", s, end)
+    end += 1
+    while True:
+        if end + 1 < len(s) and s[end] == nextchar and s[end + 1] == nextchar:
+            # Handle the case where the next two characters are the same as nextchar
+            key, end = scanstring(s, end + 2, strict, delimiter=nextchar * 3)
+        else:
+            # Handle the case where the next two characters are not the same as nextchar
+            key, end = scanstring(s, end, strict, delimiter=nextchar)
+        key = memo_get(key, key)
+        # To skip some function call overhead we optimize the fast paths where
+        # the JSON key separator is ": " or just ":".
+        if s[end : end + 1] != ":":
+            end = _w(s, end).end()
+            if s[end : end + 1] != ":":
+                raise JSONDecodeError("Expecting ':' delimiter", s, end)
+        end += 1
+
+        try:
+            if s[end] in _ws:
+                end += 1
+                if s[end] in _ws:
+                    end = _w(s, end + 1).end()
+        except IndexError:
+            pass
+
+        try:
+            value, end = scan_once(s, end)
+        except StopIteration as err:
+            raise JSONDecodeError("Expecting value", s, err.value) from None
+        pairs_append((key, value))
+        try:
+            nextchar = s[end]
+            if nextchar in _ws:
+                end = _w(s, end + 1).end()
+                nextchar = s[end]
+        except IndexError:
+            nextchar = ""
+        end += 1
+
+        if nextchar == "}":
+            break
+        elif nextchar != ",":
+            raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
+        end = _w(s, end).end()
+        nextchar = s[end : end + 1]
+        end += 1
+        if nextchar != '"':
+            raise JSONDecodeError("Expecting property name enclosed in double quotes", s, end - 1)
+    if object_pairs_hook is not None:
+        result = object_pairs_hook(pairs)
+        return result, end
+    pairs = dict(pairs)
+    if object_hook is not None:
+        pairs = object_hook(pairs)
+    return pairs, end
+
+
+def py_scanstring(s, end, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match, delimiter='"'):
+    """Scan the string s for a JSON string.
+
+    Args:
+        s (str): The input string to be scanned for a JSON string.
+        end (int): The index of the character in `s` after the quote that started the JSON string.
+        strict (bool): If `True`, enforces strict JSON string decoding rules.
+            If `False`, allows literal control characters in the string. Defaults to `True`.
+        _b (dict): A dictionary containing escape sequence mappings.
+        _m (function): A regular expression matching function for string chunks.
+        delimiter (str): The string delimiter used to define the start and end of the JSON string.
+            Can be one of: '"', "'", '\"""', or "'''". Defaults to '"'.
+
+    Returns:
+        tuple: A tuple containing the decoded string and the index of the character in `s`
+        after the end quote.
+    """
+
+    chunks = []
+    _append = chunks.append
+    begin = end - 1
+    if delimiter == '"':
+        _m = STRINGCHUNK.match
+    elif delimiter == "'":
+        _m = STRINGCHUNK_SINGLEQUOTE.match
+    elif delimiter == '"""':
+        _m = STRINGCHUNK_TRIPLE_DOUBLE_QUOTE.match
+    else:
+        _m = STRINGCHUNK_TRIPLE_SINGLEQUOTE.match
+    while 1:
+        chunk = _m(s, end)
+        if chunk is None:
+            raise JSONDecodeError("Unterminated string starting at", s, begin)
+        end = chunk.end()
+        content, terminator = chunk.groups()
+        # Content is contains zero or more unescaped string characters
+        if content:
+            _append(content)
+        # Terminator is the end of string, a literal control character,
+        # or a backslash denoting that an escape sequence follows
+        if terminator == delimiter:
+            break
+        elif terminator != "\\":
+            if strict:
+                # msg = "Invalid control character %r at" % (terminator,)
+                msg = "Invalid control character {0!r} at".format(terminator)
+                raise JSONDecodeError(msg, s, end)
+            else:
+                _append(terminator)
+                continue
+        try:
+            esc = s[end]
+        except IndexError:
+            raise JSONDecodeError("Unterminated string starting at", s, begin) from None
+        # If not a unicode escape sequence, must be in the lookup table
+        if esc != "u":
+            try:
+                char = _b[esc]
+            except KeyError:
+                msg = "Invalid \\escape: {0!r}".format(esc)
+                raise JSONDecodeError(msg, s, end)
+            end += 1
+        else:
+            uni = _decode_uXXXX(s, end)
+            end += 5
+            if 0xD800 <= uni <= 0xDBFF and s[end : end + 2] == "\\u":
+                uni2 = _decode_uXXXX(s, end + 1)
+                if 0xDC00 <= uni2 <= 0xDFFF:
+                    uni = 0x10000 + (((uni - 0xD800) << 10) | (uni2 - 0xDC00))
+                    end += 6
+            char = chr(uni)
+        _append(char)
+    return "".join(chunks), end
+
+
+scanstring = py_scanstring
+
+
+class CustomDecoder(json.JSONDecoder):
+    def __init__(
+        self,
+        *,
+        object_hook=None,
+        parse_float=None,
+        parse_int=None,
+        parse_constant=None,
+        strict=True,
+        object_pairs_hook=None
+    ):
+        super().__init__(
+            object_hook=object_hook,
+            parse_float=parse_float,
+            parse_int=parse_int,
+            parse_constant=parse_constant,
+            strict=strict,
+            object_pairs_hook=object_pairs_hook,
+        )
+        self.parse_object = JSONObject
+        self.parse_string = py_scanstring
+        self.scan_once = py_make_scanner(self)
+
+    def decode(self, s, _w=json.decoder.WHITESPACE.match):
+        return super().decode(s)

metagpt/utils/file.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# _*_ coding: utf-8 _*_
+"""
+@Time    : 2023/9/4 15:40:40
+@Author  : Stitch-z
+@File    : file.py
+@Describe : General file operations.
+"""
+import aiofiles
+from pathlib import Path
+
+from metagpt.logs import logger
+
+
+class File:
+    """A general util for file operations."""
+
+    CHUNK_SIZE = 64 * 1024
+
+    @classmethod
+    async def write(cls, root_path: Path, filename: str, content: bytes) -> Path:
+        """Write the file content to the local specified path.
+
+        Args:
+            root_path: The root path of file, such as "/data".
+            filename: The name of file, such as "test.txt".
+            content: The binary content of file.
+
+        Returns:
+            The full filename of file, such as "/data/test.txt".
+
+        Raises:
+            Exception: If an unexpected error occurs during the file writing process.
+        """
+        try:
+            root_path.mkdir(parents=True, exist_ok=True)
+            full_path = root_path / filename
+            async with aiofiles.open(full_path, mode="wb") as writer:
+                await writer.write(content)
+                logger.debug(f"Successfully write file: {full_path}")
+                return full_path
+        except Exception as e:
+            logger.error(f"Error writing file: {e}")
+            raise e
+
+    @classmethod
+    async def read(cls, file_path: Path, chunk_size: int = None) -> bytes:
+        """Partitioning read the file content from the local specified path.
+
+        Args:
+            file_path: The full file name of file, such as "/data/test.txt".
+            chunk_size: The size of each chunk in bytes (default is 64kb).
+
+        Returns:
+            The binary content of file.
+
+        Raises:
+            Exception: If an unexpected error occurs during the file reading process.
+        """
+        try:
+            chunk_size = chunk_size or cls.CHUNK_SIZE
+            async with aiofiles.open(file_path, mode="rb") as reader:
+                chunks = list()
+                while True:
+                    chunk = await reader.read(chunk_size)
+                    if not chunk:
+                        break
+                    chunks.append(chunk)
+                content = b''.join(chunks)
+                logger.debug(f"Successfully read file, the path of file: {file_path}")
+                return content
+        except Exception as e:
+            logger.error(f"Error reading file: {e}")
+            raise e
+

metagpt/utils/get_template.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/9/19 20:39
+@Author  : femto Zheng
+@File    : get_template.py
+"""
+from metagpt.config import CONFIG
+
+
+def get_template(templates, format=CONFIG.prompt_format):
+    selected_templates = templates.get(format)
+    if selected_templates is None:
+        raise ValueError(f"Can't find {format} in passed in templates")
+
+    # Extract the selected templates
+    prompt_template = selected_templates["PROMPT_TEMPLATE"]
+    format_example = selected_templates["FORMAT_EXAMPLE"]
+
+    return prompt_template, format_example

metagpt/utils/highlight.py

@@ -0,0 +1,25 @@
+# 添加代码语法高亮显示
+from pygments import highlight as highlight_
+from pygments.lexers import PythonLexer, SqlLexer
+from pygments.formatters import TerminalFormatter, HtmlFormatter
+
+
+def highlight(code: str, language: str = 'python', formatter: str = 'terminal'):
+    # 指定要高亮的语言
+    if language.lower() == 'python':
+        lexer = PythonLexer()
+    elif language.lower() == 'sql':
+        lexer = SqlLexer()
+    else:
+        raise ValueError(f"Unsupported language: {language}")
+
+    # 指定输出格式
+    if formatter.lower() == 'terminal':
+        formatter = TerminalFormatter()
+    elif formatter.lower() == 'html':
+        formatter = HtmlFormatter()
+    else:
+        raise ValueError(f"Unsupported formatter: {formatter}")
+
+    # 使用 Pygments 高亮代码片段
+    return highlight_(code, lexer, formatter)

metagpt/utils/json_to_markdown.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/9/11 11:50
+@Author  : femto Zheng
+@File    : json_to_markdown.py
+"""
+
+
+# since we original write docs/*.md in markdown format, so I convert json back to markdown
+def json_to_markdown(data, depth=2):
+    """
+    Convert a JSON object to Markdown with headings for keys and lists for arrays, supporting nested objects.
+
+    Args:
+        data: JSON object (dictionary) or value.
+        depth (int): Current depth level for Markdown headings.
+
+    Returns:
+        str: Markdown representation of the JSON data.
+    """
+    markdown = ""
+
+    if isinstance(data, dict):
+        for key, value in data.items():
+            if isinstance(value, list):
+                # Handle JSON arrays
+                markdown += "#" * depth + f" {key}\n\n"
+                items = [str(item) for item in value]
+                markdown += "- " + "\n- ".join(items) + "\n\n"
+            elif isinstance(value, dict):
+                # Handle nested JSON objects
+                markdown += "#" * depth + f" {key}\n\n"
+                markdown += json_to_markdown(value, depth + 1)
+            else:
+                # Handle other values
+                markdown += "#" * depth + f" {key}\n\n{value}\n\n"
+    else:
+        # Handle non-dictionary JSON data
+        markdown = str(data)
+
+    return markdown

metagpt/utils/make_sk_kernel.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/9/13 12:29
+@Author  : femto Zheng
+@File    : make_sk_kernel.py
+"""
+import semantic_kernel as sk
+from semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion import (
+    AzureChatCompletion,
+)
+from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import (
+    OpenAIChatCompletion,
+)
+
+from metagpt.config import CONFIG
+
+
+def make_sk_kernel():
+    kernel = sk.Kernel()
+    if CONFIG.openai_api_type == "azure":
+        kernel.add_chat_service(
+            "chat_completion",
+            AzureChatCompletion(CONFIG.deployment_name, CONFIG.openai_api_base, CONFIG.openai_api_key),
+        )
+    else:
+        kernel.add_chat_service(
+            "chat_completion",
+            OpenAIChatCompletion(
+                CONFIG.openai_api_model, CONFIG.openai_api_key, org_id=None, endpoint=CONFIG.openai_api_base
+            ),
+        )
+
+    return kernel

metagpt/utils/mermaid.py

@@ -2,10 +2,11 @@
 # -*- coding: utf-8 -*-
 """
 @Time    : 2023/7/4 10:53
-@Author  : alexanderwu
+@Author  : alexanderwu alitrack
 @File    : mermaid.py
 """
-import subprocess
+import asyncio
+import os
 from pathlib import Path
 
 from metagpt.config import CONFIG
@@ -14,31 +15,35 @@ from metagpt.logs import logger
 from metagpt.utils.common import check_cmd_exists
 
 
-def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
+async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
     """suffix: png/svg/pdf
 
     :param mermaid_code: mermaid code
     :param output_file_without_suffix: output filename
     :param width:
     :param height:
-    :return: 0 if succed, -1 if failed
+    :return: 0 if succeed, -1 if failed
     """
     # Write the Mermaid code to a temporary file
+    dir_name = os.path.dirname(output_file_without_suffix)
+    if dir_name and not os.path.exists(dir_name):
+        os.makedirs(dir_name)
     tmp = Path(f"{output_file_without_suffix}.mmd")
     tmp.write_text(mermaid_code, encoding="utf-8")
 
-    if check_cmd_exists("mmdc") != 0:
-        logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
-        return -1
+    engine = CONFIG.mermaid_engine.lower()
+    if engine == "nodejs":
+        if check_cmd_exists(CONFIG.mmdc) != 0:
+            logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
+            return -1
 
-    for suffix in ["pdf", "svg", "png"]:
-        output_file = f"{output_file_without_suffix}.{suffix}"
-        # Call the `mmdc` command to convert the Mermaid code to a PNG
-        logger.info(f"Generating {output_file}..")
+        for suffix in ["pdf", "svg", "png"]:
+            output_file = f"{output_file_without_suffix}.{suffix}"
+            # Call the `mmdc` command to convert the Mermaid code to a PNG
+            logger.info(f"Generating {output_file}..")
 
-        if CONFIG.puppeteer_config:
-            subprocess.run(
-                [
+            if CONFIG.puppeteer_config:
+                commands = [
                     CONFIG.mmdc,
                     "-p",
                     CONFIG.puppeteer_config,
@@ -51,9 +56,32 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height
                     "-H",
                     str(height),
                 ]
+            else:
+                commands = [CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]
+            process = await asyncio.create_subprocess_shell(
+                " ".join(commands), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
             )
+
+            stdout, stderr = await process.communicate()
+            if stdout:
+                logger.info(stdout.decode())
+            if stderr:
+                logger.error(stderr.decode())
+    else:
+        if engine == "playwright":
+            from metagpt.utils.mmdc_playwright import mermaid_to_file
+
+            return await mermaid_to_file(mermaid_code, output_file_without_suffix, width, height)
+        elif engine == "pyppeteer":
+            from metagpt.utils.mmdc_pyppeteer import mermaid_to_file
+
+            return await mermaid_to_file(mermaid_code, output_file_without_suffix, width, height)
+        elif engine == "ink":
+            from metagpt.utils.mmdc_ink import mermaid_to_file
+
+            return await mermaid_to_file(mermaid_code, output_file_without_suffix)
         else:
-            subprocess.run([CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)])
+            logger.warning(f"Unsupported mermaid engine: {engine}")
     return 0
 
 
@@ -109,6 +137,7 @@ MMC2 = """sequenceDiagram
 
 
 if __name__ == "__main__":
-    # logger.info(print_members(print_members))
-    mermaid_to_file(MMC1, PROJECT_ROOT / "tmp/1.png")
-    mermaid_to_file(MMC2, PROJECT_ROOT / "tmp/2.png")
+    loop = asyncio.new_event_loop()
+    result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1"))
+    result = loop.run_until_complete(mermaid_to_file(MMC2, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1"))
+    loop.close()

metagpt/utils/minecraft/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# @Date    : 2023/9/24 0:32
+# @Author  : stellahong (stellahong@fuzhi.ai)
+# @Desc    :
+from .load_prompts import load_prompt
+from .json_utils import *
+from .file_utils import *
+from .action_rsp_parser import parse_js_code, parse_action_response

metagpt/utils/minecraft/action_rsp_parser.py

@@ -0,0 +1,91 @@
+import re
+import time
+from javascript import require
+
+def parse_js_code(msg: str):
+    '''
+    Extract and Parse JavaScript code blocks
+    '''
+    babel = require("@babel/core")
+    code_pattern = re.compile(r"```(?:javascript|js)(.*?)```", re.DOTALL)
+    code = "\n".join(code_pattern.findall(msg))
+    parsed = babel.parse(code)
+    return parsed
+
+def parse_action_response(msg: str):
+    """
+    Input:
+    '''
+    Explain: ...
+    Plan: ...
+    Code:
+    ```javascript
+    ...
+    ```
+    '''
+
+    Return:
+    {
+        "program_code": program_code,
+        "program_name": main_function["name"],
+        "exec_code": exec_code,
+    } or 
+    
+    "{error}"
+
+    Refer to @ https://github.com/MineDojo/Voyager/blob/main/voyager/agents/action.py
+    """
+
+    retry = 3
+    error = None # 3 times failed return error
+    babel_generator = require("@babel/generator").default
+    while retry > 0:
+        try:
+            parsed = parse_js_code(msg)
+            # Collect func list: check if func & async
+            functions = []
+            assert len(list(parsed.program.body)) > 0, "No functions found"
+            for i, node in enumerate(parsed.program.body):
+                if node.type != "FunctionDeclaration":
+                    continue
+                node_type = (
+                    "AsyncFunctionDeclaration"
+                    if node["async"]
+                    else "FunctionDeclaration"
+                )
+                functions.append(
+                    {
+                        "name": node.id.name,
+                        "type": node_type,
+                        "body": babel_generator(node).code,
+                        "params": list(node["params"]),
+                    }
+                )
+
+            # Ensure main_function is the last async function
+            main_function = None
+            for function in reversed(functions):
+                if function["type"] == "AsyncFunctionDeclaration":
+                    main_function = function
+                    break
+            assert (
+                main_function is not None
+            ), "No async function found. Your main function must be async."
+            assert (
+                len(main_function["params"]) == 1
+                and main_function["params"][0].name == "bot"
+            ), f"Main function {main_function['name']} must take a single argument named 'bot'"
+
+            # Split to program_code & exec_code for output
+            program_code = "\n\n".join(function["body"] for function in functions)
+            exec_code = f"await {main_function['name']}(bot);"
+            return {
+                "program_code": program_code,
+                "program_name": main_function["name"],
+                "exec_code": exec_code,
+            }
+        except Exception as e:
+            retry -= 1
+            error = e
+            time.sleep(1)
+    return f"Error parsing action response (before program execution): {error}"

metagpt/utils/minecraft/file_utils.py

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+# @Date    : 2023/09/25 16:13
+# @Author  : yuymf
+# @Desc    : Temp Using :File system utils.@ https://github.com/MineDojo/Voyager/blob/main/voyager/utils/file_utils.py
+import collections
+import os
+
+f_ext = os.path.splitext
+
+f_size = os.path.getsize
+
+is_file = os.path.isfile
+
+is_dir = os.path.isdir
+
+get_dir = os.path.dirname
+
+
+def is_sequence(obj):
+    """
+    Returns:
+      True if the sequence is a collections.Sequence and not a string.
+    """
+    return isinstance(obj, collections.abc.Sequence) and not isinstance(obj, str)
+
+
+def pack_varargs(args):
+    """
+    Pack *args or a single list arg as list
+
+    def f(*args):
+        arg_list = pack_varargs(args)
+        # arg_list is now packed as a list
+    """
+    assert isinstance(args, tuple), "please input the tuple `args` as in *args"
+    if len(args) == 1 and is_sequence(args[0]):
+        return args[0]
+    else:
+        return args
+
+
+def f_expand(fpath):
+    return os.path.expandvars(os.path.expanduser(fpath))
+
+
+def f_exists(*fpaths):
+    return os.path.exists(f_join(*fpaths))
+
+
+def f_join(*fpaths):
+    """
+    join file paths and expand special symbols like `~` for home dir
+    """
+    fpaths = pack_varargs(fpaths)
+    fpath = f_expand(os.path.join(*fpaths))
+    if isinstance(fpath, str):
+        fpath = fpath.strip()
+    return fpath
+
+
+def f_mkdir(*fpaths):
+    """
+    Recursively creates all the subdirs
+    If exist, do nothing.
+    """
+    fpath = f_join(*fpaths)
+    os.makedirs(fpath, exist_ok=True)
+    return fpath
+
+
+def load_text(*fpaths, by_lines=False):
+    with open(f_join(*fpaths), "r") as fp:
+        if by_lines:
+            return fp.readlines()
+        else:
+            return fp.read()
+
+
+def load_text_lines(*fpaths):
+    return load_text(*fpaths, by_lines=True)
+
+
+# aliases to be consistent with other load_* and dump_*
+text_load = load_text
+read_text = load_text
+read_text_lines = load_text_lines

metagpt/utils/minecraft/json_utils.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+# @Date    : 2023/09/25 19:27
+# @Author  : yuymf
+# @Desc    : Temp using @https://github.com/MineDojo/Voyager/blob/main/voyager/utils/json_utils.py
+
+import json
+import re
+from typing import Any, Dict, Union
+
+
+def extract_char_position(error_message: str) -> int:
+    """Extract the character position from the JSONDecodeError message.
+    Args:
+        error_message (str): The error message from the JSONDecodeError
+          exception.
+    Returns:
+        int: The character position.
+    """
+    import re
+
+    char_pattern = re.compile(r"\(char (\d+)\)")
+    if match := char_pattern.search(error_message):
+        return int(match[1])
+    else:
+        raise ValueError("Character position not found in the error message.")
+
+
+def add_quotes_to_property_names(json_string: str) -> str:
+    """
+    Add quotes to property names in a JSON string.
+    Args:
+        json_string (str): The JSON string.
+    Returns:
+        str: The JSON string with quotes added to property names.
+    """
+
+    def replace_func(match):
+        return f'"{match.group(1)}":'
+
+    property_name_pattern = re.compile(r"(\w+):")
+    corrected_json_string = property_name_pattern.sub(replace_func, json_string)
+
+    try:
+        json.loads(corrected_json_string)
+        return corrected_json_string
+    except json.JSONDecodeError as e:
+        raise e
+
+
+def balance_braces(json_string: str) -> str:
+    """
+    Balance the braces in a JSON string.
+    Args:
+        json_string (str): The JSON string.
+    Returns:
+        str: The JSON string with braces balanced.
+    """
+
+    open_braces_count = json_string.count("{")
+    close_braces_count = json_string.count("}")
+
+    while open_braces_count > close_braces_count:
+        json_string += "}"
+        close_braces_count += 1
+
+    while close_braces_count > open_braces_count:
+        json_string = json_string.rstrip("}")
+        close_braces_count -= 1
+
+    try:
+        json.loads(json_string)
+        return json_string
+    except json.JSONDecodeError as e:
+        raise e
+
+
+def fix_invalid_escape(json_str: str, error_message: str) -> str:
+    while error_message.startswith("Invalid \\escape"):
+        bad_escape_location = extract_char_position(error_message)
+        json_str = json_str[:bad_escape_location] + json_str[bad_escape_location + 1 :]
+        try:
+            json.loads(json_str)
+            return json_str
+        except json.JSONDecodeError as e:
+            error_message = str(e)
+    return json_str
+
+
+def correct_json(json_str: str) -> str:
+    """
+    Correct common JSON errors.
+    Args:
+        json_str (str): The JSON string.
+    """
+
+    try:
+        json.loads(json_str)
+        return json_str
+    except json.JSONDecodeError as e:
+        error_message = str(e)
+        if error_message.startswith("Invalid \\escape"):
+            json_str = fix_invalid_escape(json_str, error_message)
+        if error_message.startswith(
+            "Expecting property name enclosed in double quotes"
+        ):
+            json_str = add_quotes_to_property_names(json_str)
+            try:
+                json.loads(json_str)
+                return json_str
+            except json.JSONDecodeError as e:
+                error_message = str(e)
+        if balanced_str := balance_braces(json_str):
+            return balanced_str
+    return json_str
+
+
+def fix_and_parse_json(
+    json_str: str, try_to_fix_with_gpt: bool = True
+) -> Union[str, Dict[Any, Any]]:
+    """Fix and parse JSON string"""
+    try:
+        json_str = json_str.replace("\t", "")
+        return json.loads(json_str)
+    except json.JSONDecodeError as _:  # noqa: F841
+        json_str = correct_json(json_str)
+        try:
+            return json.loads(json_str)
+        except json.JSONDecodeError as _:  # noqa: F841
+            pass
+    try:
+        brace_index = json_str.index("{")
+        json_str = json_str[brace_index:]
+        last_brace_index = json_str.rindex("}")
+        json_str = json_str[: last_brace_index + 1]
+        return json.loads(json_str)
+    except json.JSONDecodeError as e:  # noqa: F841
+        raise e

metagpt/utils/minecraft/load_prompts.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+# @Date    : 2023/9/24 11:03
+# @Author  : stellahong (stellahong@fuzhi.ai)
+# @Desc    :
+import pkg_resources
+from .file_utils import load_text
+
+
+def load_prompt(prompt):
+    package_path = pkg_resources.resource_filename("metagpt", "")
+    return load_text(f"{package_path}/prompts/minecraft/{prompt}.txt")

metagpt/utils/minecraft/process_monitor.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+# @Date    : 2023/09/25 16:12
+# @Author  : yuymf
+# @Desc    : Temp using:@https://github.com/MineDojo/Voyager/blob/main/voyager/env/process_monitor.py
+import time
+import re
+import warnings
+from typing import List
+
+import psutil
+import subprocess
+import logging
+import threading
+
+import metagpt.utils.minecraft as U
+
+
+class SubprocessMonitor:
+    def __init__(
+        self,
+        commands: List[str],
+        name: str,
+        ready_match: str = r".*",
+        log_path: str = "logs",
+        callback_match: str = r"^(?!x)x$",  # regex that will never match
+        callback: callable = None,
+        finished_callback: callable = None,
+    ):
+        self.commands = commands
+        start_time = time.strftime("%Y%m%d_%H%M%S")
+        self.name = name
+        self.logger = logging.getLogger(name)
+        handler = logging.FileHandler(U.f_join(log_path, f"{start_time}.log"))
+        formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        )
+        handler.setFormatter(formatter)
+        self.logger.addHandler(handler)
+        self.logger.setLevel(logging.INFO)
+        self.process = None
+        self.ready_match = ready_match
+        self.ready_event = None
+        self.ready_line = None
+        self.callback_match = callback_match
+        self.callback = callback
+        self.finished_callback = finished_callback
+        self.thread = None
+
+    def _start(self):
+        self.logger.info(f"Starting subprocess with commands: {self.commands}")
+
+        self.process = psutil.Popen(
+            self.commands,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+        )
+        print(f"Subprocess {self.name} started with PID {self.process.pid}.")
+        for line in iter(self.process.stdout.readline, ""):
+            self.logger.info(line.strip())
+            if re.search(self.ready_match, line):
+                self.ready_line = line
+                self.logger.info("Subprocess is ready.")
+                self.ready_event.set()
+            if re.search(self.callback_match, line):
+                self.callback()
+        if not self.ready_event.is_set():
+            self.ready_event.set()
+            warnings.warn(f"Subprocess {self.name} failed to start.")
+        if self.finished_callback:
+            self.finished_callback()
+
+    def run(self):
+        self.ready_event = threading.Event()
+        self.ready_line = None
+        self.thread = threading.Thread(target=self._start)
+        self.thread.start()
+        self.ready_event.wait()
+
+    def stop(self):
+        self.logger.info("Stopping subprocess.")
+        if self.process and self.process.is_running():
+            self.process.terminate()
+            self.process.wait()
+
+    @property
+    def is_running(self):
+        if self.process is None:
+            return False
+        return self.process.is_running()

metagpt/utils/mmdc_ink.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/9/4 16:12
+@Author  : alitrack
+@File    : mermaid.py
+"""
+import base64
+import os
+
+from aiohttp import ClientSession,ClientError
+from metagpt.logs import logger
+
+
+async def mermaid_to_file(mermaid_code, output_file_without_suffix):
+    """suffix: png/svg
+    :param mermaid_code: mermaid code
+    :param output_file_without_suffix: output filename without suffix
+    :return: 0 if succeed, -1 if failed
+    """
+    encoded_string = base64.b64encode(mermaid_code.encode()).decode()
+
+    for suffix in ["svg", "png"]:
+        output_file = f"{output_file_without_suffix}.{suffix}"
+        path_type = "svg" if suffix == "svg" else "img"
+        url = f"https://mermaid.ink/{path_type}/{encoded_string}"
+        async with ClientSession() as session:
+            try:
+                async with session.get(url) as response:
+                    if response.status == 200:
+                        text = await response.content.read()
+                        with open(output_file, 'wb') as f:
+                            f.write(text)
+                        logger.info(f"Generating {output_file}..")
+                    else:
+                        logger.error(f"Failed to generate {output_file}")
+                        return -1
+            except ClientError as e:
+                logger.error(f"network error: {e}")
+                return -1
+    return 0

metagpt/utils/mmdc_playwright.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/9/4 16:12
+@Author  : Steven Lee
+@File    : mmdc_playwright.py
+"""
+
+import os
+from urllib.parse import urljoin
+from playwright.async_api import async_playwright
+from metagpt.logs import logger
+
+async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int:
+    """
+    Converts the given Mermaid code to various output formats and saves them to files.
+
+    Args:
+        mermaid_code (str): The Mermaid code to convert.
+        output_file_without_suffix (str): The output file name without the file extension.
+        width (int, optional): The width of the output image in pixels. Defaults to 2048.
+        height (int, optional): The height of the output image in pixels. Defaults to 2048.
+
+    Returns:
+        int: Returns 1 if the conversion and saving were successful, -1 otherwise.
+    """
+    suffixes=['png', 'svg', 'pdf']
+    __dirname = os.path.dirname(os.path.abspath(__file__))
+
+    async with async_playwright() as p:
+        browser = await p.chromium.launch()
+        device_scale_factor = 1.0
+        context = await browser.new_context(
+                viewport={'width': width, 'height': height},
+                device_scale_factor=device_scale_factor,
+            )
+        page = await context.new_page()
+
+        async def console_message(msg):
+            logger.info(msg.text)
+        page.on('console', console_message)
+
+        try:
+            await page.set_viewport_size({'width': width, 'height': height})
+
+            mermaid_html_path = os.path.abspath(
+                os.path.join(__dirname, 'index.html'))
+            mermaid_html_url = urljoin('file:', mermaid_html_path)
+            await page.goto(mermaid_html_url)
+            await page.wait_for_load_state("networkidle")
+
+            await page.wait_for_selector("div#container", state="attached")
+            mermaid_config = {}
+            background_color = "#ffffff"
+            my_css = ""
+            await page.evaluate(f'document.body.style.background = "{background_color}";')
+
+            metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => {
+                const { mermaid, zenuml } = globalThis;
+                await mermaid.registerExternalDiagrams([zenuml]);
+                mermaid.initialize({ startOnLoad: false, ...mermaidConfig });
+                const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container'));
+                document.getElementById('container').innerHTML = svg;
+                const svgElement = document.querySelector('svg');
+                svgElement.style.backgroundColor = backgroundColor;
+
+                if (myCSS) {
+                    const style = document.createElementNS('http://www.w3.org/2000/svg', 'style');
+                    style.appendChild(document.createTextNode(myCSS));
+                    svgElement.appendChild(style);
+                }
+
+            }''', [mermaid_code, mermaid_config, my_css, background_color])
+
+            if 'svg' in suffixes :
+                svg_xml = await page.evaluate('''() => {
+                    const svg = document.querySelector('svg');
+                    const xmlSerializer = new XMLSerializer();
+                    return xmlSerializer.serializeToString(svg);
+                }''')
+                logger.info(f"Generating {output_file_without_suffix}.svg..")
+                with open(f'{output_file_without_suffix}.svg', 'wb') as f:
+                    f.write(svg_xml.encode('utf-8'))
+
+            if  'png' in suffixes:
+                clip = await page.evaluate('''() => {
+                    const svg = document.querySelector('svg');
+                    const rect = svg.getBoundingClientRect();
+                    return {
+                        x: Math.floor(rect.left),
+                        y: Math.floor(rect.top),
+                        width: Math.ceil(rect.width),
+                        height: Math.ceil(rect.height)
+                    };
+                }''')
+                await page.set_viewport_size({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height']})
+                screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device')
+                logger.info(f"Generating {output_file_without_suffix}.png..")
+                with open(f'{output_file_without_suffix}.png', 'wb') as f:
+                    f.write(screenshot)
+            if 'pdf' in suffixes:
+                pdf_data = await page.pdf(scale=device_scale_factor)
+                logger.info(f"Generating {output_file_without_suffix}.pdf..")
+                with open(f'{output_file_without_suffix}.pdf', 'wb') as f:
+                    f.write(pdf_data)
+            return 0
+        except Exception as e:
+            logger.error(e)
+            return -1
+        finally:
+            await browser.close()

metagpt/utils/mmdc_pyppeteer.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/9/4 16:12
+@Author  : alitrack
+@File    : mmdc_pyppeteer.py
+"""
+import os
+from urllib.parse import urljoin
+from pyppeteer import launch
+from metagpt.logs import logger
+from metagpt.config import CONFIG
+
+async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int:
+    """
+    Converts the given Mermaid code to various output formats and saves them to files.
+
+    Args:
+        mermaid_code (str): The Mermaid code to convert.
+        output_file_without_suffix (str): The output file name without the file extension.
+        width (int, optional): The width of the output image in pixels. Defaults to 2048.
+        height (int, optional): The height of the output image in pixels. Defaults to 2048.
+
+    Returns:
+        int: Returns 1 if the conversion and saving were successful, -1 otherwise.
+    """
+    suffixes = ['png', 'svg', 'pdf']   
+    __dirname = os.path.dirname(os.path.abspath(__file__))
+
+    
+    if CONFIG.pyppeteer_executable_path:
+        browser = await launch(headless=True,
+                            executablePath=CONFIG.pyppeteer_executable_path,
+                            args=['--disable-extensions',"--no-sandbox"] 
+                            )
+    else:
+        logger.error("Please set the environment variable:PYPPETEER_EXECUTABLE_PATH.")
+        return -1
+    page = await browser.newPage()
+    device_scale_factor = 1.0
+
+    async def console_message(msg):
+        logger.info(msg.text)
+    page.on('console', console_message)
+
+    try:
+        await page.setViewport(viewport={'width': width, 'height': height, 'deviceScaleFactor': device_scale_factor})
+
+        mermaid_html_path = os.path.abspath(
+            os.path.join(__dirname, 'index.html'))
+        mermaid_html_url = urljoin('file:', mermaid_html_path)
+        await page.goto(mermaid_html_url)
+
+        await page.querySelector("div#container")
+        mermaid_config = {}
+        background_color = "#ffffff"
+        my_css = ""
+        await page.evaluate(f'document.body.style.background = "{background_color}";')
+
+        metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => {
+            const { mermaid, zenuml } = globalThis;
+            await mermaid.registerExternalDiagrams([zenuml]);
+            mermaid.initialize({ startOnLoad: false, ...mermaidConfig });
+            const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container'));
+            document.getElementById('container').innerHTML = svg;
+            const svgElement = document.querySelector('svg');
+            svgElement.style.backgroundColor = backgroundColor;
+
+            if (myCSS) {
+                const style = document.createElementNS('http://www.w3.org/2000/svg', 'style');
+                style.appendChild(document.createTextNode(myCSS));
+                svgElement.appendChild(style);
+            }
+        }''', [mermaid_code, mermaid_config, my_css, background_color])
+
+        if 'svg' in suffixes :
+            svg_xml = await page.evaluate('''() => {
+                const svg = document.querySelector('svg');
+                const xmlSerializer = new XMLSerializer();
+                return xmlSerializer.serializeToString(svg);
+            }''')
+            logger.info(f"Generating {output_file_without_suffix}.svg..")
+            with open(f'{output_file_without_suffix}.svg', 'wb') as f:
+                f.write(svg_xml.encode('utf-8'))
+
+        if  'png' in suffixes:
+            clip = await page.evaluate('''() => {
+                const svg = document.querySelector('svg');
+                const rect = svg.getBoundingClientRect();
+                return {
+                    x: Math.floor(rect.left),
+                    y: Math.floor(rect.top),
+                    width: Math.ceil(rect.width),
+                    height: Math.ceil(rect.height)
+                };
+            }''')
+            await page.setViewport({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height'], 'deviceScaleFactor': device_scale_factor})
+            screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device')
+            logger.info(f"Generating {output_file_without_suffix}.png..")
+            with open(f'{output_file_without_suffix}.png', 'wb') as f:
+                f.write(screenshot)
+        if 'pdf' in suffixes:
+            pdf_data = await page.pdf(scale=device_scale_factor)
+            logger.info(f"Generating {output_file_without_suffix}.pdf..")
+            with open(f'{output_file_without_suffix}.pdf', 'wb') as f:
+                f.write(pdf_data)
+        return 0
+    except Exception as e:
+        logger.error(e)
+        return -1
+    finally:
+        await browser.close()
+