feat: merge geekan:cli-etc

metagpt/utils/common.py

@@ -9,6 +9,8 @@
 @Modified By: mashenquan, 2023/11/27. Bug fix: `parse_recipient` failed to parse the recipient in certain GPT-3.5
         responses.
 """
+from __future__ import annotations
+
 import ast
 import contextlib
 import inspect

metagpt/utils/file_repository.py

@@ -54,7 +54,7 @@ class FileRepository:
         """
         pathname = self.workdir / filename
         pathname.parent.mkdir(parents=True, exist_ok=True)
-        async with aiofiles.open(str(pathname), mode="w") as writer:
+        async with aiofiles.open(str(pathname), mode="wb") as writer:
             await writer.write(content)
         logger.info(f"save to: {str(pathname)}")
 
@@ -98,7 +98,7 @@ class FileRepository:
         if not path_name.exists():
             return None
         try:
-            async with aiofiles.open(str(path_name), mode="r") as reader:
+            async with aiofiles.open(str(path_name), mode="rb") as reader:
                 doc.content = await reader.read()
         except FileNotFoundError as e:
             logger.info(f"open {str(path_name)} failed:{e}")
@@ -178,7 +178,7 @@ class FileRepository:
         # guid_suffix = str(uuid.uuid4())[:8]
         # return f"{current_time}x{guid_suffix}"
 
-    async def save_doc(self, doc: Document, with_suffix:str = None, dependencies: List[str] = None):
+    async def save_doc(self, doc: Document, with_suffix: str = None, dependencies: List[str] = None):
         """Save a Document instance as a PDF file.
 
         This method converts the content of the Document instance to Markdown,
@@ -238,7 +238,9 @@ class FileRepository:
         return await file_repo.save(filename=filename, content=content, dependencies=dependencies)
 
     @staticmethod
-    async def save_as(doc:Document, with_suffix:str = None, dependencies: List[str] = None, relative_path: Path | str = "."):
+    async def save_as(
+        doc: Document, with_suffix: str = None, dependencies: List[str] = None, relative_path: Path | str = "."
+    ):
         """Save a Document instance with optional modifications.
 
         This static method creates a new FileRepository, saves the Document instance

metagpt/utils/mermaid.py

@@ -10,7 +10,7 @@ import os
 from pathlib import Path
 
 from metagpt.config import CONFIG
-from metagpt.const import PROJECT_ROOT
+from metagpt.const import METAGPT_ROOT
 from metagpt.logs import logger
 from metagpt.utils.common import check_cmd_exists
 
@@ -34,7 +34,10 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
     engine = CONFIG.mermaid_engine.lower()
     if engine == "nodejs":
         if check_cmd_exists(CONFIG.mmdc) != 0:
-            logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
+            logger.warning(
+                "RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc,"
+                "or consider changing MERMAID_ENGINE to `playwright`, `pyppeteer`, or `ink`."
+            )
             return -1
 
         for suffix in ["pdf", "svg", "png"]:
@@ -66,7 +69,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
             if stdout:
                 logger.info(stdout.decode())
             if stderr:
-                logger.error(stderr.decode())
+                logger.warning(stderr.decode())
     else:
         if engine == "playwright":
             from metagpt.utils.mmdc_playwright import mermaid_to_file
@@ -138,6 +141,6 @@ MMC2 = """sequenceDiagram
 
 if __name__ == "__main__":
     loop = asyncio.new_event_loop()
-    result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1"))
-    result = loop.run_until_complete(mermaid_to_file(MMC2, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1"))
+    result = loop.run_until_complete(mermaid_to_file(MMC1, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1"))
+    result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1"))
     loop.close()

metagpt/utils/token_counter.py

@@ -21,7 +21,9 @@ TOKEN_COSTS = {
     "gpt-4-32k": {"prompt": 0.06, "completion": 0.12},
     "gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12},
     "gpt-4-0613": {"prompt": 0.06, "completion": 0.12},
+    "gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03},
     "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0},
+    "chatglm_turbo": {"prompt": 0.0, "completion": 0.00069},  # 32k version, prompt + completion tokens=0.005￥/k-tokens
 }
 
 
@@ -36,7 +38,9 @@ TOKEN_MAX = {
     "gpt-4-32k": 32768,
     "gpt-4-32k-0314": 32768,
     "gpt-4-0613": 8192,
+    "gpt-4-1106-preview": 128000,
     "text-embedding-ada-002": 8192,
+    "chatglm_turbo": 32768,
 }
 
 
@@ -54,21 +58,24 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"):
         "gpt-4-32k-0314",
         "gpt-4-0613",
         "gpt-4-32k-0613",
+        "gpt-4-1106-preview",
     }:
         tokens_per_message = 3
         tokens_per_name = 1
     elif model == "gpt-3.5-turbo-0301":
         tokens_per_message = 4  # every message follows <|start|>{role/name}\n{content}<|end|>\n
         tokens_per_name = -1  # if there's a name, the role is omitted
-    elif "gpt-3.5-turbo" in model:
+    elif "gpt-3.5-turbo" == model:
         print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.")
         return count_message_tokens(messages, model="gpt-3.5-turbo-0613")
-    elif "gpt-4" in model:
+    elif "gpt-4" == model:
         print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.")
         return count_message_tokens(messages, model="gpt-4-0613")
     else:
         raise NotImplementedError(
-            f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."""
+            f"num_tokens_from_messages() is not implemented for model {model}. "
+            f"See https://github.com/openai/openai-python/blob/main/chatml.md "
+            f"for information on how messages are converted to tokens."
         )
     num_tokens = 0
     for message in messages: