feat: merge v0.6.9

metagpt/document.py

@@ -20,6 +20,7 @@ from langchain.text_splitter import CharacterTextSplitter
 from pydantic import BaseModel, ConfigDict, Field
 from tqdm import tqdm
 
+from metagpt.logs import logger
 from metagpt.repo_parser import RepoParser
 
 
@@ -130,9 +131,12 @@ class IndexableDocument(Document):
         if isinstance(data, pd.DataFrame):
             validate_cols(content_col, data)
             return cls(data=data, content=str(data), content_col=content_col, meta_col=meta_col)
-        else:
+        try:
             content = data_path.read_text()
-            return cls(data=data, content=content, content_col=content_col, meta_col=meta_col)
+        except Exception as e:
+            logger.debug(f"Load {str(data_path)} error: {e}")
+            content = ""
+        return cls(data=data, content=content, content_col=content_col, meta_col=meta_col)
 
     def _get_docs_and_metadatas_by_df(self) -> (list, list):
         df = self.data

metagpt/memory/brain_memory.py

@@ -201,11 +201,14 @@ class BrainMemory(BaseModel):
 
     @staticmethod
     async def _openai_is_related(text1, text2, llm, **kwargs):
-        command = (
-            f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there "
-            "any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear."
+        context = f"## Paragraph 1\n{text2}\n---\n## Paragraph 2\n{text1}\n"
+        rsp = await llm.aask(
+            msg=context,
+            system_msgs=[
+                "You are a tool capable of determining whether two paragraphs are semantically related."
+                'Return "TRUE" if "Paragraph 1" is semantically relevant to "Paragraph 2", otherwise return "FALSE".'
+            ],
         )
-        rsp = await llm.aask(msg=command, system_msgs=[])
         result = True if "TRUE" in rsp else False
         p2 = text2.replace("\n", "")
         p1 = text1.replace("\n", "")
@@ -223,12 +226,16 @@ class BrainMemory(BaseModel):
 
     @staticmethod
     async def _openai_rewrite(sentence: str, context: str, llm):
-        command = (
-            f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly "
-            f"supplement or rewrite the following text in brief and clear:\n{sentence}"
+        prompt = f"## Context\n{context}\n---\n## Sentence\n{sentence}\n"
+        rsp = await llm.aask(
+            msg=prompt,
+            system_msgs=[
+                'You are a tool augmenting the "Sentence" with information from the "Context".',
+                "Do not supplement the context with information that is not present, especially regarding the subject and object.",
+                "Return the augmented sentence.",
+            ],
         )
-        rsp = await llm.aask(msg=command, system_msgs=[])
-        logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}\n")
+        logger.info(f"REWRITE:\nCommand: {prompt}\nRESULT: {rsp}\n")
         return rsp
 
     @staticmethod
@@ -293,14 +300,14 @@ class BrainMemory(BaseModel):
         """Generate text summary"""
         if len(text) < max_words:
             return text
+        system_msgs = [
+            "You are a tool for summarizing and abstracting text.",
+            f"Return the summarized text to less than {max_words} words.",
+        ]
         if keep_language:
-            command = f".Translate the above content into a summary of less than {max_words} words in language of the content strictly."
-        else:
-            command = f"Translate the above content into a summary of less than {max_words} words."
-        msg = text + "\n\n" + command
-        logger.debug(f"summary ask:{msg}")
-        response = await self.llm.aask(msg=msg, system_msgs=[])
-        logger.debug(f"summary rsp: {response}")
+            system_msgs.append("The generated summary should be in the same language as the original text.")
+        response = await self.llm.aask(msg=text, system_msgs=system_msgs)
+        logger.debug(f"{text}\nsummary rsp: {response}")
         return response
 
     @staticmethod

metagpt/provider/metagpt_api.py

@@ -5,11 +5,24 @@
 @File    : metagpt_api.py
 @Desc    : MetaGPT LLM provider.
 """
+from openai.types import CompletionUsage
+
 from metagpt.configs.llm_config import LLMType
 from metagpt.provider import OpenAILLM
 from metagpt.provider.llm_provider_registry import register_provider
+from metagpt.utils.exceptions import handle_exception
 
 
 @register_provider(LLMType.METAGPT)
 class MetaGPTLLM(OpenAILLM):
-    pass
+    def _calc_usage(self, messages: list[dict], rsp: str) -> CompletionUsage:
+        usage = CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
+
+        # The current billing is based on usage frequency. If there is a future billing logic based on the
+        # number of tokens, please refine the logic here accordingly.
+
+        return usage
+
+    @handle_exception
+    def _update_costs(self, usage: CompletionUsage):
+        pass

metagpt/utils/cost_manager.py

@@ -39,6 +39,8 @@ class CostManager(BaseModel):
         completion_tokens (int): The number of tokens used in the completion.
         model (str): The model used for the API call.
         """
+        if prompt_tokens + completion_tokens == 0:
+            return
         self.total_prompt_tokens += prompt_tokens
         self.total_completion_tokens += completion_tokens
         cost = (