Merge branch 'main' into reasoning

metagpt/actions/action.py

@@ -24,7 +24,6 @@ from metagpt.schema import (
     SerializationMixin,
     TestingContext,
 )
-from metagpt.utils.project_repo import ProjectRepo
 
 
 class Action(SerializationMixin, ContextMixin, BaseModel):
@@ -51,12 +50,6 @@ class Action(SerializationMixin, ContextMixin, BaseModel):
             data.llm = llm
         return data
 
-    @property
-    def repo(self) -> ProjectRepo:
-        if not self.context.repo:
-            self.context.repo = ProjectRepo(self.context.git_repo)
-        return self.context.repo
-
     @property
     def prompt_schema(self):
         return self.config.prompt_schema
@@ -112,10 +105,15 @@ class Action(SerializationMixin, ContextMixin, BaseModel):
         msgs = args[0]
         context = "## History Messages\n"
         context += "\n".join([f"{idx}: {i}" for idx, i in enumerate(reversed(msgs))])
-        return await self.node.fill(context=context, llm=self.llm)
+        return await self.node.fill(req=context, llm=self.llm)
 
     async def run(self, *args, **kwargs):
         """Run action"""
         if self.node:
             return await self._run_action_node(*args, **kwargs)
         raise NotImplementedError("The run method should be implemented in a subclass.")
+
+    def override_context(self):
+        """Set `private_context` and `context` to the same `Context` object."""
+        if not self.private_context:
+            self.private_context = self.context

metagpt/actions/action_node.py

@@ -18,7 +18,9 @@ from pydantic import BaseModel, Field, create_model, model_validator
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions.action_outcls_registry import register_action_outcls
-from metagpt.const import USE_CONFIG_TIMEOUT
+from metagpt.const import MARKDOWN_TITLE_PREFIX, USE_CONFIG_TIMEOUT
+from metagpt.exp_pool import exp_cache
+from metagpt.exp_pool.serializers import ActionNodeSerializer
 from metagpt.llm import BaseLLM
 from metagpt.logs import logger
 from metagpt.provider.postprocess.llm_output_postprocess import llm_output_postprocess
@@ -123,7 +125,7 @@ Follow format example's {prompt_schema} format, generate output and make sure it
 """
 
 
-def dict_to_markdown(d, prefix="- ", kv_sep="\n", postfix="\n"):
+def dict_to_markdown(d, prefix=MARKDOWN_TITLE_PREFIX, kv_sep="\n", postfix="\n"):
     markdown_str = ""
     for key, value in d.items():
         markdown_str += f"{prefix}{key}{kv_sep}{value}{postfix}"
@@ -591,9 +593,11 @@ class ActionNode:
 
         return extracted_data
 
+    @exp_cache(serializer=ActionNodeSerializer())
     async def fill(
         self,
-        context,
+        *,
+        req,
         llm,
         schema="json",
         mode="auto",
@@ -605,7 +609,7 @@ class ActionNode:
     ):
         """Fill the node(s) with mode.
 
-        :param context: Everything we should know when filling node.
+        :param req: Everything we should know when filling node.
         :param llm: Large Language Model with pre-defined system message.
         :param schema: json/markdown, determine example and output format.
          - raw: free form text
@@ -624,7 +628,7 @@ class ActionNode:
         :return: self
         """
         self.set_llm(llm)
-        self.set_context(context)
+        self.set_context(req)
         if self.schema:
             schema = self.schema
 

metagpt/actions/analyze_requirements.py

@@ -0,0 +1,76 @@
+from metagpt.actions import Action
+
+ANALYZE_REQUIREMENTS = """
+# Example
+{examples}
+
+----------------
+
+# Requirements
+{requirements}
+
+# Instructions
+{instructions}
+
+# Output Format
+{output_format}
+
+Follow the instructions and output format. Do not include any additional content.
+"""
+
+EXAMPLES = """
+Example 1
+Requirements: 
+创建一个贪吃蛇，只需要给出设计文档和代码
+Outputs:
+[User Restrictions] : 只需要给出设计文档和代码.
+[Language Restrictions] : The response, message and instruction must be in Chinese.
+[Programming Language] : HTML (*.html), CSS (*.css), and JavaScript (*.js)
+
+Example 2
+Requirements:
+Create 2048 game using Python. Do not write PRD.
+Outputs:
+[User Restrictions] : Do not write PRD.
+[Language Restrictions] : The response, message and instruction must be in English.
+[Programming Language] : Python
+
+Example 3
+Requirements:
+You must ignore create PRD and TRD. Help me write a schedule display program for the Paris Olympics. 
+Outputs:
+[User Restrictions] : You must ignore create PRD and TRD.
+[Language Restrictions] : The response, message and instruction must be in English.
+[Programming Language] : HTML (*.html), CSS (*.css), and JavaScript (*.js)
+"""
+
+INSTRUCTIONS = """
+You must output in the same language as the Requirements.
+First, This language should be consistent with the language used in the requirement description. determine the natural language you must respond in. If the requirements specify a special language, follow those instructions. The default language for responses is English.
+Second, extract the restrictions in the requirements, specifically the steps. Do not include detailed demand descriptions; focus only on the restrictions.
+Third, if the requirements is a software development, extract the program language. If no specific programming language is required, Use HTML (*.html), CSS (*.css), and JavaScript (*.js)
+
+Note:
+1. if there is not restrictions, requirements_restrictions must be ""
+2. if the requirements is a not software development, programming language must be ""
+"""
+
+OUTPUT_FORMAT = """
+[User Restrictions] : the restrictions in the requirements
+[Language Restrictions] :  The response, message and instruction must be in {{language}}
+[Programming Language] : Your program must use ...
+"""
+
+
+class AnalyzeRequirementsRestrictions(Action):
+    """Write a review for the given context."""
+
+    name: str = "AnalyzeRequirementsRestrictions"
+
+    async def run(self, requirements, isinstance=INSTRUCTIONS, output_format=OUTPUT_FORMAT):
+        """Analyze the constraints and the language used in the requirements."""
+        prompt = ANALYZE_REQUIREMENTS.format(
+            examples=EXAMPLES, requirements=requirements, instructions=isinstance, output_format=output_format
+        )
+        rsp = await self.llm.aask(prompt)
+        return rsp

metagpt/actions/debug_error.py

@@ -9,13 +9,15 @@
         2. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
 """
 import re
+from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 
 from metagpt.actions.action import Action
 from metagpt.logs import logger
 from metagpt.schema import RunCodeContext, RunCodeResult
 from metagpt.utils.common import CodeParser
+from metagpt.utils.project_repo import ProjectRepo
 
 PROMPT_TEMPLATE = """
 NOTICE
@@ -47,6 +49,8 @@ Now you should start rewriting the code:
 
 class DebugError(Action):
     i_context: RunCodeContext = Field(default_factory=RunCodeContext)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     async def run(self, *args, **kwargs) -> str:
         output_doc = await self.repo.test_outputs.get(filename=self.i_context.output_filename)
@@ -59,9 +63,7 @@ class DebugError(Action):
             return ""
 
         logger.info(f"Debug and rewrite {self.i_context.test_filename}")
-        code_doc = await self.repo.with_src_path(self.context.src_workspace).srcs.get(
-            filename=self.i_context.code_filename
-        )
+        code_doc = await self.repo.srcs.get(filename=self.i_context.code_filename)
         if not code_doc:
             return ""
         test_doc = await self.repo.tests.get(filename=self.i_context.test_filename)
@@ -70,6 +72,6 @@ class DebugError(Action):
         prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr)
 
         rsp = await self._aask(prompt)
-        code = CodeParser.parse_code(block="", text=rsp)
+        code = CodeParser.parse_code(text=rsp)
 
         return code

metagpt/actions/design_api.py

@@ -8,12 +8,15 @@
             1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
             2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality.
 @Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
+@Modified By: mashenquan, 2024/5/31. Implement Chapter 3 of RFC 236.
 """
 import json
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional, Union
 
-from metagpt.actions import Action, ActionOutput
+from pydantic import BaseModel, Field
+
+from metagpt.actions import Action
 from metagpt.actions.design_api_an import (
     DATA_STRUCTURES_AND_INTERFACES,
     DESIGN_API_NODE,
@@ -24,8 +27,18 @@ from metagpt.actions.design_api_an import (
 )
 from metagpt.const import DATA_API_DESIGN_FILE_REPO, SEQ_FLOW_FILE_REPO
 from metagpt.logs import logger
-from metagpt.schema import Document, Documents, Message
+from metagpt.schema import AIMessage, Document, Documents, Message
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import (
+    aread,
+    awrite,
+    rectify_pathname,
+    save_json_to_markdown,
+    to_markdown_code_block,
+)
 from metagpt.utils.mermaid import mermaid_to_file
+from metagpt.utils.project_repo import ProjectRepo
+from metagpt.utils.report import DocsReporter, GalleryReporter
 
 NEW_REQ_TEMPLATE = """
 ### Legacy Content
@@ -36,6 +49,7 @@ NEW_REQ_TEMPLATE = """
 """
 
 
+@register_tool(include_functions=["run"])
 class WriteDesign(Action):
     name: str = ""
     i_context: Optional[str] = None
@@ -44,21 +58,98 @@ class WriteDesign(Action):
         "data structures, library tables, processes, and paths. Please provide your design, feedback "
         "clearly and in detail."
     )
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
-    async def run(self, with_messages: Message, schema: str = None):
-        # Use `git status` to identify which PRD documents have been modified in the `docs/prd` directory.
-        changed_prds = self.repo.docs.prd.changed_files
-        # Use `git status` to identify which design documents in the `docs/system_designs` directory have undergone
-        # changes.
-        changed_system_designs = self.repo.docs.system_design.changed_files
+    async def run(
+        self,
+        with_messages: List[Message] = None,
+        *,
+        user_requirement: str = "",
+        prd_filename: str = "",
+        legacy_design_filename: str = "",
+        extra_info: str = "",
+        output_pathname: str = "",
+        **kwargs,
+    ) -> Union[AIMessage, str]:
+        """
+        Write a system design.
+
+        Args:
+            user_requirement (str): The user's requirements for the system design.
+            prd_filename (str, optional): The filename of the Product Requirement Document (PRD).
+            legacy_design_filename (str, optional): The filename of the legacy design document.
+            extra_info (str, optional): Additional information to be included in the system design.
+            output_pathname (str, optional): The output file path of the document.
+
+        Returns:
+            str: The file path of the generated system design.
+
+        Example:
+            # Write a new system design and save to the path name.
+            >>> user_requirement = "Write system design for a snake game"
+            >>> extra_info = "Your extra information"
+            >>> output_pathname = "snake_game/docs/system_design.json"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, output_pathname=output_pathname)
+            >>> print(result)
+            System Design filename: "/absolute/path/to/snake_game/docs/system_design.json"
+
+            # Rewrite an existing system design and save to the path name.
+            >>> user_requirement = "Write system design for a snake game, include new features such as a web UI"
+            >>> extra_info = "Your extra information"
+            >>> legacy_design_filename = "/absolute/path/to/snake_game/docs/system_design.json"
+            >>> output_pathname = "/absolute/path/to/snake_game/docs/system_design_new.json"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, output_pathname=output_pathname)
+            >>> print(result)
+            System Design filename: "/absolute/path/to/snake_game/docs/system_design_new.json"
+
+            # Write a new system design with the given PRD(Product Requirement Document) and save to the path name.
+            >>> user_requirement = "Write system design for a snake game based on the PRD at /absolute/path/to/snake_game/docs/prd.json"
+            >>> extra_info = "Your extra information"
+            >>> prd_filename = "/absolute/path/to/snake_game/docs/prd.json"
+            >>> output_pathname = "/absolute/path/to/snake_game/docs/sytem_design.json"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename, output_pathname=output_pathname)
+            >>> print(result)
+            System Design filename: "/absolute/path/to/snake_game/docs/sytem_design.json"
+
+            # Rewrite an existing system design with the given PRD(Product Requirement Document) and save to the path name.
+            >>> user_requirement = "Write system design for a snake game, include new features such as a web UI"
+            >>> extra_info = "Your extra information"
+            >>> prd_filename = "/absolute/path/to/snake_game/docs/prd.json"
+            >>> legacy_design_filename = "/absolute/path/to/snake_game/docs/system_design.json"
+            >>> output_pathname = "/absolute/path/to/snake_game/docs/system_design_new.json"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename, legacy_design_filename=legacy_design_filename, output_pathname=output_pathname)
+            >>> print(result)
+            System Design filename: "/absolute/path/to/snake_game/docs/system_design_new.json"
+        """
+        if not with_messages:
+            return await self._execute_api(
+                user_requirement=user_requirement,
+                prd_filename=prd_filename,
+                legacy_design_filename=legacy_design_filename,
+                extra_info=extra_info,
+                output_pathname=output_pathname,
+            )
+
+        self.input_args = with_messages[-1].instruct_content
+        self.repo = ProjectRepo(self.input_args.project_path)
+        changed_prds = self.input_args.changed_prd_filenames
+        changed_system_designs = [
+            str(self.repo.docs.system_design.workdir / i)
+            for i in list(self.repo.docs.system_design.changed_files.keys())
+        ]
 
         # For those PRDs and design documents that have undergone changes, regenerate the design content.
         changed_files = Documents()
-        for filename in changed_prds.keys():
+        for filename in changed_prds:
             doc = await self._update_system_design(filename=filename)
             changed_files.docs[filename] = doc
 
-        for filename in changed_system_designs.keys():
+        for filename in changed_system_designs:
             if filename in changed_files.docs:
                 continue
             doc = await self._update_system_design(filename=filename)
@@ -67,54 +158,122 @@ class WriteDesign(Action):
             logger.info("Nothing has changed.")
         # Wait until all files under `docs/system_designs/` are processed before sending the publish message,
         # leaving room for global optimization in subsequent steps.
-        return ActionOutput(content=changed_files.model_dump_json(), instruct_content=changed_files)
+        kvs = self.input_args.model_dump()
+        kvs["changed_system_design_filenames"] = [
+            str(self.repo.docs.system_design.workdir / i)
+            for i in list(self.repo.docs.system_design.changed_files.keys())
+        ]
+        return AIMessage(
+            content="Designing is complete. "
+            + "\n".join(
+                list(self.repo.docs.system_design.changed_files.keys())
+                + list(self.repo.resources.data_api_design.changed_files.keys())
+                + list(self.repo.resources.seq_flow.changed_files.keys())
+            ),
+            instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteDesignOutput"),
+            cause_by=self,
+        )
 
     async def _new_system_design(self, context):
-        node = await DESIGN_API_NODE.fill(context=context, llm=self.llm)
+        node = await DESIGN_API_NODE.fill(req=context, llm=self.llm, schema=self.prompt_schema)
         return node
 
     async def _merge(self, prd_doc, system_design_doc):
         context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content)
-        node = await REFINED_DESIGN_NODE.fill(context=context, llm=self.llm)
+        node = await REFINED_DESIGN_NODE.fill(req=context, llm=self.llm, schema=self.prompt_schema)
         system_design_doc.content = node.instruct_content.model_dump_json()
         return system_design_doc
 
     async def _update_system_design(self, filename) -> Document:
-        prd = await self.repo.docs.prd.get(filename)
-        old_system_design_doc = await self.repo.docs.system_design.get(filename)
-        if not old_system_design_doc:
-            system_design = await self._new_system_design(context=prd.content)
-            doc = await self.repo.docs.system_design.save(
-                filename=filename,
-                content=system_design.instruct_content.model_dump_json(),
-                dependencies={prd.root_relative_path},
-            )
-        else:
-            doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc)
-            await self.repo.docs.system_design.save_doc(doc=doc, dependencies={prd.root_relative_path})
-        await self._save_data_api_design(doc)
-        await self._save_seq_flow(doc)
-        await self.repo.resources.system_design.save_pdf(doc=doc)
+        root_relative_path = Path(filename).relative_to(self.repo.workdir)
+        prd = await Document.load(filename=filename, project_path=self.repo.workdir)
+        old_system_design_doc = await self.repo.docs.system_design.get(root_relative_path.name)
+        async with DocsReporter(enable_llm_stream=True) as reporter:
+            await reporter.async_report({"type": "design"}, "meta")
+            if not old_system_design_doc:
+                system_design = await self._new_system_design(context=prd.content)
+                doc = await self.repo.docs.system_design.save(
+                    filename=prd.filename,
+                    content=system_design.instruct_content.model_dump_json(),
+                    dependencies={prd.root_relative_path},
+                )
+            else:
+                doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc)
+                await self.repo.docs.system_design.save_doc(doc=doc, dependencies={prd.root_relative_path})
+            await self._save_data_api_design(doc)
+            await self._save_seq_flow(doc)
+            md = await self.repo.resources.system_design.save_pdf(doc=doc)
+            await reporter.async_report(self.repo.workdir / md.root_relative_path, "path")
         return doc
 
-    async def _save_data_api_design(self, design_doc):
+    async def _save_data_api_design(self, design_doc, output_filename: Path = None):
         m = json.loads(design_doc.content)
         data_api_design = m.get(DATA_STRUCTURES_AND_INTERFACES.key) or m.get(REFINED_DATA_STRUCTURES_AND_INTERFACES.key)
         if not data_api_design:
             return
-        pathname = self.repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("")
+        pathname = output_filename or self.repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(
+            design_doc.filename
+        ).with_suffix("")
         await self._save_mermaid_file(data_api_design, pathname)
         logger.info(f"Save class view to {str(pathname)}")
 
-    async def _save_seq_flow(self, design_doc):
+    async def _save_seq_flow(self, design_doc, output_filename: Path = None):
         m = json.loads(design_doc.content)
         seq_flow = m.get(PROGRAM_CALL_FLOW.key) or m.get(REFINED_PROGRAM_CALL_FLOW.key)
         if not seq_flow:
             return
-        pathname = self.repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("")
+        pathname = output_filename or self.repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(
+            design_doc.filename
+        ).with_suffix("")
         await self._save_mermaid_file(seq_flow, pathname)
         logger.info(f"Saving sequence flow to {str(pathname)}")
 
     async def _save_mermaid_file(self, data: str, pathname: Path):
         pathname.parent.mkdir(parents=True, exist_ok=True)
         await mermaid_to_file(self.config.mermaid.engine, data, pathname)
+        image_path = pathname.parent / f"{pathname.name}.svg"
+        if image_path.exists():
+            await GalleryReporter().async_report(image_path, "path")
+
+    async def _execute_api(
+        self,
+        user_requirement: str = "",
+        prd_filename: str = "",
+        legacy_design_filename: str = "",
+        extra_info: str = "",
+        output_pathname: str = "",
+    ) -> str:
+        prd_content = ""
+        if prd_filename:
+            prd_filename = rectify_pathname(path=prd_filename, default_filename="prd.json")
+            prd_content = await aread(filename=prd_filename)
+        context = "### User Requirements\n{user_requirement}\n### Extra_info\n{extra_info}\n### PRD\n{prd}\n".format(
+            user_requirement=to_markdown_code_block(user_requirement),
+            extra_info=to_markdown_code_block(extra_info),
+            prd=to_markdown_code_block(prd_content),
+        )
+        async with DocsReporter(enable_llm_stream=True) as reporter:
+            await reporter.async_report({"type": "design"}, "meta")
+            if not legacy_design_filename:
+                node = await self._new_system_design(context=context)
+                design = Document(content=node.instruct_content.model_dump_json())
+            else:
+                old_design_content = await aread(filename=legacy_design_filename)
+                design = await self._merge(
+                    prd_doc=Document(content=context), system_design_doc=Document(content=old_design_content)
+                )
+
+            if not output_pathname:
+                output_pathname = Path(output_pathname) / "docs" / "system_design.json"
+            elif not Path(output_pathname).is_absolute():
+                output_pathname = self.config.workspace.path / output_pathname
+            output_pathname = rectify_pathname(path=output_pathname, default_filename="system_design.json")
+            await awrite(filename=output_pathname, data=design.content)
+            output_filename = output_pathname.parent / f"{output_pathname.stem}-class-diagram"
+            await self._save_data_api_design(design_doc=design, output_filename=output_filename)
+            output_filename = output_pathname.parent / f"{output_pathname.stem}-sequence-diagram"
+            await self._save_seq_flow(design_doc=design, output_filename=output_filename)
+            md_output_filename = output_pathname.with_suffix(".md")
+            await save_json_to_markdown(content=design.content, output_filename=md_output_filename)
+            await reporter.async_report(md_output_filename, "path")
+        return f'System Design filename: "{str(output_pathname)}". \n The System Design has been completed.'

metagpt/actions/design_api_an.py

@@ -13,7 +13,7 @@ from metagpt.utils.mermaid import MMC1, MMC2
 IMPLEMENTATION_APPROACH = ActionNode(
     key="Implementation approach",
     expected_type=str,
-    instruction="Analyze the difficult points of the requirements, select the appropriate open-source framework",
+    instruction="Analyze the difficult points of the requirements, select the appropriate open-source framework.",
     example="We will ...",
 )
 
@@ -33,8 +33,8 @@ PROJECT_NAME = ActionNode(
 FILE_LIST = ActionNode(
     key="File list",
     expected_type=List[str],
-    instruction="Only need relative paths. ALWAYS write a main.py or app.py here",
-    example=["main.py", "game.py"],
+    instruction="Only need relative paths. Succinctly designate the correct entry file for your project based on the programming language: use main.js for JavaScript, main.py for Python, and so on for other languages.",
+    example=["a.js", "b.py", "c.css", "d.html"],
 )
 
 REFINED_FILE_LIST = ActionNode(

metagpt/actions/di/ask_review.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from typing import Tuple
 
 from metagpt.actions import Action
-from metagpt.logs import logger
+from metagpt.logs import get_human_input, logger
 from metagpt.schema import Message, Plan
 
 
@@ -50,7 +50,7 @@ class AskReview(Action):
             "Please type your review below:\n"
         )
 
-        rsp = input(prompt)
+        rsp = await get_human_input(prompt)
 
         if rsp.lower() in ReviewConst.EXIT_WORDS:
             exit()

metagpt/actions/di/execute_nb_code.py

@@ -13,9 +13,10 @@ from typing import Literal, Tuple
 
 import nbformat
 from nbclient import NotebookClient
-from nbclient.exceptions import CellTimeoutError, DeadKernelError
+from nbclient.exceptions import CellExecutionComplete, CellTimeoutError, DeadKernelError
+from nbclient.util import ensure_async
 from nbformat import NotebookNode
-from nbformat.v4 import new_code_cell, new_markdown_cell, new_output
+from nbformat.v4 import new_code_cell, new_markdown_cell, new_output, output_from_msg
 from rich.box import MINIMAL
 from rich.console import Console, Group
 from rich.live import Live
@@ -25,29 +26,79 @@ from rich.syntax import Syntax
 
 from metagpt.actions import Action
 from metagpt.logs import logger
+from metagpt.utils.report import NotebookReporter
+
+INSTALL_KEEPLEN = 500
+INI_CODE = """import warnings
+import logging
+
+root_logger = logging.getLogger()
+root_logger.setLevel(logging.ERROR)
+warnings.filterwarnings('ignore')"""
+
+
+class RealtimeOutputNotebookClient(NotebookClient):
+    """Realtime output of Notebook execution."""
+
+    def __init__(self, *args, notebook_reporter=None, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.notebook_reporter = notebook_reporter or NotebookReporter()
+
+    async def _async_poll_output_msg(self, parent_msg_id: str, cell: NotebookNode, cell_index: int) -> None:
+        """Implement a feature to enable sending messages."""
+        assert self.kc is not None
+        while True:
+            msg = await ensure_async(self.kc.iopub_channel.get_msg(timeout=None))
+            await self._send_msg(msg)
+
+            if msg["parent_header"].get("msg_id") == parent_msg_id:
+                try:
+                    # Will raise CellExecutionComplete when completed
+                    self.process_message(msg, cell, cell_index)
+                except CellExecutionComplete:
+                    return
+
+    async def _send_msg(self, msg: dict):
+        msg_type = msg.get("header", {}).get("msg_type")
+        if msg_type not in ["stream", "error", "execute_result"]:
+            return
+
+        await self.notebook_reporter.async_report(output_from_msg(msg), "content")
 
 
 class ExecuteNbCode(Action):
     """execute notebook code block, return result to llm, and display it."""
 
     nb: NotebookNode
-    nb_client: NotebookClient
+    nb_client: RealtimeOutputNotebookClient = None
     console: Console
     interaction: str
     timeout: int = 600
 
-    def __init__(
-        self,
-        nb=nbformat.v4.new_notebook(),
-        timeout=600,
-    ):
+    def __init__(self, nb=nbformat.v4.new_notebook(), timeout=600):
         super().__init__(
             nb=nb,
-            nb_client=NotebookClient(nb, timeout=timeout),
             timeout=timeout,
             console=Console(),
             interaction=("ipython" if self.is_ipython() else "terminal"),
         )
+        self.reporter = NotebookReporter()
+        self.set_nb_client()
+        self.init_called = False
+
+    async def init_code(self):
+        if not self.init_called:
+            await self.run(INI_CODE)
+            self.init_called = True
+
+    def set_nb_client(self):
+        self.nb_client = RealtimeOutputNotebookClient(
+            self.nb,
+            timeout=self.timeout,
+            resources={"metadata": {"path": self.config.workspace.path}},
+            notebook_reporter=self.reporter,
+            coalesce_streams=True,
+        )
 
     async def build(self):
         if self.nb_client.kc is None or not await self.nb_client.kc.is_alive():
@@ -82,7 +133,7 @@ class ExecuteNbCode(Action):
         # sleep 1s to wait for the kernel to be cleaned up completely
         await asyncio.sleep(1)
         await self.build()
-        self.nb_client = NotebookClient(self.nb, timeout=self.timeout)
+        self.set_nb_client()
 
     def add_code_cell(self, code: str):
         self.nb.cells.append(new_code_cell(source=code))
@@ -106,7 +157,7 @@ class ExecuteNbCode(Action):
         else:
             cell["outputs"].append(new_output(output_type="stream", name="stdout", text=str(output)))
 
-    def parse_outputs(self, outputs: list[str], keep_len: int = 2000) -> Tuple[bool, str]:
+    def parse_outputs(self, outputs: list[str], keep_len: int = 5000) -> Tuple[bool, str]:
         """Parses the outputs received from notebook execution."""
         assert isinstance(outputs, list)
         parsed_output, is_success = [], True
@@ -135,9 +186,12 @@ class ExecuteNbCode(Action):
                 is_success = False
 
             output_text = remove_escape_and_color_codes(output_text)
+            if is_success:
+                output_text = remove_log_and_warning_lines(output_text)
             # The useful information of the exception is at the end,
             # the useful information of normal output is at the begining.
-            output_text = output_text[:keep_len] if is_success else output_text[-keep_len:]
+            if "<!DOCTYPE html>" not in output_text:
+                output_text = output_text[:keep_len] if is_success else output_text[-keep_len:]
 
             parsed_output.append(output_text)
         return is_success, ",".join(parsed_output)
@@ -172,6 +226,8 @@ class ExecuteNbCode(Action):
         """set timeout for run code.
         returns the success or failure of the cell execution, and an optional error message.
         """
+        await self.reporter.async_report(cell, "content")
+
         try:
             await self.nb_client.async_execute_cell(cell, cell_index)
             return self.parse_outputs(self.nb.cells[-1].outputs)
@@ -193,29 +249,45 @@ class ExecuteNbCode(Action):
         """
         self._display(code, language)
 
-        if language == "python":
-            # add code to the notebook
-            self.add_code_cell(code=code)
+        async with self.reporter:
+            if language == "python":
+                # add code to the notebook
+                self.add_code_cell(code=code)
 
-            # build code executor
-            await self.build()
+                # build code executor
+                await self.build()
 
-            # run code
-            cell_index = len(self.nb.cells) - 1
-            success, outputs = await self.run_cell(self.nb.cells[-1], cell_index)
+                # run code
+                cell_index = len(self.nb.cells) - 1
+                success, outputs = await self.run_cell(self.nb.cells[-1], cell_index)
 
-            if "!pip" in code:
-                success = False
+                if "!pip" in code:
+                    success = False
+                    outputs = outputs[-INSTALL_KEEPLEN:]
+                elif "git clone" in code:
+                    outputs = outputs[:INSTALL_KEEPLEN] + "..." + outputs[-INSTALL_KEEPLEN:]
+
+            elif language == "markdown":
+                # add markdown content to markdown cell in a notebook.
+                self.add_markdown_cell(code)
+                # return True, beacuse there is no execution failure for markdown cell.
+                outputs, success = code, True
+            else:
+                raise ValueError(f"Only support for language: python, markdown, but got {language}, ")
+
+            file_path = self.config.workspace.path / "code.ipynb"
+            nbformat.write(self.nb, file_path)
+            await self.reporter.async_report(file_path, "path")
 
             return outputs, success
 
-        elif language == "markdown":
-            # add markdown content to markdown cell in a notebook.
-            self.add_markdown_cell(code)
-            # return True, beacuse there is no execution failure for markdown cell.
-            return code, True
-        else:
-            raise ValueError(f"Only support for language: python, markdown, but got {language}, ")
+
+def remove_log_and_warning_lines(input_str: str) -> str:
+    delete_lines = ["[warning]", "warning:", "[cv]", "[info]"]
+    result = "\n".join(
+        [line for line in input_str.split("\n") if not any(dl in line.lower() for dl in delete_lines)]
+    ).strip()
+    return result
 
 
 def remove_escape_and_color_codes(input_str: str):

metagpt/actions/di/run_command.py

@@ -0,0 +1,5 @@
+from metagpt.actions import Action
+
+
+class RunCommand(Action):
+    """A dummy RunCommand action used as a symbol only"""

metagpt/actions/di/write_analysis_code.py

@@ -40,6 +40,7 @@ class WriteAnalysisCode(Action):
         tool_info: str = "",
         working_memory: list[Message] = None,
         use_reflection: bool = False,
+        memory: list[Message] = None,
         **kwargs,
     ) -> str:
         structual_prompt = STRUCTUAL_PROMPT.format(
@@ -49,14 +50,15 @@ class WriteAnalysisCode(Action):
         )
 
         working_memory = working_memory or []
-        context = self.llm.format_msg([Message(content=structual_prompt, role="user")] + working_memory)
+        memory = memory or []
+        context = self.llm.format_msg(memory + [Message(content=structual_prompt, role="user")] + working_memory)
 
         # LLM call
         if use_reflection:
             code = await self._debug_with_reflection(context=context, working_memory=working_memory)
         else:
             rsp = await self.llm.aask(context, system_msgs=[INTERPRETER_SYSTEM_MSG], **kwargs)
-            code = CodeParser.parse_code(block=None, text=rsp)
+            code = CodeParser.parse_code(text=rsp, lang="python")
 
         return code
 
@@ -68,5 +70,5 @@ class CheckData(Action):
         code_written = "\n\n".join(code_written)
         prompt = CHECK_DATA_PROMPT.format(code_written=code_written)
         rsp = await self._aask(prompt)
-        code = CodeParser.parse_code(block=None, text=rsp)
+        code = CodeParser.parse_code(text=rsp)
         return code

metagpt/actions/di/write_plan.py

@@ -16,38 +16,38 @@ from metagpt.schema import Message, Plan, Task
 from metagpt.strategy.task_type import TaskType
 from metagpt.utils.common import CodeParser
 
+PROMPT_TEMPLATE: str = """
+# Context:
+{context}
+# Available Task Types:
+{task_type_desc}
+# Task:
+Based on the context, write a plan or modify an existing plan of what you should do to achieve the goal. A plan consists of one to {max_tasks} tasks.
+If you are modifying an existing plan, carefully follow the instruction, don't make unnecessary changes. Give the whole plan unless instructed to modify only one task of the plan.
+If you encounter errors on the current task, revise and output the current single task only.
+Output a list of jsons following the format:
+```json
+[
+    {{
+        "task_id": str = "unique identifier for a task in plan, can be an ordinal",
+        "dependent_task_ids": list[str] = "ids of tasks prerequisite to this task",
+        "instruction": "what you should do in this task, one short phrase or sentence.",
+        "task_type": "type of this task, should be one of Available Task Types.",
+    }},
+    ...
+]
+```
+"""
+
 
 class WritePlan(Action):
-    PROMPT_TEMPLATE: str = """
-    # Context:
-    {context}
-    # Available Task Types:
-    {task_type_desc}
-    # Task:
-    Based on the context, write a plan or modify an existing plan of what you should do to achieve the goal. A plan consists of one to {max_tasks} tasks.
-    If you are modifying an existing plan, carefully follow the instruction, don't make unnecessary changes. Give the whole plan unless instructed to modify only one task of the plan.
-    If you encounter errors on the current task, revise and output the current single task only.
-    Output a list of jsons following the format:
-    ```json
-    [
-        {{
-            "task_id": str = "unique identifier for a task in plan, can be an ordinal",
-            "dependent_task_ids": list[str] = "ids of tasks prerequisite to this task",
-            "instruction": "what you should do in this task, one short phrase or sentence",
-            "task_type": "type of this task, should be one of Available Task Types",
-        }},
-        ...
-    ]
-    ```
-    """
-
     async def run(self, context: list[Message], max_tasks: int = 5) -> str:
         task_type_desc = "\n".join([f"- **{tt.type_name}**: {tt.value.desc}" for tt in TaskType])
-        prompt = self.PROMPT_TEMPLATE.format(
+        prompt = PROMPT_TEMPLATE.format(
             context="\n".join([str(ct) for ct in context]), max_tasks=max_tasks, task_type_desc=task_type_desc
         )
         rsp = await self._aask(prompt)
-        rsp = CodeParser.parse_code(block=None, text=rsp)
+        rsp = CodeParser.parse_code(text=rsp)
         return rsp
 
 
@@ -65,10 +65,14 @@ def update_plan_from_rsp(rsp: str, current_plan: Plan):
         # handle a single task
         if current_plan.has_task_id(tasks[0].task_id):
             # replace an existing task
-            current_plan.replace_task(tasks[0])
+            current_plan.replace_task(
+                tasks[0].task_id, tasks[0].dependent_task_ids, tasks[0].instruction, tasks[0].assignee
+            )
         else:
             # append one task
-            current_plan.append_task(tasks[0])
+            current_plan.append_task(
+                tasks[0].task_id, tasks[0].dependent_task_ids, tasks[0].instruction, tasks[0].assignee
+            )
 
     else:
         # add tasks in general

metagpt/actions/extract_readme.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Module Description: This script defines the LearnReadMe class, which is an action to learn from the contents of
+    a README.md file.
+Author: mashenquan
+Date: 2024-3-20
+"""
+from pathlib import Path
+from typing import Optional
+
+from pydantic import Field
+
+from metagpt.actions import Action
+from metagpt.const import GRAPH_REPO_FILE_REPO
+from metagpt.schema import Message
+from metagpt.utils.common import aread
+from metagpt.utils.di_graph_repository import DiGraphRepository
+from metagpt.utils.graph_repository import GraphKeyword, GraphRepository
+
+
+class ExtractReadMe(Action):
+    """
+    An action to extract summary, installation, configuration, usages from the contents of a README.md file.
+
+    Attributes:
+        graph_db (Optional[GraphRepository]): A graph database repository.
+        install_to_path (Optional[str]): The path where the repository to install to.
+    """
+
+    graph_db: Optional[GraphRepository] = None
+    install_to_path: Optional[str] = Field(default="/TO/PATH")
+    _readme: Optional[str] = None
+    _filename: Optional[str] = None
+
+    async def run(self, with_messages=None, **kwargs):
+        """
+        Implementation of `Action`'s `run` method.
+
+        Args:
+            with_messages (Optional[Type]): An optional argument specifying messages to react to.
+        """
+        graph_repo_pathname = self.context.git_repo.workdir / GRAPH_REPO_FILE_REPO / self.context.git_repo.workdir.name
+        self.graph_db = await DiGraphRepository.load_from(str(graph_repo_pathname.with_suffix(".json")))
+        summary = await self._summarize()
+        await self.graph_db.insert(subject=self._filename, predicate=GraphKeyword.HAS_SUMMARY, object_=summary)
+        install = await self._extract_install()
+        await self.graph_db.insert(subject=self._filename, predicate=GraphKeyword.HAS_INSTALL, object_=install)
+        conf = await self._extract_configuration()
+        await self.graph_db.insert(subject=self._filename, predicate=GraphKeyword.HAS_CONFIG, object_=conf)
+        usage = await self._extract_usage()
+        await self.graph_db.insert(subject=self._filename, predicate=GraphKeyword.HAS_USAGE, object_=usage)
+
+        await self.graph_db.save()
+
+        return Message(content="", cause_by=self)
+
+    async def _summarize(self) -> str:
+        readme = await self._get()
+        summary = await self.llm.aask(
+            readme,
+            system_msgs=[
+                "You are a tool can summarize git repository README.md file.",
+                "Return the summary about what is the repository.",
+            ],
+            stream=False,
+        )
+        return summary
+
+    async def _extract_install(self) -> str:
+        await self._get()
+        install = await self.llm.aask(
+            self._readme,
+            system_msgs=[
+                "You are a tool can install git repository according to README.md file.",
+                "Return a bash code block of markdown including:\n"
+                f"1. git clone the repository to the directory `{self.install_to_path}`;\n"
+                f"2. cd `{self.install_to_path}`;\n"
+                f"3. install the repository.",
+            ],
+            stream=False,
+        )
+        return install
+
+    async def _extract_configuration(self) -> str:
+        await self._get()
+        configuration = await self.llm.aask(
+            self._readme,
+            system_msgs=[
+                "You are a tool can configure git repository according to README.md file.",
+                "Return a bash code block of markdown object to configure the repository if necessary, otherwise return"
+                " a empty bash code block of markdown object",
+            ],
+            stream=False,
+        )
+        return configuration
+
+    async def _extract_usage(self) -> str:
+        await self._get()
+        usage = await self.llm.aask(
+            self._readme,
+            system_msgs=[
+                "You are a tool can summarize all usages of git repository according to README.md file.",
+                "Return a list of code block of markdown objects to demonstrates the usage of the repository.",
+            ],
+            stream=False,
+        )
+        return usage
+
+    async def _get(self) -> str:
+        if self._readme is not None:
+            return self._readme
+        root = Path(self.i_context).resolve()
+        filename = None
+        for file_path in root.iterdir():
+            if file_path.is_file() and file_path.stem == "README":
+                filename = file_path
+                break
+        if not filename:
+            return ""
+        self._readme = await aread(filename=filename, encoding="utf-8")
+        self._filename = str(filename)
+        return self._readme

metagpt/actions/generate_questions.py

@@ -22,4 +22,4 @@ class GenerateQuestions(Action):
     name: str = "GenerateQuestions"
 
     async def run(self, context) -> ActionNode:
-        return await QUESTIONS.fill(context=context, llm=self.llm)
+        return await QUESTIONS.fill(req=context, llm=self.llm)

metagpt/actions/import_repo.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+
+This script defines an action to import a Git repository into the MetaGPT project format, enabling incremental
+ appending of requirements.
+The MetaGPT project format encompasses a structured representation of project data compatible with MetaGPT's
+ capabilities, facilitating the integration of Git repositories into MetaGPT workflows while allowing for the gradual
+ addition of requirements.
+
+"""
+import json
+import re
+from pathlib import Path
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+from metagpt.actions import Action
+from metagpt.actions.extract_readme import ExtractReadMe
+from metagpt.actions.rebuild_class_view import RebuildClassView
+from metagpt.actions.rebuild_sequence_view import RebuildSequenceView
+from metagpt.const import GRAPH_REPO_FILE_REPO
+from metagpt.logs import logger
+from metagpt.schema import Message
+from metagpt.tools.libs.git import git_clone
+from metagpt.utils.common import (
+    aread,
+    awrite,
+    list_files,
+    parse_json_code_block,
+    split_namespace,
+)
+from metagpt.utils.di_graph_repository import DiGraphRepository
+from metagpt.utils.file_repository import FileRepository
+from metagpt.utils.git_repository import GitRepository
+from metagpt.utils.graph_repository import GraphKeyword, GraphRepository
+from metagpt.utils.project_repo import ProjectRepo
+
+
+class ImportRepo(Action):
+    """
+    An action to import a Git repository into a graph database and create related artifacts.
+
+    Attributes:
+        repo_path (str): The URL of the Git repository to import.
+        graph_db (Optional[GraphRepository]): The output graph database of the Git repository.
+        rid (str): The output requirement ID.
+    """
+
+    repo_path: str  # input, git repo url.
+    graph_db: Optional[GraphRepository] = None  # output. graph db of the git repository
+    rid: str = ""  # output, requirement ID.
+
+    async def run(self, with_messages: List[Message] = None, **kwargs) -> Message:
+        """
+        Runs the import process for the Git repository.
+
+        Args:
+            with_messages (List[Message], optional): Additional messages to include.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            Message: A message indicating the completion of the import process.
+        """
+        await self._create_repo()
+        await self._create_prd()
+        await self._create_system_design()
+        self.context.git_repo.archive(comments="Import")
+
+    async def _create_repo(self):
+        path = await git_clone(url=self.repo_path, output_dir=self.config.workspace.path)
+        self.repo_path = str(path)
+        self.config.project_path = path
+        self.context.git_repo = GitRepository(local_path=path, auto_init=True)
+        self.context.repo = ProjectRepo(self.context.git_repo)
+        self.context.src_workspace = await self._guess_src_workspace()
+        await awrite(
+            filename=self.context.repo.workdir / ".src_workspace",
+            data=str(self.context.src_workspace.relative_to(self.context.repo.workdir)),
+        )
+
+    async def _create_prd(self):
+        action = ExtractReadMe(i_context=str(self.context.repo.workdir), context=self.context)
+        await action.run()
+        graph_repo_pathname = self.context.git_repo.workdir / GRAPH_REPO_FILE_REPO / self.context.git_repo.workdir.name
+        self.graph_db = await DiGraphRepository.load_from(str(graph_repo_pathname.with_suffix(".json")))
+        rows = await self.graph_db.select(predicate=GraphKeyword.HAS_SUMMARY)
+        prd = {"Project Name": self.context.repo.workdir.name}
+        for r in rows:
+            if Path(r.subject).stem == "README":
+                prd["Original Requirements"] = r.object_
+                break
+        self.rid = FileRepository.new_filename()
+        await self.repo.docs.prd.save(filename=self.rid + ".json", content=json.dumps(prd))
+
+    async def _create_system_design(self):
+        action = RebuildClassView(
+            name="ReverseEngineering", i_context=str(self.context.src_workspace), context=self.context
+        )
+        await action.run()
+        rows = await action.graph_db.select(predicate="hasMermaidClassDiagramFile")
+        class_view_filename = rows[0].object_
+        logger.info(f"class view:{class_view_filename}")
+
+        rows = await action.graph_db.select(predicate=GraphKeyword.HAS_PAGE_INFO)
+        tag = "__name__:__main__"
+        entries = []
+        src_workspace = self.context.src_workspace.relative_to(self.context.repo.workdir)
+        for r in rows:
+            if tag in r.subject:
+                path = split_namespace(r.subject)[0]
+            elif tag in r.object_:
+                path = split_namespace(r.object_)[0]
+            else:
+                continue
+            if Path(path).is_relative_to(src_workspace):
+                entries.append(Path(path))
+        main_entry = await self._guess_main_entry(entries)
+        full_path = RebuildSequenceView.get_full_filename(self.context.repo.workdir, main_entry)
+        action = RebuildSequenceView(context=self.context, i_context=str(full_path))
+        try:
+            await action.run()
+        except Exception as e:
+            logger.warning(f"{e}, use the last successful version.")
+        files = list_files(self.context.repo.resources.data_api_design.workdir)
+        pattern = re.compile(r"[^a-zA-Z0-9]")
+        name = re.sub(pattern, "_", str(main_entry))
+        filename = Path(name).with_suffix(".sequence_diagram.mmd")
+        postfix = str(filename)
+        sequence_files = [i for i in files if postfix in str(i)]
+        content = await aread(filename=sequence_files[0])
+        await self.context.repo.resources.data_api_design.save(
+            filename=self.repo.workdir.stem + ".sequence_diagram.mmd", content=content
+        )
+        await self._save_system_design()
+
+    async def _save_system_design(self):
+        class_view = await self.context.repo.resources.data_api_design.get(
+            filename=self.repo.workdir.stem + ".class_diagram.mmd"
+        )
+        sequence_view = await self.context.repo.resources.data_api_design.get(
+            filename=self.repo.workdir.stem + ".sequence_diagram.mmd"
+        )
+        file_list = self.context.git_repo.get_files(relative_path=".", root_relative_path=self.context.src_workspace)
+        data = {
+            "Data structures and interfaces": class_view.content,
+            "Program call flow": sequence_view.content,
+            "File list": [str(i) for i in file_list],
+        }
+        await self.context.repo.docs.system_design.save(filename=self.rid + ".json", content=json.dumps(data))
+
+    async def _guess_src_workspace(self) -> Path:
+        files = list_files(self.context.repo.workdir)
+        dirs = [i.parent for i in files if i.name == "__init__.py"]
+        distinct = set()
+        for i in dirs:
+            done = False
+            for j in distinct:
+                if i.is_relative_to(j):
+                    done = True
+                    break
+                if j.is_relative_to(i):
+                    break
+            if not done:
+                distinct = {j for j in distinct if not j.is_relative_to(i)}
+                distinct.add(i)
+        if len(distinct) == 1:
+            return list(distinct)[0]
+        prompt = "\n".join([f"- {str(i)}" for i in distinct])
+        rsp = await self.llm.aask(
+            prompt,
+            system_msgs=[
+                "You are a tool to choose the source code path from a list of paths based on the directory name.",
+                "You should identify the source code path among paths such as unit test path, examples path, etc.",
+                "Return a markdown JSON object containing:\n"
+                '- a "src" field containing the source code path;\n'
+                '- a "reason" field containing explaining why other paths is not the source code path\n',
+            ],
+        )
+        logger.debug(rsp)
+        json_blocks = parse_json_code_block(rsp)
+
+        class Data(BaseModel):
+            src: str
+            reason: str
+
+        data = Data.model_validate_json(json_blocks[0])
+        logger.info(f"src_workspace: {data.src}")
+        return Path(data.src)
+
+    async def _guess_main_entry(self, entries: List[Path]) -> Path:
+        if len(entries) == 1:
+            return entries[0]
+
+        file_list = "## File List\n"
+        file_list += "\n".join([f"- {i}" for i in entries])
+
+        rows = await self.graph_db.select(predicate=GraphKeyword.HAS_USAGE)
+        usage = "## Usage\n"
+        for r in rows:
+            if Path(r.subject).stem == "README":
+                usage += r.object_
+
+        prompt = file_list + "\n---\n" + usage
+        rsp = await self.llm.aask(
+            prompt,
+            system_msgs=[
+                'You are a tool to choose the source file path from "File List" which is used in "Usage".',
+                'You choose the source file path based on the name of file and the class name and package name used in "Usage".',
+                "Return a markdown JSON object containing:\n"
+                '- a "filename" field containing the chosen source file path from "File List" which is used in "Usage";\n'
+                '- a "reason" field explaining why.',
+            ],
+            stream=False,
+        )
+        logger.debug(rsp)
+        json_blocks = parse_json_code_block(rsp)
+
+        class Data(BaseModel):
+            filename: str
+            reason: str
+
+        data = Data.model_validate_json(json_blocks[0])
+        logger.info(f"main: {data.filename}")
+        return Path(data.filename)

metagpt/actions/prepare_documents.py

@@ -9,12 +9,14 @@
 """
 import shutil
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
-from metagpt.actions import Action, ActionOutput
+from metagpt.actions import Action, UserRequirement
 from metagpt.const import REQUIREMENT_FILENAME
+from metagpt.logs import logger
+from metagpt.schema import AIMessage
+from metagpt.utils.common import any_to_str
 from metagpt.utils.file_repository import FileRepository
-from metagpt.utils.git_repository import GitRepository
 from metagpt.utils.project_repo import ProjectRepo
 
 
@@ -23,12 +25,19 @@ class PrepareDocuments(Action):
 
     name: str = "PrepareDocuments"
     i_context: Optional[str] = None
+    key_descriptions: Optional[Dict[str, str]] = None
+    send_to: str
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        if not self.key_descriptions:
+            self.key_descriptions = {"project_path": 'the project path if exists in "Original Requirement"'}
 
     @property
     def config(self):
         return self.context.config
 
-    def _init_repo(self):
+    def _init_repo(self) -> ProjectRepo:
         """Initialize the Git environment."""
         if not self.config.project_path:
             name = self.config.project_name or FileRepository.new_filename()
@@ -37,16 +46,45 @@ class PrepareDocuments(Action):
             path = Path(self.config.project_path)
         if path.exists() and not self.config.inc:
             shutil.rmtree(path)
-        self.config.project_path = path
-        self.context.git_repo = GitRepository(local_path=path, auto_init=True)
-        self.context.repo = ProjectRepo(self.context.git_repo)
+        self.context.kwargs.project_path = path
+        self.context.kwargs.inc = self.config.inc
+        return ProjectRepo(path)
 
     async def run(self, with_messages, **kwargs):
         """Create and initialize the workspace folder, initialize the Git environment."""
-        self._init_repo()
+        user_requirements = [i for i in with_messages if i.cause_by == any_to_str(UserRequirement)]
+        if not self.config.project_path and user_requirements and self.key_descriptions:
+            args = await user_requirements[0].parse_resources(llm=self.llm, key_descriptions=self.key_descriptions)
+            for k, v in args.items():
+                if not v or k in ["resources", "reason"]:
+                    continue
+                self.context.kwargs.set(k, v)
+                logger.info(f"{k}={v}")
+            if self.context.kwargs.project_path:
+                self.config.update_via_cli(
+                    project_path=self.context.kwargs.project_path,
+                    project_name="",
+                    inc=False,
+                    reqa_file=self.context.kwargs.reqa_file or "",
+                    max_auto_summarize_code=0,
+                )
+
+        repo = self._init_repo()
 
         # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`.
-        doc = await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
+        await repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
         # Send a Message notification to the WritePRD action, instructing it to process requirements using
         # `docs/requirement.txt` and `docs/prd/`.
-        return ActionOutput(content=doc.content, instruct_content=doc)
+        return AIMessage(
+            content="",
+            instruct_content=AIMessage.create_instruct_value(
+                kvs={
+                    "project_path": str(repo.workdir),
+                    "requirements_filename": str(repo.docs.workdir / REQUIREMENT_FILENAME),
+                    "prd_filenames": [str(repo.docs.prd.workdir / i) for i in repo.docs.prd.all_files],
+                },
+                class_name="PrepareDocumentsOutput",
+            ),
+            cause_by=self,
+            send_to=self.send_to,
+        )

metagpt/actions/prepare_interview.py

@@ -22,4 +22,4 @@ class PrepareInterview(Action):
     name: str = "PrepareInterview"
 
     async def run(self, context):
-        return await QUESTIONS.fill(context=context, llm=self.llm)
+        return await QUESTIONS.fill(req=context, llm=self.llm)

metagpt/actions/project_management.py

@@ -8,17 +8,30 @@
         1. Divide the context into three components: legacy code, unit test code, and console log.
         2. Move the document storage operations related to WritePRD from the save operation of WriteDesign.
         3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality.
+@Modified By: mashenquan, 2024/5/31. Implement Chapter 3 of RFC 236.
 """
 
 import json
-from typing import Optional
+from pathlib import Path
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, Field
 
 from metagpt.actions.action import Action
-from metagpt.actions.action_output import ActionOutput
 from metagpt.actions.project_management_an import PM_NODE, REFINED_PM_NODE
 from metagpt.const import PACKAGE_REQUIREMENTS_FILENAME
 from metagpt.logs import logger
-from metagpt.schema import Document, Documents
+from metagpt.schema import AIMessage, Document, Documents, Message
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import (
+    aread,
+    awrite,
+    rectify_pathname,
+    save_json_to_markdown,
+    to_markdown_code_block,
+)
+from metagpt.utils.project_repo import ProjectRepo
+from metagpt.utils.report import DocsReporter
 
 NEW_REQ_TEMPLATE = """
 ### Legacy Content
@@ -29,19 +42,67 @@ NEW_REQ_TEMPLATE = """
 """
 
 
+@register_tool(include_functions=["run"])
 class WriteTasks(Action):
     name: str = "CreateTasks"
     i_context: Optional[str] = None
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
-    async def run(self, with_messages):
-        changed_system_designs = self.repo.docs.system_design.changed_files
-        changed_tasks = self.repo.docs.task.changed_files
+    async def run(
+        self,
+        with_messages: List[Message] = None,
+        *,
+        user_requirement: str = "",
+        design_filename: str = "",
+        output_pathname: str = "",
+        **kwargs,
+    ) -> Union[AIMessage, str]:
+        """
+        Write a project schedule given a project system design file.
+
+        Args:
+            user_requirement (str, optional): A string specifying the user's requirements. Defaults to an empty string.
+            design_filename (str): The output file path of the document. Defaults to an empty string.
+            output_pathname (str, optional): The output path name of file that the project schedule should be saved to.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            str: Path to the generated project schedule.
+
+        Example:
+            # Write a project schedule with a given system design.
+            >>> design_filename = "/absolute/path/to/snake_game/docs/system_design.json"
+            >>> output_pathname = "/absolute/path/to/snake_game/docs/project_schedule.json"
+            >>> user_requirement = "Write project schedule for a snake game following these requirements:..."
+            >>> action = WriteTasks()
+            >>> result = await action.run(user_requirement=user_requirement, design_filename=design_filename, output_pathname=output_pathname)
+            >>> print(result)
+            The project schedule is at /absolute/path/to/snake_game/docs/project_schedule.json
+
+            # Write a project schedule with a user requirement.
+            >>> user_requirement = "Write project schedule for a snake game following these requirements: ..."
+            >>> output_pathname = "/absolute/path/to/snake_game/docs/project_schedule.json"
+            >>> action = WriteTasks()
+            >>> result = await action.run(user_requirement=user_requirement, output_pathname=output_pathname)
+            >>> print(result)
+            The project schedule is at /absolute/path/to/snake_game/docs/project_schedule.json
+        """
+        if not with_messages:
+            return await self._execute_api(
+                user_requirement=user_requirement, design_filename=design_filename, output_pathname=output_pathname
+            )
+
+        self.input_args = with_messages[-1].instruct_content
+        self.repo = ProjectRepo(self.input_args.project_path)
+        changed_system_designs = self.input_args.changed_system_design_filenames
+        changed_tasks = [str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys())]
         change_files = Documents()
         # Rewrite the system designs that have undergone changes based on the git head diff under
         # `docs/system_designs/`.
         for filename in changed_system_designs:
             task_doc = await self._update_tasks(filename=filename)
-            change_files.docs[filename] = task_doc
+            change_files.docs[str(self.repo.docs.task.workdir / task_doc.filename)] = task_doc
 
         # Rewrite the task files that have undergone changes based on the git head diff under `docs/tasks/`.
         for filename in changed_tasks:
@@ -54,31 +115,50 @@ class WriteTasks(Action):
             logger.info("Nothing has changed.")
         # Wait until all files under `docs/tasks/` are processed before sending the publish_message, leaving room for
         # global optimization in subsequent steps.
-        return ActionOutput(content=change_files.model_dump_json(), instruct_content=change_files)
+        kvs = self.input_args.model_dump()
+        kvs["changed_task_filenames"] = [
+            str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys())
+        ]
+        kvs["python_package_dependency_filename"] = str(self.repo.workdir / PACKAGE_REQUIREMENTS_FILENAME)
+        return AIMessage(
+            content="WBS is completed. "
+            + "\n".join(
+                [PACKAGE_REQUIREMENTS_FILENAME]
+                + list(self.repo.docs.task.changed_files.keys())
+                + list(self.repo.resources.api_spec_and_task.changed_files.keys())
+            ),
+            instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteTaskOutput"),
+            cause_by=self,
+        )
 
     async def _update_tasks(self, filename):
-        system_design_doc = await self.repo.docs.system_design.get(filename)
-        task_doc = await self.repo.docs.task.get(filename)
-        if task_doc:
-            task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc)
-            await self.repo.docs.task.save_doc(doc=task_doc, dependencies={system_design_doc.root_relative_path})
-        else:
-            rsp = await self._run_new_tasks(context=system_design_doc.content)
-            task_doc = await self.repo.docs.task.save(
-                filename=filename,
-                content=rsp.instruct_content.model_dump_json(),
-                dependencies={system_design_doc.root_relative_path},
-            )
-        await self._update_requirements(task_doc)
+        root_relative_path = Path(filename).relative_to(self.repo.workdir)
+        system_design_doc = await Document.load(filename=filename, project_path=self.repo.workdir)
+        task_doc = await self.repo.docs.task.get(root_relative_path.name)
+        async with DocsReporter(enable_llm_stream=True) as reporter:
+            await reporter.async_report({"type": "task"}, "meta")
+            if task_doc:
+                task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc)
+                await self.repo.docs.task.save_doc(doc=task_doc, dependencies={system_design_doc.root_relative_path})
+            else:
+                rsp = await self._run_new_tasks(context=system_design_doc.content)
+                task_doc = await self.repo.docs.task.save(
+                    filename=system_design_doc.filename,
+                    content=rsp.instruct_content.model_dump_json(),
+                    dependencies={system_design_doc.root_relative_path},
+                )
+            await self._update_requirements(task_doc)
+            md = await self.repo.resources.api_spec_and_task.save_pdf(doc=task_doc)
+            await reporter.async_report(self.repo.workdir / md.root_relative_path, "path")
         return task_doc
 
-    async def _run_new_tasks(self, context):
-        node = await PM_NODE.fill(context, self.llm, schema=self.prompt_schema)
+    async def _run_new_tasks(self, context: str):
+        node = await PM_NODE.fill(req=context, llm=self.llm, schema=self.prompt_schema)
         return node
 
     async def _merge(self, system_design_doc, task_doc) -> Document:
         context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_task=task_doc.content)
-        node = await REFINED_PM_NODE.fill(context, self.llm, schema=self.prompt_schema)
+        node = await REFINED_PM_NODE.fill(req=context, llm=self.llm, schema=self.prompt_schema)
         task_doc.content = node.instruct_content.model_dump_json()
         return task_doc
 
@@ -94,3 +174,28 @@ class WriteTasks(Action):
                 continue
             packages.add(pkg)
         await self.repo.save(filename=PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages))
+
+    async def _execute_api(
+        self, user_requirement: str = "", design_filename: str = "", output_pathname: str = ""
+    ) -> str:
+        context = to_markdown_code_block(user_requirement)
+        if design_filename:
+            design_filename = rectify_pathname(path=design_filename, default_filename="system_design.md")
+            content = await aread(filename=design_filename)
+            context += to_markdown_code_block(content)
+
+        async with DocsReporter(enable_llm_stream=True) as reporter:
+            await reporter.async_report({"type": "task"}, "meta")
+            node = await self._run_new_tasks(context)
+            file_content = node.instruct_content.model_dump_json()
+
+            if not output_pathname:
+                output_pathname = Path(output_pathname) / "docs" / "project_schedule.json"
+            elif not Path(output_pathname).is_absolute():
+                output_pathname = self.config.workspace.path / output_pathname
+            output_pathname = rectify_pathname(path=output_pathname, default_filename="project_schedule.json")
+            await awrite(filename=output_pathname, data=file_content)
+            md_output_filename = output_pathname.with_suffix(".md")
+            await save_json_to_markdown(content=file_content, output_filename=md_output_filename)
+            await reporter.async_report(md_output_filename, "path")
+        return f'Project Schedule filename: "{str(output_pathname)}"'

metagpt/actions/project_management_an.py

@@ -12,7 +12,7 @@ from metagpt.actions.action_node import ActionNode
 REQUIRED_PACKAGES = ActionNode(
     key="Required packages",
     expected_type=Optional[List[str]],
-    instruction="Provide required third-party packages in requirements.txt format.",
+    instruction="Provide required packages The response language should correspond to the context and requirements.",
     example=["flask==1.1.2", "bcrypt==3.2.0"],
 )
 
@@ -27,7 +27,9 @@ LOGIC_ANALYSIS = ActionNode(
     key="Logic Analysis",
     expected_type=List[List[str]],
     instruction="Provide a list of files with the classes/methods/functions to be implemented, "
-    "including dependency analysis and imports.",
+    "including dependency analysis and imports."
+    "Ensure consistency between System Design and Logic Analysis; the files must match exactly. "
+    "If the file is written in Vue or React, use Tailwind CSS for styling.",
     example=[
         ["game.py", "Contains Game class and ... functions"],
         ["main.py", "Contains main function, from game import Game"],

metagpt/actions/rebuild_class_view.py

@@ -14,7 +14,6 @@ from typing import Optional, Set, Tuple
 import aiofiles
 
 from metagpt.actions import Action
-from metagpt.config2 import config
 from metagpt.const import (
     AGGREGATION,
     COMPOSITION,
@@ -40,7 +39,7 @@ class RebuildClassView(Action):
 
     graph_db: Optional[GraphRepository] = None
 
-    async def run(self, with_messages=None, format=config.prompt_schema):
+    async def run(self, with_messages=None, format=None):
         """
         Implementation of `Action`'s `run` method.
 
@@ -48,6 +47,7 @@ class RebuildClassView(Action):
             with_messages (Optional[Type]): An optional argument specifying messages to react to.
             format (str): The format for the prompt schema.
         """
+        format = format if format else self.config.prompt_schema
         graph_repo_pathname = self.context.git_repo.workdir / GRAPH_REPO_FILE_REPO / self.context.git_repo.workdir.name
         self.graph_db = await DiGraphRepository.load_from(str(graph_repo_pathname.with_suffix(".json")))
         repo_parser = RepoParser(base_directory=Path(self.i_context))

metagpt/actions/rebuild_sequence_view.py

@@ -18,7 +18,6 @@ from pydantic import BaseModel
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions import Action
-from metagpt.config2 import config
 from metagpt.const import GRAPH_REPO_FILE_REPO
 from metagpt.logs import logger
 from metagpt.repo_parser import CodeBlockInfo, DotClassInfo
@@ -84,7 +83,7 @@ class RebuildSequenceView(Action):
 
     graph_db: Optional[GraphRepository] = None
 
-    async def run(self, with_messages=None, format=config.prompt_schema):
+    async def run(self, with_messages=None, format=None):
         """
         Implementation of `Action`'s `run` method.
 
@@ -92,6 +91,7 @@ class RebuildSequenceView(Action):
             with_messages (Optional[Type]): An optional argument specifying messages to react to.
             format (str): The format for the prompt schema.
         """
+        format = format if format else self.config.prompt_schema
         graph_repo_pathname = self.context.git_repo.workdir / GRAPH_REPO_FILE_REPO / self.context.git_repo.workdir.name
         self.graph_db = await DiGraphRepository.load_from(str(graph_repo_pathname.with_suffix(".json")))
         if not self.i_context:
@@ -244,15 +244,6 @@ class RebuildSequenceView(Action):
         class_view = await self._get_uml_class_view(ns_class_name)
         source_code = await self._get_source_code(ns_class_name)
 
-        # prompt_blocks = [
-        #     "## Instruction\n"
-        #     "You are a python code to UML 2.0 Use Case translator.\n"
-        #     'The generated UML 2.0 Use Case must include the roles or entities listed in "Participants".\n'
-        #     "The functional descriptions of Actors and Use Cases in the generated UML 2.0 Use Case must not "
-        #     'conflict with the information in "Mermaid Class Views".\n'
-        #     'The section under `if __name__ == "__main__":` of "Source Code" contains information about external '
-        #     "system interactions with the internal system.\n"
-        # ]
         prompt_blocks = []
         block = "## Participants\n"
         for p in participants:
@@ -340,6 +331,7 @@ class RebuildSequenceView(Action):
             system_msgs=[
                 "You are a Mermaid Sequence Diagram translator in function detail.",
                 "Translate the markdown text to a Mermaid Sequence Diagram.",
+                "Response must be concise.",
                 "Return a markdown mermaid code block.",
             ],
             stream=False,
@@ -440,7 +432,7 @@ class RebuildSequenceView(Action):
         rows = await self.graph_db.select(subject=ns_class_name, predicate=GraphKeyword.HAS_PAGE_INFO)
         filename = split_namespace(ns_class_name=ns_class_name)[0]
         if not rows:
-            src_filename = RebuildSequenceView._get_full_filename(root=self.i_context, pathname=filename)
+            src_filename = RebuildSequenceView.get_full_filename(root=self.i_context, pathname=filename)
             if not src_filename:
                 return ""
             return await aread(filename=src_filename, encoding="utf-8")
@@ -450,7 +442,7 @@ class RebuildSequenceView(Action):
         )
 
     @staticmethod
-    def _get_full_filename(root: str | Path, pathname: str | Path) -> Path | None:
+    def get_full_filename(root: str | Path, pathname: str | Path) -> Path | None:
         """
         Convert package name to the full path of the module.
 
@@ -466,7 +458,7 @@ class RebuildSequenceView(Action):
             "metagpt/management/skill_manager.py", then the returned value will be
             "/User/xxx/github/MetaGPT/metagpt/management/skill_manager.py"
         """
-        if re.match(r"^/.+", pathname):
+        if re.match(r"^/.+", str(pathname)):
             return pathname
         files = list_files(root=root)
         postfix = "/" + str(pathname)

metagpt/actions/requirement_analysis/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/13
+@Author  : mashenquan
+@File    : __init__.py
+@Desc    : The implementation of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb
+"""
+from metagpt.actions.requirement_analysis.evaluate_action import EvaluationData, EvaluateAction
+
+__all__ = [EvaluationData, EvaluateAction]

metagpt/actions/requirement_analysis/evaluate_action.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/13
+@Author  : mashenquan
+@File    : evaluate_action.py
+@Desc    : The implementation of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb
+"""
+from typing import Optional
+
+from pydantic import BaseModel
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+from metagpt.actions import Action
+from metagpt.logs import logger
+from metagpt.utils.common import CodeParser, general_after_log, to_markdown_code_block
+
+
+class EvaluationData(BaseModel):
+    """Model to represent evaluation data.
+
+    Attributes:
+        is_pass (bool): Indicates if the evaluation passed or failed.
+        conclusion (Optional[str]): Conclusion or remarks about the evaluation.
+    """
+
+    is_pass: bool
+    conclusion: Optional[str] = None
+
+
+class EvaluateAction(Action):
+    """The base class for an evaluation action.
+
+    This class provides methods to evaluate prompts using a specified language model.
+    """
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
+    async def _evaluate(self, prompt: str) -> (bool, str):
+        """Evaluates a given prompt.
+
+        Args:
+            prompt (str): The prompt to be evaluated.
+
+        Returns:
+            tuple: A tuple containing:
+                - bool: Indicates if the evaluation passed.
+                - str: The JSON string containing the evaluation data.
+        """
+        rsp = await self.llm.aask(prompt)
+        json_data = CodeParser.parse_code(text=rsp, lang="json")
+        data = EvaluationData.model_validate_json(json_data)
+        return data.is_pass, to_markdown_code_block(val=json_data, type_="json")
+
+    async def _vote(self, prompt: str) -> EvaluationData:
+        """Evaluates a prompt multiple times and returns the consensus.
+
+        Args:
+            prompt (str): The prompt to be evaluated.
+
+        Returns:
+            EvaluationData: An object containing the evaluation result and a summary of evaluations.
+        """
+        evaluations = {}
+        for i in range(3):
+            vote, evaluation = await self._evaluate(prompt)
+            val = evaluations.get(vote, [])
+            val.append(evaluation)
+            if len(val) > 1:
+                return EvaluationData(is_pass=vote, conclusion="\n".join(val))
+            evaluations[vote] = val

metagpt/actions/requirement_analysis/framework/__init__.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/13
+@Author  : mashenquan
+@File    : __init__.py
+@Desc    : The implementation of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb
+"""
+import json
+import uuid
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Union, List
+
+from pydantic import BaseModel
+
+from metagpt.actions.requirement_analysis.framework.evaluate_framework import EvaluateFramework
+from metagpt.actions.requirement_analysis.framework.write_framework import WriteFramework
+from metagpt.config2 import config
+from metagpt.utils.common import awrite
+
+
+async def save_framework(
+    dir_data: str, trd: Optional[str] = None, output_dir: Optional[Union[str, Path]] = None
+) -> List[str]:
+    """
+    Saves framework data to files based on input JSON data and optionally saves a TRD (technical requirements document).
+
+    Args:
+        dir_data (str): JSON data in string format enclosed in triple backticks ("```json" "...data..." "```").
+        trd (str, optional): Technical requirements document content to be saved. Defaults to None.
+        output_dir (Union[str, Path], optional): Output directory path where files will be saved. If not provided,
+            a default directory is created based on the current timestamp and a random UUID suffix.
+
+    Returns:
+        List[str]: List of file paths where data was saved.
+
+    Raises:
+        Any exceptions raised during file writing operations.
+
+    Notes:
+        - JSON data should be provided in the format "```json ...data... ```".
+        - The function ensures that paths and filenames are correctly formatted and creates necessary directories.
+
+    Example:
+        ```python
+        dir_data = "```json\n[{\"path\": \"/folder\", \"filename\": \"file1.txt\", \"content\": \"Some content\"}]\n```"
+        trd = "Technical requirements document content."
+        output_dir = '/path/to/output/dir'
+        saved_files = await save_framework(dir_data, trd, output_dir)
+        print(saved_files)
+        ```
+    """
+    output_dir = (
+        Path(output_dir)
+        if output_dir
+        else config.workspace.path / (datetime.now().strftime("%Y%m%d%H%M%ST") + uuid.uuid4().hex[0:8])
+    )
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    json_data = dir_data.removeprefix("```json").removesuffix("```")
+    items = json.loads(json_data)
+
+    class Data(BaseModel):
+        path: str
+        filename: str
+        content: str
+
+    if trd:
+        pathname = output_dir / "TRD.md"
+        await awrite(filename=pathname, data=trd)
+
+    files = []
+    for i in items:
+        v = Data.model_validate(i)
+        if v.path and v.path[0] == "/":
+            v.path = "." + v.path
+        pathname = output_dir / v.path
+        pathname.mkdir(parents=True, exist_ok=True)
+        pathname = pathname / v.filename
+        await awrite(filename=pathname, data=v.content)
+        files.append(str(pathname))
+    return files
+
+
+__all__ = [WriteFramework, EvaluateFramework]

metagpt/actions/requirement_analysis/framework/evaluate_framework.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/13
+@Author  : mashenquan
+@File    : evaluate_framework.py
+@Desc    : The implementation of Chapter 2.1.8 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb
+"""
+
+from metagpt.actions.requirement_analysis import EvaluateAction, EvaluationData
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import to_markdown_code_block
+
+
+@register_tool(include_functions=["run"])
+class EvaluateFramework(EvaluateAction):
+    """WriteFramework deal with the following situations:
+    1. Given a TRD and the software framework based on the TRD, evaluate the quality of the software framework.
+    """
+
+    async def run(
+        self,
+        *,
+        use_case_actors: str,
+        trd: str,
+        acknowledge: str,
+        legacy_output: str,
+        additional_technical_requirements: str,
+    ) -> EvaluationData:
+        """
+        Run the evaluation of the software framework based on the provided TRD and related parameters.
+
+        Args:
+            use_case_actors (str): A description of the actors involved in the use case.
+            trd (str): The Technical Requirements Document (TRD) that outlines the requirements for the software framework.
+            acknowledge (str): External acknowledgments or acknowledgments information related to the framework.
+            legacy_output (str): The previous versions of software framework returned by `WriteFramework`.
+            additional_technical_requirements (str): Additional technical requirements that need to be considered during evaluation.
+
+        Returns:
+            EvaluationData: An object containing the results of the evaluation.
+
+        Example:
+            >>> evaluate_framework = EvaluateFramework()
+            >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;"
+            >>> trd = "## TRD\\n..."
+            >>> acknowledge = "## Interfaces\\n..."
+            >>> framework = '{"path":"balabala", "filename":"...", ...'
+            >>> constraint = "Using Java language, ..."
+            >>> evaluation = await evaluate_framework.run(
+            >>>     use_case_actors=use_case_actors,
+            >>>     trd=trd,
+            >>>     acknowledge=acknowledge,
+            >>>     legacy_output=framework,
+            >>>     additional_technical_requirements=constraint,
+            >>> )
+            >>> is_pass = evaluation.is_pass
+            >>> print(is_pass)
+            True
+            >>> evaluation_conclusion = evaluation.conclusion
+            >>> print(evaluation_conclusion)
+            Balabala...
+        """
+        prompt = PROMPT.format(
+            use_case_actors=use_case_actors,
+            trd=to_markdown_code_block(val=trd),
+            acknowledge=to_markdown_code_block(val=acknowledge),
+            legacy_output=to_markdown_code_block(val=legacy_output),
+            additional_technical_requirements=to_markdown_code_block(val=additional_technical_requirements),
+        )
+        return await self._vote(prompt)
+
+
+PROMPT = """
+## Actor, System, External System
+{use_case_actors}
+
+## Legacy TRD
+{trd}
+
+## Acknowledge
+{acknowledge}
+
+## Legacy Outputs
+{legacy_output}
+
+## Additional Technical Requirements
+{additional_technical_requirements}
+
+---
+You are a tool that evaluates the quality of framework code based on the TRD content;
+You need to refer to the content of the "Legacy TRD" section to check for any errors or omissions in the framework code found in "Legacy Outputs";
+The content of "Actor, System, External System" provides an explanation of actors and systems that appear in UML Use Case diagram;
+Information about the external system missing from the "Legacy TRD" can be found in the "Acknowledge" section;
+Which interfaces defined in "Acknowledge" are used in the "Legacy TRD"?
+Do not implement the interface in "Acknowledge" section until it is used in "Legacy TRD", you can check whether they are the same interface by looking at its ID or url;
+Parts not mentioned in the "Legacy TRD" will be handled by other TRDs, therefore, processes not present in the "Legacy TRD" are considered ready;
+"Additional Technical Requirements" specifies the additional technical requirements that the generated software framework code must meet;
+Do the parameters of the interface of the external system used in the code comply with it's specifications in 'Acknowledge'?
+Is there a lack of necessary configuration files?
+Return a markdown JSON object with:
+- an "issues" key containing a string list of natural text about the issues that need to addressed, found in the "Legacy Outputs" if any exits, each issue found must provide a detailed description and include reasons;
+- a "conclusion" key containing the evaluation conclusion;
+- a "misalignment" key containing the judgement detail of the natural text string list about the misalignment with "Legacy TRD";
+- a "is_pass" key containing a true boolean value if there is not any issue in the "Legacy Outputs";
+"""

metagpt/actions/requirement_analysis/framework/write_framework.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/13
+@Author  : mashenquan
+@File    : write_framework.py
+@Desc    : The implementation of Chapter 2.1.8 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb
+"""
+import json
+
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+from metagpt.actions import Action
+from metagpt.logs import logger
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import general_after_log, to_markdown_code_block
+
+
+@register_tool(include_functions=["run"])
+class WriteFramework(Action):
+    """WriteFramework deal with the following situations:
+    1. Given a TRD, write out the software framework.
+    """
+
+    async def run(
+        self,
+        *,
+        use_case_actors: str,
+        trd: str,
+        acknowledge: str,
+        legacy_output: str,
+        evaluation_conclusion: str,
+        additional_technical_requirements: str,
+    ) -> str:
+        """
+        Run the action to generate a software framework based on the provided TRD and related information.
+
+        Args:
+            use_case_actors (str): Description of the use case actors involved.
+            trd (str): Technical Requirements Document detailing the requirements.
+            acknowledge (str): External acknowledgements or acknowledgements required.
+            legacy_output (str): Previous version of the software framework returned by `WriteFramework.run`.
+            evaluation_conclusion (str): Conclusion from the evaluation of the requirements.
+            additional_technical_requirements (str): Any additional technical requirements.
+
+        Returns:
+            str: The generated software framework as a string.
+
+        Example:
+            >>> write_framework = WriteFramework()
+            >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;"
+            >>> trd = "## TRD\\n..."
+            >>> acknowledge = "## Interfaces\\n..."
+            >>> legacy_output = '{"path":"balabala", "filename":"...", ...'
+            >>> evaluation_conclusion = "Balabala..."
+            >>> constraint = "Using Java language, ..."
+            >>> framework = await write_framework.run(
+            >>>    use_case_actors=use_case_actors,
+            >>>    trd=trd,
+            >>>    acknowledge=acknowledge,
+            >>>    legacy_output=framework,
+            >>>    evaluation_conclusion=evaluation_conclusion,
+            >>>    additional_technical_requirements=constraint,
+            >>> )
+            >>> print(framework)
+            {"path":"balabala", "filename":"...", ...
+
+        """
+        acknowledge = await self._extract_external_interfaces(trd=trd, knowledge=acknowledge)
+        prompt = PROMPT.format(
+            use_case_actors=use_case_actors,
+            trd=to_markdown_code_block(val=trd),
+            acknowledge=to_markdown_code_block(val=acknowledge),
+            legacy_output=to_markdown_code_block(val=legacy_output),
+            evaluation_conclusion=evaluation_conclusion,
+            additional_technical_requirements=to_markdown_code_block(val=additional_technical_requirements),
+        )
+        return await self._write(prompt)
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
+    async def _write(self, prompt: str) -> str:
+        rsp = await self.llm.aask(prompt)
+        # Do not use `CodeParser` here.
+        tags = ["```json", "```"]
+        bix = rsp.find(tags[0])
+        eix = rsp.rfind(tags[1])
+        if bix >= 0:
+            rsp = rsp[bix : eix + len(tags[1])]
+        json_data = rsp.removeprefix("```json").removesuffix("```")
+        json.loads(json_data)  # validate
+        return json_data
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
+    async def _extract_external_interfaces(self, trd: str, knowledge: str) -> str:
+        prompt = f"## TRD\n{to_markdown_code_block(val=trd)}\n\n## Knowledge\n{to_markdown_code_block(val=knowledge)}\n"
+        rsp = await self.llm.aask(
+            prompt,
+            system_msgs=[
+                "You are a tool that removes impurities from articles; you can remove irrelevant content from articles.",
+                'Identify which interfaces are used in "TRD"? Remove the relevant content of the interfaces NOT used in "TRD" from "Knowledge" and return the simplified content of "Knowledge".',
+            ],
+        )
+        return rsp
+
+
+PROMPT = """
+## Actor, System, External System
+{use_case_actors}
+
+## TRD
+{trd}
+
+## Acknowledge
+{acknowledge}
+
+## Legacy Outputs
+{legacy_output}
+
+## Evaluation Conclusion
+{evaluation_conclusion}
+
+## Additional Technical Requirements
+{additional_technical_requirements}
+
+---
+You are a tool that generates software framework code based on TRD.
+The content of "Actor, System, External System" provides an explanation of actors and systems that appear in UML Use Case diagram;
+The descriptions of the interfaces of the external system used in the "TRD" can be found in the "Acknowledge" section; Do not implement the interface of the external system in "Acknowledge" section until it is used in "TRD";
+"Legacy Outputs" contains the software framework code generated by you last time, which you can improve by addressing the issues raised in "Evaluation Conclusion";
+"Additional Technical Requirements" specifies the additional technical requirements that the generated software framework code must meet;
+Develop the software framework based on the "TRD", the output files should include:
+- The `README.md` file should include:
+  - The folder structure diagram of the entire project;
+  - Correspondence between classes, interfaces, and functions with the content in the "TRD" section；
+  - Prerequisites if necessary;
+  - Installation if necessary;
+  - Configuration if necessary;
+  - Usage if necessary;
+- The `CLASS.md` file should include the class diagram in PlantUML format based on the "TRD";
+- The `SEQUENCE.md` file should include the sequence diagram in PlantUML format based on the "TRD";
+- The source code files that implement the "TRD" and "Additional Technical Requirements"; do not add comments to source code files;
+- The configuration files that required by the source code files, "TRD" and "Additional Technical Requirements";
+  
+Return a markdown JSON object list, each object containing:
+- a "path" key with a value specifying its path;
+- a "filename" key with a value specifying its file name;
+- a "content" key with a value containing its file content;
+"""

metagpt/actions/requirement_analysis/requirement/pic2txt.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/27
+@Author  : mashenquan
+@File    : pic2txt.py
+"""
+import json
+from pathlib import Path
+from typing import List
+
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+from metagpt.actions import Action
+from metagpt.logs import logger
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import encode_image, general_after_log, to_markdown_code_block
+
+
+@register_tool(include_functions=["run"])
+class Pic2Txt(Action):
+    """Pic2Txt deal with the following situations:
+    Given some pictures depicting user requirements alongside contextual description, write out the intact textual user requirements.
+    """
+
+    async def run(
+        self,
+        *,
+        image_paths: List[str],
+        textual_user_requirement: str = "",
+        legacy_output: str = "",
+        evaluation_conclusion: str = "",
+        additional_technical_requirements: str = "",
+    ) -> str:
+        """
+        Given some pictures depicting user requirements alongside contextual description, write out the intact textual user requirements
+
+        Args:
+            image_paths (List[str]): A list of file paths to the input image(s) depicting user requirements.
+            textual_user_requirement (str, optional): Textual user requirement that alongside the given images, if any.
+            legacy_output (str, optional): The intact textual user requirements generated by you last time, if any.
+            evaluation_conclusion (str, optional): Conclusion or evaluation based on the processed requirements.
+            additional_technical_requirements (str, optional): Any supplementary technical details relevant to the process.
+
+        Returns:
+            str: Textual representation of user requirements extracted from the provided image(s).
+
+        Raises:
+            ValueError: If image_paths list is empty.
+            OSError: If there is an issue accessing or reading the image files.
+
+        Example:
+            >>> images = ["requirements/pic/1.png", "requirements/pic/2.png", "requirements/pic/3.png"]
+            >>> textual_user_requirements = "User requirement paragraph 1 ..., ![](1.png). paragraph 2...![](2.png)..."
+            >>> action = Pic2Txt()
+            >>> intact_textual_user_requirements = await action.run(image_paths=images, textual_user_requirement=textual_user_requirements)
+            >>> print(intact_textual_user_requirements)
+            "User requirement paragraph 1 ..., ![...](1.png) This picture describes... paragraph 2...![...](2.png)..."
+
+        """
+        descriptions = {}
+        for i in image_paths:
+            filename = Path(i)
+            base64_image = encode_image(filename)
+            rsp = await self._pic2txt(
+                "Generate a paragraph of text based on the content of the image, the language of the text is consistent with the language in the image.",
+                base64_image=base64_image,
+            )
+            descriptions[filename.name] = rsp
+
+        prompt = PROMPT.format(
+            textual_user_requirement=textual_user_requirement,
+            acknowledge=to_markdown_code_block(val=json.dumps(descriptions), type_="json"),
+            legacy_output=to_markdown_code_block(val=legacy_output),
+            evaluation_conclusion=evaluation_conclusion,
+            additional_technical_requirements=to_markdown_code_block(val=additional_technical_requirements),
+        )
+        return await self._write(prompt)
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
+    async def _write(self, prompt: str) -> str:
+        rsp = await self.llm.aask(prompt)
+        return rsp
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
+    async def _pic2txt(self, prompt: str, base64_image: str) -> str:
+        rsp = await self.llm.aask(prompt, images=base64_image)
+        return rsp
+
+
+PROMPT = """
+## Textual User Requirements
+{textual_user_requirement}
+
+## Acknowledge
+{acknowledge}
+
+## Legacy Outputs
+{legacy_output}
+
+## Evaluation Conclusion
+{evaluation_conclusion}
+
+## Additional Technical Requirements
+{additional_technical_requirements}
+
+---
+You are a tool that generates an intact textual user requirements given a few of textual fragments of user requirements and some fragments of UI pictures.
+The content of "Textual User Requirements" provides a few of textual fragments of user requirements;
+The content of "Acknowledge" provides the descriptions of pictures used in "Textual User Requirements";
+"Legacy Outputs" contains the intact textual user requirements generated by you last time, which you can improve by addressing the issues raised in "Evaluation Conclusion";
+"Additional Technical Requirements" specifies the additional technical requirements that the generated textual user requirements must meet;
+You need to merge the text content of the corresponding image in the "Acknowledge" into the "Textual User Requirements" to generate a complete, natural and coherent description of the user requirements;
+Return the intact textual user requirements according to the given fragments of the user requirement of "Textual User Requirements" and the UI pictures;
+"""

metagpt/actions/requirement_analysis/trd/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/13
+@Author  : mashenquan
+@File    : __init__.py
+@Desc    : The implementation of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb
+"""
+
+
+from metagpt.actions.requirement_analysis.trd.detect_interaction import DetectInteraction
+from metagpt.actions.requirement_analysis.trd.evaluate_trd import EvaluateTRD
+from metagpt.actions.requirement_analysis.trd.write_trd import WriteTRD
+from metagpt.actions.requirement_analysis.trd.compress_external_interfaces import CompressExternalInterfaces
+
+__all__ = [CompressExternalInterfaces, DetectInteraction, WriteTRD, EvaluateTRD]

metagpt/actions/requirement_analysis/trd/compress_external_interfaces.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/13
+@Author  : mashenquan
+@File    : compress_external_interfaces.py
+@Desc    : The implementation of Chapter 2.1.5 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb
+"""
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+from metagpt.actions import Action
+from metagpt.logs import logger
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import general_after_log
+
+
+@register_tool(include_functions=["run"])
+class CompressExternalInterfaces(Action):
+    """CompressExternalInterfaces deal with the following situations:
+    1. Given a natural text of acknowledgement, it extracts and compresses the information about external system interfaces.
+    """
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
+    async def run(
+        self,
+        *,
+        acknowledge: str,
+    ) -> str:
+        """
+        Extracts and compresses information about external system interfaces from a given acknowledgement text.
+
+        Args:
+            acknowledge (str): A natural text of acknowledgement containing details about external system interfaces.
+
+        Returns:
+            str: A compressed version of the information about external system interfaces.
+
+        Example:
+            >>> compress_acknowledge = CompressExternalInterfaces()
+            >>> acknowledge = "## Interfaces\\n..."
+            >>> available_external_interfaces = await compress_acknowledge.run(acknowledge=acknowledge)
+            >>> print(available_external_interfaces)
+            ```json\n[\n{\n"id": 1,\n"inputs": {...
+        """
+        return await self.llm.aask(
+            msg=acknowledge,
+            system_msgs=[
+                "Extracts and compresses the information about external system interfaces.",
+                "Return a markdown JSON list of objects, each object containing:\n"
+                '- an "id" key containing the interface id;\n'
+                '- an "inputs" key containing a dict of input parameters that consist of name and description pairs;\n'
+                '- an "outputs" key containing a dict of returns that consist of name and description pairs;\n',
+            ],
+        )

metagpt/actions/requirement_analysis/trd/detect_interaction.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/13
+@Author  : mashenquan
+@File    : detect_interaction.py
+@Desc    : The implementation of Chapter 2.1.6 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb
+"""
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+from metagpt.actions import Action
+from metagpt.logs import logger
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import general_after_log, to_markdown_code_block
+
+
+@register_tool(include_functions=["run"])
+class DetectInteraction(Action):
+    """DetectInteraction deal with the following situations:
+    1. Given a natural text of user requirements, it identifies the interaction events and the participants of those interactions from the original text.
+    """
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
+    async def run(
+        self,
+        *,
+        user_requirements: str,
+        use_case_actors: str,
+        legacy_interaction_events: str,
+        evaluation_conclusion: str,
+    ) -> str:
+        """
+        Identifies interaction events and participants from the user requirements.
+
+        Args:
+            user_requirements (str): A natural language text detailing the user's requirements.
+            use_case_actors (str): A description of the actors involved in the use case.
+            legacy_interaction_events (str): The previous version of the interaction events identified by you.
+            evaluation_conclusion (str): The external evaluation conclusions regarding the interactions events identified by you.
+
+        Returns:
+            str: A string summarizing the identified interaction events and their participants.
+
+        Example:
+            >>> detect_interaction = DetectInteraction()
+            >>> user_requirements = "User requirements 1. ..."
+            >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;"
+            >>> previous_version_interaction_events = "['interaction ...', ...]"
+            >>> evaluation_conclusion = "Issues: ..."
+            >>> interaction_events = await detect_interaction.run(
+            >>>    user_requirements=user_requirements,
+            >>>    use_case_actors=use_case_actors,
+            >>>    legacy_interaction_events=previous_version_interaction_events,
+            >>>    evaluation_conclusion=evaluation_conclusion,
+            >>> )
+            >>> print(interaction_events)
+            "['interaction ...', ...]"
+        """
+        msg = PROMPT.format(
+            use_case_actors=use_case_actors,
+            original_user_requirements=to_markdown_code_block(val=user_requirements),
+            previous_version_of_interaction_events=legacy_interaction_events,
+            the_evaluation_conclusion_of_previous_version_of_trd=evaluation_conclusion,
+        )
+        return await self.llm.aask(msg=msg)
+
+
+PROMPT = """
+## Actor, System, External System
+{use_case_actors}
+
+## User Requirements
+{original_user_requirements}
+
+## Legacy Interaction Events
+{previous_version_of_interaction_events}
+
+## Evaluation Conclusion
+{the_evaluation_conclusion_of_previous_version_of_trd}
+
+---
+You are a tool for capturing interaction events.
+"Actor, System, External System" provides the possible participants of the interaction event;
+"Legacy Interaction Events" is the contents of the interaction events that you output earlier;
+Some descriptions in the "Evaluation Conclusion" relate to the content of "User Requirements", and these descriptions in the "Evaluation Conclusion" address some issues regarding the content of "Legacy Interaction Events";
+You need to capture the interaction events occurring in the description within the content of "User Requirements" word-for-word, including:
+1. Who is interacting with whom. An interaction event has a maximum of 2 participants. If there are multiple participants, it indicates that multiple events are combined into one event and should be further split;
+2. When an interaction event occurs, who is the initiator? What data did the initiator enter?
+3. What data does the interaction event ultimately return according to the "User Requirements"?
+
+You can check the data flow described in the "User Requirements" to see if there are any missing interaction events;
+Return a markdown JSON object list, each object of the list containing:
+- a "name" key containing the name of the interaction event;
+- a "participants" key containing a string list of the names of the two participants;
+- a "initiator" key containing the name of the participant who initiate the interaction;
+- a "input" key containing a natural text description about the input data;
+"""

metagpt/actions/requirement_analysis/trd/evaluate_trd.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/13
+@Author  : mashenquan
+@File    : evaluate_trd.py
+@Desc    : The implementation of Chapter 2.1.6~2.1.7 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb
+"""
+
+from metagpt.actions.requirement_analysis import EvaluateAction, EvaluationData
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import to_markdown_code_block
+
+
+@register_tool(include_functions=["run"])
+class EvaluateTRD(EvaluateAction):
+    """EvaluateTRD deal with the following situations:
+    1. Given a TRD, evaluates the quality and returns a conclusion.
+    """
+
+    async def run(
+        self,
+        *,
+        user_requirements: str,
+        use_case_actors: str,
+        trd: str,
+        interaction_events: str,
+        legacy_user_requirements_interaction_events: str = "",
+    ) -> EvaluationData:
+        """
+        Evaluates the given TRD based on user requirements, use case actors, interaction events, and optionally external legacy interaction events.
+
+        Args:
+            user_requirements (str): The requirements provided by the user.
+            use_case_actors (str): The actors involved in the use case.
+            trd (str): The TRD (Technical Requirements Document) to be evaluated.
+            interaction_events (str): The interaction events related to the user requirements and the TRD.
+            legacy_user_requirements_interaction_events (str, optional): External legacy interaction events tied to the user requirements. Defaults to an empty string.
+
+        Returns:
+            EvaluationData: The conclusion of the TRD evaluation.
+
+        Example:
+            >>> evaluate_trd = EvaluateTRD()
+            >>> user_requirements = "User requirements 1. ..."
+            >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;"
+            >>> trd = "## TRD\\n..."
+            >>> interaction_events = "['interaction ...', ...]"
+            >>> evaluation_conclusion = "Issues: ..."
+            >>> legacy_user_requirements_interaction_events = ["user requirements 1. ...", ...]
+            >>> evaluation = await evaluate_trd.run(
+            >>>    user_requirements=user_requirements,
+            >>>    use_case_actors=use_case_actors,
+            >>>    trd=trd,
+            >>>    interaction_events=interaction_events,
+            >>>    legacy_user_requirements_interaction_events=str(legacy_user_requirements_interaction_events),
+            >>> )
+            >>> is_pass = evaluation.is_pass
+            >>> print(is_pass)
+            True
+            >>> evaluation_conclusion = evaluation.conclusion
+            >>> print(evaluation_conclusion)
+            ## Conclustion\n balabalabala...
+
+        """
+        prompt = PROMPT.format(
+            use_case_actors=use_case_actors,
+            user_requirements=to_markdown_code_block(val=user_requirements),
+            trd=to_markdown_code_block(val=trd),
+            legacy_user_requirements_interaction_events=legacy_user_requirements_interaction_events,
+            interaction_events=interaction_events,
+        )
+        return await self._vote(prompt)
+
+
+PROMPT = """
+## Actor, System, External System
+{use_case_actors}
+
+## User Requirements
+{user_requirements}
+
+## TRD Design
+{trd}
+
+## External Interaction Events
+{legacy_user_requirements_interaction_events}
+
+## Interaction Events
+{legacy_user_requirements_interaction_events}
+{interaction_events}
+
+---
+You are a tool to evaluate the TRD design.
+"Actor, System, External System" provides the all possible participants in interaction events;
+"User Requirements" provides the original requirements description, any parts not mentioned in this description will be handled by other modules, so do not fabricate requirements;
+"External Interaction Events" is provided by an external module for your use, its content is also referred to "Interaction Events" section; The content in "External Interaction Events" can be determined to be problem-free;
+"External Interaction Events" provides some identified interaction events and the interacting participants based on the part of the content of the "User Requirements";
+"Interaction Events" provides some identified interaction events and the interacting participants based on the content of the "User Requirements";
+"TRD Design" provides a comprehensive design of the implementation steps for the original requirements, incorporating the interaction events from "Interaction Events" and adding additional steps to connect the complete upstream and downstream data flows;
+In order to integrate the full upstream and downstream data flow, the "TRD Design" allows for the inclusion of steps that do not appear in the original requirements description, but do not conflict with those explicitly described in the "User Requirements";
+Which interactions from "Interaction Events" correspond to which steps in "TRD Design"? Please provide reasons.
+Which aspects of "TRD Design" and "Interaction Events" do not align with the descriptions in "User Requirements"? Please provide detailed descriptions and reasons.
+If the descriptions in "User Requirements" are divided into multiple steps in "TRD Design" and "Interaction Events," it can be considered compliant with the descriptions in "User Requirements" as long as it does not conflict with them;
+There is a possibility of missing details in the descriptions of "User Requirements". Any additional steps in "TRD Design" and "Interaction Events" are considered compliant with "User Requirements" as long as they do not conflict with the descriptions provided in "User Requirements";
+If there are interaction events with external systems in "TRD Design", you must explicitly specify the ID of the external interface to use for the interaction events, the input and output parameters of the used external interface must explictly match the input and output of the interaction event；
+Does the sequence of steps in "Interaction Events" cause performance or cost issues? Please provide detailed descriptions and reasons;
+If each step of "TRD Design" has input data, its input data is provided either by the output of the previous steps or by participants of "Actor, System, External System", and there should be no passive data;
+Return a markdown JSON object with:
+- an "issues" key containing a string list of natural text about the issues that need to be addressed, found in the "TRD Design" if any exist, each issue found must provide a detailed description and include reasons;
+- a "conclusion" key containing the evaluation conclusion;
+- a "correspondence_between" key containing the judgement detail of the natural text string list about the correspondence between "Interaction Events" and "TRD Design" steps;
+- a "misalignment" key containing the judgement detail of the natural text string list about the misalignment with "User Requirements";
+- a "is_pass" key containing a true boolean value if there is not any issue in the "TRD Design";
+"""

metagpt/actions/requirement_analysis/trd/write_trd.py

@@ -0,0 +1,261 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/6/13
+@Author  : mashenquan
+@File    : write_trd.py
+@Desc    : The implementation of Chapter 2.1.6~2.1.7 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb
+"""
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+from metagpt.actions import Action
+from metagpt.logs import logger
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import general_after_log, to_markdown_code_block
+
+
+@register_tool(include_functions=["run"])
+class WriteTRD(Action):
+    """WriteTRD deal with the following situations:
+    1. Given some new user requirements, write out a new TRD(Technical Requirements Document).
+    2. Given some incremental user requirements, update the legacy TRD.
+    """
+
+    async def run(
+        self,
+        *,
+        user_requirements: str = "",
+        use_case_actors: str,
+        available_external_interfaces: str,
+        evaluation_conclusion: str = "",
+        interaction_events: str,
+        previous_version_trd: str = "",
+        legacy_user_requirements: str = "",
+        legacy_user_requirements_trd: str = "",
+        legacy_user_requirements_interaction_events: str = "",
+    ) -> str:
+        """
+        Handles the writing or updating of a Technical Requirements Document (TRD) based on user requirements.
+
+        Args:
+            user_requirements (str): The new/incremental user requirements.
+            use_case_actors (str): Description of the actors involved in the use case.
+            available_external_interfaces (str): List of available external interfaces.
+            evaluation_conclusion (str, optional): The conclusion of the evaluation of the TRD written by you. Defaults to an empty string.
+            interaction_events (str): The interaction events related to the user requirements that you are handling.
+            previous_version_trd (str, optional): The previous version of the TRD written by you, for updating.
+            legacy_user_requirements (str, optional): Existing user requirements handled by an external object for your use. Defaults to an empty string.
+            legacy_user_requirements_trd (str, optional): The TRD associated with the existing user requirements handled by an external object for your use. Defaults to an empty string.
+            legacy_user_requirements_interaction_events (str, optional): Interaction events related to the existing user requirements handled by an external object for your use. Defaults to an empty string.
+
+        Returns:
+            str: The newly created or updated TRD written by you.
+
+        Example:
+            >>> # Given a new user requirements, write out a new TRD.
+            >>> user_requirements = "Write a 'snake game' TRD."
+            >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;"
+            >>> available_external_interfaces = "The available external interfaces returned by `CompressExternalInterfaces.run` are ..."
+            >>> previous_version_trd = "TRD ..." # The last version of the TRD written out if there is.
+            >>> evaluation_conclusion = "Conclusion ..." # The conclusion returned by `EvaluateTRD.run` if there is.
+            >>> interaction_events = "Interaction ..." # The interaction events returned by `DetectInteraction.run`.
+            >>> write_trd = WriteTRD()
+            >>> new_version_trd = await write_trd.run(
+            >>>     user_requirements=user_requirements,
+            >>>     use_case_actors=use_case_actors,
+            >>>     available_external_interfaces=available_external_interfaces,
+            >>>     evaluation_conclusion=evaluation_conclusion,
+            >>>     interaction_events=interaction_events,
+            >>>     previous_version_trd=previous_version_trd,
+            >>> )
+            >>> print(new_version_trd)
+            ## Technical Requirements Document\n ...
+
+            >>> # Given an incremental requirements, update the legacy TRD.
+            >>> legacy_user_requirements = ["User requirements 1. ...", "User requirements 2. ...", ...]
+            >>> legacy_user_requirements_trd = "## Technical Requirements Document\\n ..." # The TRD before integrating more user requirements.
+            >>> legacy_user_requirements_interaction_events = ["The interaction events list of user requirements 1 ...", "The interaction events list of user requiremnts 2 ...", ...]
+            >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;"
+            >>> available_external_interfaces = "The available external interfaces returned by `CompressExternalInterfaces.run` are ..."
+            >>> increment_requirements = "The incremental user requirements are ..."
+            >>> evaluation_conclusion = "Conclusion ..." # The conclusion returned by `EvaluateTRD.run` if there is.
+            >>> previous_version_trd = "TRD ..." # The last version of the TRD written out if there is.
+            >>> write_trd = WriteTRD()
+            >>> new_version_trd = await write_trd.run(
+            >>>     user_requirements=increment_requirements,
+            >>>     use_case_actors=use_case_actors,
+            >>>     available_external_interfaces=available_external_interfaces,
+            >>>     evaluation_conclusion=evaluation_conclusion,
+            >>>     interaction_events=interaction_events,
+            >>>     previous_version_trd=previous_version_trd,
+            >>>     legacy_user_requirements=str(legacy_user_requirements),
+            >>>     legacy_user_requirements_trd=legacy_user_requirements_trd,
+            >>>     legacy_user_requirements_interaction_events=str(legacy_user_requirements_interaction_events),
+            >>> )
+            >>> print(new_version_trd)
+            ## Technical Requirements Document\n ...
+        """
+        if legacy_user_requirements:
+            return await self._write_incremental_trd(
+                use_case_actors=use_case_actors,
+                legacy_user_requirements=legacy_user_requirements,
+                available_external_interfaces=available_external_interfaces,
+                legacy_user_requirements_trd=legacy_user_requirements_trd,
+                legacy_user_requirements_interaction_events=legacy_user_requirements_interaction_events,
+                incremental_user_requirements=user_requirements,
+                previous_version_trd=previous_version_trd,
+                evaluation_conclusion=evaluation_conclusion,
+                incremental_user_requirements_interaction_events=interaction_events,
+            )
+        return await self._write_new_trd(
+            use_case_actors=use_case_actors,
+            original_user_requirement=user_requirements,
+            available_external_interfaces=available_external_interfaces,
+            legacy_trd=previous_version_trd,
+            evaluation_conclusion=evaluation_conclusion,
+            interaction_events=interaction_events,
+        )
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
+    async def _write_new_trd(
+        self,
+        *,
+        use_case_actors: str,
+        original_user_requirement: str,
+        available_external_interfaces: str,
+        legacy_trd: str,
+        evaluation_conclusion: str,
+        interaction_events: str,
+    ) -> str:
+        prompt = NEW_PROMPT.format(
+            use_case_actors=use_case_actors,
+            original_user_requirement=to_markdown_code_block(val=original_user_requirement),
+            available_external_interfaces=available_external_interfaces,
+            legacy_trd=to_markdown_code_block(val=legacy_trd),
+            evaluation_conclusion=evaluation_conclusion,
+            interaction_events=interaction_events,
+        )
+        return await self.llm.aask(prompt)
+
+    @retry(
+        wait=wait_random_exponential(min=1, max=20),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
+    async def _write_incremental_trd(
+        self,
+        *,
+        use_case_actors: str,
+        legacy_user_requirements: str,
+        available_external_interfaces: str,
+        legacy_user_requirements_trd: str,
+        legacy_user_requirements_interaction_events: str,
+        incremental_user_requirements: str,
+        previous_version_trd: str,
+        evaluation_conclusion: str,
+        incremental_user_requirements_interaction_events: str,
+    ):
+        prompt = INCREMENTAL_PROMPT.format(
+            use_case_actors=use_case_actors,
+            legacy_user_requirements=to_markdown_code_block(val=legacy_user_requirements),
+            available_external_interfaces=available_external_interfaces,
+            legacy_user_requirements_trd=to_markdown_code_block(val=legacy_user_requirements_trd),
+            legacy_user_requirements_interaction_events=legacy_user_requirements_interaction_events,
+            incremental_user_requirements=to_markdown_code_block(val=incremental_user_requirements),
+            previous_version_trd=to_markdown_code_block(val=previous_version_trd),
+            evaluation_conclusion=evaluation_conclusion,
+            incremental_user_requirements_interaction_events=incremental_user_requirements_interaction_events,
+        )
+        return await self.llm.aask(prompt)
+
+
+NEW_PROMPT = """
+## Actor, System, External System
+{use_case_actors}
+
+## User Requirements
+{original_user_requirement}
+
+## Available External Interfaces
+{available_external_interfaces}
+
+## Legacy TRD
+{legacy_trd}
+
+## Evaluation Conclusion
+{evaluation_conclusion}
+
+## Interaction Events
+{interaction_events}
+
+---
+You are a TRD generator.
+The content of "Actor, System, External System" provides an explanation of actors and systems that appear in UML Use Case diagram;
+The content of "Available External Interfaces" provides the candidate steps, along with the inputs and outputs of each step;
+"User Requirements" provides the original requirements description, any parts not mentioned in this description will be handled by other modules, so do not fabricate requirements;
+"Legacy TRD" provides the old version of the TRD based on the "User Requirements" and can serve as a reference for the new TRD;
+"Evaluation Conclusion" provides a summary of the evaluation of the old TRD in the "Legacy TRD" and can serve as a reference for the new TRD;
+"Interaction Events" provides some identified interaction events and the interacting participants based on the content of the "User Requirements";
+1. What inputs and outputs are described in the "User Requirements"?
+2. How many steps are needed to achieve the inputs and outputs described in the "User Requirements"? Which actors from the "Actor, System, External System" section are involved in each step? What are the inputs and outputs of each step? Where is this output used, for example, as input for which interface or where it is required in the requirements, etc.?
+3. Output a complete Technical Requirements Document (TRD)：
+  3.1. In the description, use the actor and system names defined in the "Actor, System, External System" section to describe the interactors;
+  3.2. The content should include the original text of the requirements from "User Requirements";
+  3.3. In the TRD, each step can involve a maximum of two participants. If there are more than two participants, the step needs to be further split;
+  3.4. In the TRD, each step must include detailed descriptions, inputs, outputs, participants, initiator, and the rationale for the step's existence. The rationale should reference the original text to justify it, such as specifying which interface requires the output of this step as parameters or where in the requirements this step is mandated, etc.;
+  3.5. In the TRD, if you need to call interfaces of external systems, you must explicitly specify the interface IDs of the external systems you want to call;
+"""
+
+INCREMENTAL_PROMPT = """
+## Actor, System, External System
+{use_case_actors}
+
+## Legacy User Requirements
+{legacy_user_requirements}
+
+## Available External Interfaces
+{available_external_interfaces}
+
+## The TRD of Legacy User Requirements
+{legacy_user_requirements_trd}
+
+
+## The Interaction Events of Legacy User Requirements
+{legacy_user_requirements_interaction_events}
+
+## Incremental Requirements
+{incremental_user_requirements}
+
+## Legacy TRD
+{previous_version_trd}
+
+## Evaluation Conclusion
+{evaluation_conclusion}
+
+## Interaction Events
+{incremental_user_requirements_interaction_events}
+
+---
+You are a TRD generator.
+The content of "Actor, System, External System" provides an explanation of actors and systems that appear in UML Use Case diagram;
+The content of "Available External Interfaces" provides the candidate steps, along with the inputs and outputs of each step;
+"Legacy User Requirements" provides the original requirements description handled by other modules for your use;
+"The TRD of Legacy User Requirements" is the TRD generated by other modules based on the "Legacy User Requirements" for your use;
+"The Interaction Events of Legacy User Requirements" is the interaction events list generated by other modules based on the "Legacy User Requirements" for your use;
+"Incremental Requirements" provides the original requirements description that you need to address, any parts not mentioned in this description will be handled by other modules, so do not fabricate requirements;
+The requirements in "Legacy User Requirements" combined with the "Incremental Requirements" form a complete set of requirements, therefore, you need to add the TRD portion of the "Incremental Requirements" to "The TRD of Legacy User Requirements", the added content must not conflict with the original content of "The TRD of Legacy User Requirements";
+"Legacy TRD" provides the old version of the TRD you previously wrote based on the "Incremental Requirements" and can serve as a reference for the new TRD;
+"Evaluation Conclusion" provides a summary of the evaluation of the old TRD you generated in the "Legacy TRD", and the identified issues can serve as a reference for the new TRD you create;
+"Interaction Events" provides some identified interaction events and the interacting participants based on the content of the "Incremental Requirements";
+1. What inputs and outputs are described in the "Incremental Requirements"？
+2. How many steps are needed to achieve the inputs and outputs described in the "Incremental Requirements"? Which actors from the "Actor, System, External System" section are involved in each step? What are the inputs and outputs of each step? Where is this output used, for example, as input for which interface or where it is required in the requirements, etc.?
+3. Output a complete Technical Requirements Document (TRD)：
+  3.1. In the description, use the actor and system names defined in the "Actor, System, External System" section to describe the interactors;
+  3.2. The content should include the original text of the requirements from "User Requirements";
+  3.3. In the TRD, each step can involve a maximum of two participants. If there are more than two participants, the step needs to be further split;
+  3.4. In the TRD, each step must include detailed descriptions, inputs, outputs, participants, initiator, and the rationale for the step's existence. The rationale should reference the original text to justify it, such as specifying which interface requires the output of this step as parameters or where in the requirements this step is mandated, etc.
+    """

metagpt/actions/research.py

@@ -3,16 +3,17 @@
 from __future__ import annotations
 
 import asyncio
-from typing import Any, Callable, Optional, Union
+from datetime import datetime
+from typing import Any, Callable, Coroutine, Optional, Union
 
 from pydantic import TypeAdapter, model_validator
 
 from metagpt.actions import Action
-from metagpt.config2 import config
 from metagpt.logs import logger
 from metagpt.tools.search_engine import SearchEngine
 from metagpt.tools.web_browser_engine import WebBrowserEngine
 from metagpt.utils.common import OutputParser
+from metagpt.utils.parse_html import WebPage
 from metagpt.utils.text import generate_prompt_chunk, reduce_message_length
 
 LANG_PROMPT = "Please respond in {language}."
@@ -43,9 +44,10 @@ COLLECT_AND_RANKURLS_PROMPT = """### Topic
 {results}
 
 ### Requirements
-Please remove irrelevant search results that are not related to the query or topic. Then, sort the remaining search results \
-based on the link credibility. If two results have equal credibility, prioritize them based on the relevance. Provide the
-ranked results' indices in JSON format, like [0, 1, 3, 4, ...], without including other words.
+Please remove irrelevant search results that are not related to the query or topic.
+If the query is time-sensitive or specifies a certain time frame, please also remove search results that are outdated or outside the specified time frame. Notice that the current time is {time_stamp}.
+Then, sort the remaining search results based on the link credibility. If two results have equal credibility, prioritize them based on the relevance.
+Provide the ranked results' indices in JSON format, like [0, 1, 3, 4, ...], without including other words.
 """
 
 WEB_BROWSE_AND_SUMMARIZE_PROMPT = """### Requirements
@@ -133,8 +135,8 @@ class CollectLinks(Action):
                 if len(remove) == 0:
                     break
 
-        model_name = config.llm.model
-        prompt = reduce_message_length(gen_msg(), model_name, system_text, config.llm.max_token)
+        model_name = self.config.llm.model
+        prompt = reduce_message_length(gen_msg(), model_name, system_text, self.config.llm.max_token)
         logger.debug(prompt)
         queries = await self._aask(prompt, [system_text])
         try:
@@ -148,23 +150,27 @@ class CollectLinks(Action):
             ret[query] = await self._search_and_rank_urls(topic, query, url_per_query)
         return ret
 
-    async def _search_and_rank_urls(self, topic: str, query: str, num_results: int = 4) -> list[str]:
+    async def _search_and_rank_urls(
+        self, topic: str, query: str, num_results: int = 4, max_num_results: int = None
+    ) -> list[str]:
         """Search and rank URLs based on a query.
 
         Args:
             topic: The research topic.
             query: The search query.
             num_results: The number of URLs to collect.
+            max_num_results: The max number of URLs to collect.
 
         Returns:
             A list of ranked URLs.
         """
-        max_results = max(num_results * 2, 6)
-        results = await self.search_engine.run(query, max_results=max_results, as_string=False)
+        max_results = max_num_results or max(num_results * 2, 6)
+        results = await self._search_urls(query, max_results=max_results)
         if len(results) == 0:
             return []
         _results = "\n".join(f"{i}: {j}" for i, j in zip(range(max_results), results))
-        prompt = COLLECT_AND_RANKURLS_PROMPT.format(topic=topic, query=query, results=_results)
+        time_stamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        prompt = COLLECT_AND_RANKURLS_PROMPT.format(topic=topic, query=query, results=_results, time_stamp=time_stamp)
         logger.debug(prompt)
         indices = await self._aask(prompt)
         try:
@@ -178,6 +184,15 @@ class CollectLinks(Action):
             results = self.rank_func(results)
         return [i["link"] for i in results[:num_results]]
 
+    async def _search_urls(self, query: str, max_results: int) -> list[dict[str, str]]:
+        """Use search_engine to get urls.
+
+        Returns:
+            e.g. [{"title": "...", "link": "...", "snippet", "..."}]
+        """
+
+        return await self.search_engine.run(query, max_results=max_results, as_string=False)
+
 
 class WebBrowseAndSummarize(Action):
     """Action class to explore the web and provide summaries of articles and webpages."""
@@ -204,6 +219,8 @@ class WebBrowseAndSummarize(Action):
         *urls: str,
         query: str,
         system_text: str = RESEARCH_BASE_SYSTEM,
+        use_concurrent_summarization: bool = False,
+        per_page_timeout: Optional[float] = None,
     ) -> dict[str, str]:
         """Run the action to browse the web and provide summaries.
 
@@ -212,18 +229,41 @@ class WebBrowseAndSummarize(Action):
             urls: Additional URLs to browse.
             query: The research question.
             system_text: The system text.
+            use_concurrent_summarization: Whether to concurrently summarize the content of the webpage by LLM.
+            per_page_timeout: The maximum time for fetching a single page in seconds.
 
         Returns:
             A dictionary containing the URLs as keys and their summaries as values.
         """
-        contents = await self.web_browser_engine.run(url, *urls)
-        if not urls:
-            contents = [contents]
+        contents = await self._fetch_web_contents(url, *urls, per_page_timeout=per_page_timeout)
+
+        all_urls = [url] + list(urls)
+        summarize_tasks = [self._summarize_content(content, query, system_text) for content in contents]
+        summaries = await self._execute_summarize_tasks(summarize_tasks, use_concurrent_summarization)
+        result = {url: summary for url, summary in zip(all_urls, summaries) if summary}
+
+        return result
+
+    async def _fetch_web_contents(
+        self, url: str, *urls: str, per_page_timeout: Optional[float] = None
+    ) -> list[WebPage]:
+        """Fetch web contents from given URLs."""
+
+        contents = await self.web_browser_engine.run(url, *urls, per_page_timeout=per_page_timeout)
+
+        return [contents] if not urls else contents
+
+    async def _summarize_content(self, page: WebPage, query: str, system_text: str) -> str:
+        """Summarize web content."""
+        try:
+            prompt_template = WEB_BROWSE_AND_SUMMARIZE_PROMPT.format(query=query, content="{}")
+
+            content = page.inner_text
+
+            if self._is_content_invalid(content):
+                logger.warning(f"Invalid content detected for URL {page.url}: {content[:10]}...")
+                return None
 
-        summaries = {}
-        prompt_template = WEB_BROWSE_AND_SUMMARIZE_PROMPT.format(query=query, content="{}")
-        for u, content in zip([url, *urls], contents):
-            content = content.inner_text
             chunk_summaries = []
             for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, 4096):
                 logger.debug(prompt)
@@ -233,18 +273,33 @@ class WebBrowseAndSummarize(Action):
                 chunk_summaries.append(summary)
 
             if not chunk_summaries:
-                summaries[u] = None
-                continue
+                return None
 
             if len(chunk_summaries) == 1:
-                summaries[u] = chunk_summaries[0]
-                continue
+                return chunk_summaries[0]
 
             content = "\n".join(chunk_summaries)
             prompt = WEB_BROWSE_AND_SUMMARIZE_PROMPT.format(query=query, content=content)
             summary = await self._aask(prompt, [system_text])
-            summaries[u] = summary
-        return summaries
+            return summary
+        except Exception as e:
+            logger.error(f"Error summarizing content: {e}")
+            return None
+
+    def _is_content_invalid(self, content: str) -> bool:
+        """Check if the content is invalid based on specific starting phrases."""
+
+        invalid_starts = ["Fail to load page", "Access Denied"]
+
+        return any(content.strip().startswith(phrase) for phrase in invalid_starts)
+
+    async def _execute_summarize_tasks(self, tasks: list[Coroutine[Any, Any, str]], use_concurrent: bool) -> list[str]:
+        """Execute summarize tasks either concurrently or sequentially."""
+
+        if use_concurrent:
+            return await asyncio.gather(*tasks)
+
+        return [await task for task in tasks]
 
 
 class ConductResearch(Action):

metagpt/actions/search_enhanced_qa.py

@@ -0,0 +1,292 @@
+"""Enhancing question-answering capabilities through search engine augmentation."""
+
+from __future__ import annotations
+
+import json
+
+from pydantic import Field, PrivateAttr, model_validator
+
+from metagpt.actions import Action
+from metagpt.actions.research import CollectLinks, WebBrowseAndSummarize
+from metagpt.logs import logger
+from metagpt.tools.tool_registry import register_tool
+from metagpt.tools.web_browser_engine import WebBrowserEngine
+from metagpt.utils.common import CodeParser
+from metagpt.utils.parse_html import WebPage
+from metagpt.utils.report import ThoughtReporter
+
+REWRITE_QUERY_PROMPT = """
+Role: You are a highly efficient assistant that provide a better search query for web search engine to answer the given question.
+
+I will provide you with a question. Your task is to provide a better search query for web search engine.
+
+## Context
+### Question
+{q}
+
+## Format Example
+```json
+{{
+    "query": "the better search query for web search engine.",
+}}
+```
+
+## Instructions
+- Understand the question given by the user.
+- Provide a better search query for web search engine to answer the given question, your answer must be written in the same language as the question.
+- When rewriting, if you are unsure of the specific time, do not include the time.
+
+## Constraint
+Format: Just print the result in json format like **Format Example**.
+
+## Action
+Follow **Instructions**, generate output and make sure it follows the **Constraint**.
+"""
+
+SEARCH_ENHANCED_QA_SYSTEM_PROMPT = """
+You are a large language AI assistant built by MGX. You are given a user question, and please write clean, concise and accurate answer to the question. You will be given a set of related contexts to the question, each starting with a reference number like [[citation:x]], where x is a number. Please use the context.
+
+Your answer must be correct, accurate and written by an expert using an unbiased and professional tone. Please limit to 1024 tokens. Do not give any information that is not related to the question, and do not repeat. Say "information is missing on" followed by the related topic, if the given context do not provide sufficient information.
+
+Do not include [citation:x] in your anwser, where x is a number. Other than code and specific names and citations, your answer must be written in the same language as the question.
+
+Here are the set of contexts:
+
+{context}
+
+Remember, don't blindly repeat the contexts verbatim. And here is the user question:
+"""
+
+
+@register_tool(include_functions=["run"])
+class SearchEnhancedQA(Action):
+    """Question answering and info searching through search engine."""
+
+    name: str = "SearchEnhancedQA"
+    desc: str = "Integrating search engine results to anwser the question."
+
+    collect_links_action: CollectLinks = Field(
+        default_factory=CollectLinks, description="Action to collect relevant links from a search engine."
+    )
+    web_browse_and_summarize_action: WebBrowseAndSummarize = Field(
+        default=None,
+        description="Action to explore the web and provide summaries of articles and webpages.",
+    )
+    per_page_timeout: float = Field(
+        default=20, description="The maximum time for fetching a single page is in seconds. Defaults to 20s."
+    )
+    java_script_enabled: bool = Field(
+        default=False, description="Whether or not to enable JavaScript in the web browser context. Defaults to False."
+    )
+    user_agent: str = Field(
+        default="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.81",
+        description="Specific user agent to use in browser.",
+    )
+    extra_http_headers: dict = Field(
+        default={"sec-ch-ua": 'Chromium";v="125", "Not.A/Brand";v="24'},
+        description="An object containing additional HTTP headers to be sent with every request.",
+    )
+    max_chars_per_webpage_summary: int = Field(
+        default=4000, description="Maximum summary length for each web page content."
+    )
+    max_search_results: int = Field(
+        default=10,
+        description="Maximum number of search results (links) to collect using the collect_links_action. This controls the number of potential sources for answering the question.",
+    )
+
+    _reporter: ThoughtReporter = PrivateAttr(ThoughtReporter())
+
+    @model_validator(mode="after")
+    def initialize(self):
+        if self.web_browse_and_summarize_action is None:
+            web_browser_engine = WebBrowserEngine.from_browser_config(
+                self.config.browser,
+                proxy=self.config.proxy,
+                java_script_enabled=self.java_script_enabled,
+                extra_http_headers=self.extra_http_headers,
+                user_agent=self.user_agent,
+            )
+
+            self.web_browse_and_summarize_action = WebBrowseAndSummarize(web_browser_engine=web_browser_engine)
+
+        return self
+
+    async def run(self, query: str, rewrite_query: bool = True) -> str:
+        """Answer a query by leveraging web search results.
+
+        Args:
+            query (str): The original user query.
+            rewrite_query (bool): Whether to rewrite the query for better web search results. Defaults to True.
+
+        Returns:
+            str: A detailed answer based on web search results.
+
+        Raises:
+            ValueError: If the query is invalid.
+        """
+        async with self._reporter:
+            await self._reporter.async_report({"type": "search", "stage": "init"})
+            self._validate_query(query)
+
+            processed_query = await self._process_query(query, rewrite_query)
+            context = await self._build_context(processed_query)
+
+            return await self._generate_answer(processed_query, context)
+
+    def _validate_query(self, query: str) -> None:
+        """Validate the input query.
+
+        Args:
+            query (str): The query to validate.
+
+        Raises:
+            ValueError: If the query is invalid.
+        """
+
+        if not query.strip():
+            raise ValueError("Query cannot be empty or contain only whitespace.")
+
+    async def _process_query(self, query: str, should_rewrite: bool) -> str:
+        """Process the query, optionally rewriting it."""
+
+        if should_rewrite:
+            return await self._rewrite_query(query)
+
+        return query
+
+    async def _rewrite_query(self, query: str) -> str:
+        """Write a better search query for web search engine.
+
+        If the rewrite process fails, the original query is returned.
+
+        Args:
+            query (str): The original search query.
+
+        Returns:
+            str: The rewritten query if successful, otherwise the original query.
+        """
+
+        prompt = REWRITE_QUERY_PROMPT.format(q=query)
+
+        try:
+            resp = await self._aask(prompt)
+            rewritten_query = self._extract_rewritten_query(resp)
+
+            logger.info(f"Query rewritten: '{query}' -> '{rewritten_query}'")
+            return rewritten_query
+        except Exception as e:
+            logger.warning(f"Query rewrite failed. Returning original query. Error: {e}")
+            return query
+
+    def _extract_rewritten_query(self, response: str) -> str:
+        """Extract the rewritten query from the LLM's JSON response."""
+
+        resp_json = json.loads(CodeParser.parse_code(response, lang="json"))
+        return resp_json["query"]
+
+    async def _build_context(self, query: str) -> str:
+        """Construct a context string from web search citations.
+
+        Args:
+            query (str): The search query.
+
+        Returns:
+            str: Formatted context with numbered citations.
+        """
+
+        citations = await self._search_citations(query)
+        context = "\n\n".join([f"[[citation:{i+1}]] {c}" for i, c in enumerate(citations)])
+
+        return context
+
+    async def _search_citations(self, query: str) -> list[str]:
+        """Perform web search and summarize relevant content.
+
+        Args:
+            query (str): The search query.
+
+        Returns:
+            list[str]: Summaries of relevant web content.
+        """
+
+        relevant_urls = await self._collect_relevant_links(query)
+        await self._reporter.async_report({"type": "search", "stage": "searching", "urls": relevant_urls})
+        if not relevant_urls:
+            logger.warning(f"No relevant URLs found for query: {query}")
+            return []
+
+        logger.info(f"The Relevant links are: {relevant_urls}")
+
+        web_summaries = await self._summarize_web_content(relevant_urls)
+        if not web_summaries:
+            logger.warning(f"No summaries generated for query: {query}")
+            return []
+
+        citations = list(web_summaries.values())
+
+        return citations
+
+    async def _collect_relevant_links(self, query: str) -> list[str]:
+        """Search and rank URLs relevant to the query.
+
+        Args:
+            query (str): The search query.
+
+        Returns:
+            list[str]: Ranked list of relevant URLs.
+        """
+
+        return await self.collect_links_action._search_and_rank_urls(
+            topic=query, query=query, max_num_results=self.max_search_results
+        )
+
+    async def _summarize_web_content(self, urls: list[str]) -> dict[str, str]:
+        """Fetch and summarize content from given URLs.
+
+        Args:
+            urls (list[str]): List of URLs to summarize.
+
+        Returns:
+            dict[str, str]: Mapping of URLs to their summaries.
+        """
+
+        contents = await self._fetch_web_contents(urls)
+
+        summaries = {}
+        await self._reporter.async_report(
+            {"type": "search", "stage": "browsing", "pages": [i.model_dump() for i in contents]}
+        )
+        for content in contents:
+            url = content.url
+            inner_text = content.inner_text.replace("\n", "")
+            if self.web_browse_and_summarize_action._is_content_invalid(inner_text):
+                logger.warning(f"Invalid content detected for URL {url}: {inner_text[:10]}...")
+                continue
+
+            summary = inner_text[: self.max_chars_per_webpage_summary]
+            summaries[url] = summary
+
+        return summaries
+
+    async def _fetch_web_contents(self, urls: list[str]) -> list[WebPage]:
+        return await self.web_browse_and_summarize_action._fetch_web_contents(
+            *urls, per_page_timeout=self.per_page_timeout
+        )
+
+    async def _generate_answer(self, query: str, context: str) -> str:
+        """Generate an answer using the query and context.
+
+        Args:
+            query (str): The user's question.
+            context (str): Relevant information from web search.
+
+        Returns:
+            str: Generated answer based on the context.
+        """
+
+        system_prompt = SEARCH_ENHANCED_QA_SYSTEM_PROMPT.format(context=context)
+
+        async with ThoughtReporter(uuid=self._reporter.uuid, enable_llm_stream=True) as reporter:
+            await reporter.async_report({"type": "search", "stage": "answer"})
+            rsp = await self._aask(query, [system_prompt])
+        return rsp

metagpt/actions/summarize_code.py

@@ -6,13 +6,16 @@
 @Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode.
 """
 from pathlib import Path
+from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions.action import Action
 from metagpt.logs import logger
 from metagpt.schema import CodeSummarizeContext
+from metagpt.utils.common import get_markdown_code_block_type
+from metagpt.utils.project_repo import ProjectRepo
 
 PROMPT_TEMPLATE = """
 NOTICE
@@ -90,6 +93,8 @@ flowchart TB
 class SummarizeCode(Action):
     name: str = "SummarizeCode"
     i_context: CodeSummarizeContext = Field(default_factory=CodeSummarizeContext)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     @retry(stop=stop_after_attempt(2), wait=wait_random_exponential(min=1, max=60))
     async def summarize_code(self, prompt):
@@ -101,11 +106,10 @@ class SummarizeCode(Action):
         design_doc = await self.repo.docs.system_design.get(filename=design_pathname.name)
         task_pathname = Path(self.i_context.task_filename)
         task_doc = await self.repo.docs.task.get(filename=task_pathname.name)
-        src_file_repo = self.repo.with_src_path(self.context.src_workspace).srcs
         code_blocks = []
         for filename in self.i_context.codes_filenames:
-            code_doc = await src_file_repo.get(filename)
-            code_block = f"```python\n{code_doc.content}\n```\n-----"
+            code_doc = await self.repo.srcs.get(filename)
+            code_block = f"```{get_markdown_code_block_type(filename)}\n{code_doc.content}\n```\n---\n"
             code_blocks.append(code_block)
         format_example = FORMAT_EXAMPLE
         prompt = PROMPT_TEMPLATE.format(

metagpt/actions/talk_action.py

@@ -9,7 +9,6 @@
 from typing import Optional
 
 from metagpt.actions import Action
-from metagpt.config2 import config
 from metagpt.logs import logger
 from metagpt.schema import Message
 
@@ -26,7 +25,7 @@ class TalkAction(Action):
 
     @property
     def language(self):
-        return self.context.kwargs.language or config.language
+        return self.context.kwargs.language or self.config.language
 
     @property
     def prompt(self):

metagpt/actions/write_code.py

@@ -16,18 +16,20 @@
 """
 
 import json
+from pathlib import Path
+from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions.action import Action
 from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST
 from metagpt.actions.write_code_plan_and_change_an import REFINED_TEMPLATE
-from metagpt.const import BUGFIX_FILENAME, REQUIREMENT_FILENAME
 from metagpt.logs import logger
 from metagpt.schema import CodingContext, Document, RunCodeResult
-from metagpt.utils.common import CodeParser
+from metagpt.utils.common import CodeParser, get_markdown_code_block_type
 from metagpt.utils.project_repo import ProjectRepo
+from metagpt.utils.report import EditorReporter
 
 PROMPT_TEMPLATE = """
 NOTICE
@@ -43,9 +45,7 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
 {task}
 
 ## Legacy Code
-```Code
 {code}
-```
 
 ## Debug logs
 ```text
@@ -60,9 +60,14 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
 ```
 
 # Format example
-## Code: {filename}
+## Code: {demo_filename}.py
 ```python
-## {filename}
+## {demo_filename}.py
+...
+```
+## Code: {demo_filename}.js
+```javascript
+// {demo_filename}.js
 ...
 ```
 
@@ -83,18 +88,26 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
 class WriteCode(Action):
     name: str = "WriteCode"
     i_context: Document = Field(default_factory=Document)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
     async def write_code(self, prompt) -> str:
         code_rsp = await self._aask(prompt)
-        code = CodeParser.parse_code(block="", text=code_rsp)
+        code = CodeParser.parse_code(text=code_rsp)
         return code
 
     async def run(self, *args, **kwargs) -> CodingContext:
-        bug_feedback = await self.repo.docs.get(filename=BUGFIX_FILENAME)
+        bug_feedback = None
+        if self.input_args and hasattr(self.input_args, "issue_filename"):
+            bug_feedback = await Document.load(self.input_args.issue_filename)
         coding_context = CodingContext.loads(self.i_context.content)
+        if not coding_context.code_plan_and_change_doc:
+            coding_context.code_plan_and_change_doc = await self.repo.docs.code_plan_and_change.get(
+                filename=coding_context.task_doc.filename
+            )
         test_doc = await self.repo.test_outputs.get(filename="test_" + coding_context.filename + ".json")
-        requirement_doc = await self.repo.docs.get(filename=REQUIREMENT_FILENAME)
+        requirement_doc = await Document.load(self.input_args.requirements_filename)
         summary_doc = None
         if coding_context.design_doc and coding_context.design_doc.filename:
             summary_doc = await self.repo.docs.code_summary.get(filename=coding_context.design_doc.filename)
@@ -103,29 +116,28 @@ class WriteCode(Action):
             test_detail = RunCodeResult.loads(test_doc.content)
             logs = test_detail.stderr
 
-        if bug_feedback:
-            code_context = coding_context.code_doc.content
-        elif self.config.inc:
+        if self.config.inc or bug_feedback:
             code_context = await self.get_codes(
                 coding_context.task_doc, exclude=self.i_context.filename, project_repo=self.repo, use_inc=True
             )
         else:
             code_context = await self.get_codes(
-                coding_context.task_doc,
-                exclude=self.i_context.filename,
-                project_repo=self.repo.with_src_path(self.context.src_workspace),
+                coding_context.task_doc, exclude=self.i_context.filename, project_repo=self.repo
             )
 
         if self.config.inc:
             prompt = REFINED_TEMPLATE.format(
                 user_requirement=requirement_doc.content if requirement_doc else "",
-                code_plan_and_change=str(coding_context.code_plan_and_change_doc),
+                code_plan_and_change=coding_context.code_plan_and_change_doc.content
+                if coding_context.code_plan_and_change_doc
+                else "",
                 design=coding_context.design_doc.content if coding_context.design_doc else "",
                 task=coding_context.task_doc.content if coding_context.task_doc else "",
                 code=code_context,
                 logs=logs,
                 feedback=bug_feedback.content if bug_feedback else "",
                 filename=self.i_context.filename,
+                demo_filename=Path(self.i_context.filename).stem,
                 summary_log=summary_doc.content if summary_doc else "",
             )
         else:
@@ -136,15 +148,20 @@ class WriteCode(Action):
                 logs=logs,
                 feedback=bug_feedback.content if bug_feedback else "",
                 filename=self.i_context.filename,
+                demo_filename=Path(self.i_context.filename).stem,
                 summary_log=summary_doc.content if summary_doc else "",
             )
         logger.info(f"Writing {coding_context.filename}..")
-        code = await self.write_code(prompt)
-        if not coding_context.code_doc:
-            # avoid root_path pydantic ValidationError if use WriteCode alone
-            root_path = self.context.src_workspace if self.context.src_workspace else ""
-            coding_context.code_doc = Document(filename=coding_context.filename, root_path=str(root_path))
-        coding_context.code_doc.content = code
+        async with EditorReporter(enable_llm_stream=True) as reporter:
+            await reporter.async_report({"type": "code", "filename": coding_context.filename}, "meta")
+            code = await self.write_code(prompt)
+            if not coding_context.code_doc:
+                # avoid root_path pydantic ValidationError if use WriteCode alone
+                coding_context.code_doc = Document(
+                    filename=coding_context.filename, root_path=str(self.repo.src_relative_path)
+                )
+            coding_context.code_doc.content = code
+            await reporter.async_report(coding_context.code_doc, "document")
         return coding_context
 
     @staticmethod
@@ -169,35 +186,32 @@ class WriteCode(Action):
         code_filenames = m.get(TASK_LIST.key, []) if not use_inc else m.get(REFINED_TASK_LIST.key, [])
         codes = []
         src_file_repo = project_repo.srcs
-
         # Incremental development scenario
         if use_inc:
-            src_files = src_file_repo.all_files
-            # Get the old workspace contained the old codes and old workspace are created in previous CodePlanAndChange
-            old_file_repo = project_repo.git_repo.new_file_repository(relative_path=project_repo.old_workspace)
-            old_files = old_file_repo.all_files
-            # Get the union of the files in the src and old workspaces
-            union_files_list = list(set(src_files) | set(old_files))
-            for filename in union_files_list:
+            for filename in src_file_repo.all_files:
+                code_block_type = get_markdown_code_block_type(filename)
                 # Exclude the current file from the all code snippets
                 if filename == exclude:
                     # If the file is in the old workspace, use the old code
                     # Exclude unnecessary code to maintain a clean and focused main.py file, ensuring only relevant and
                     # essential functionality is included for the project’s requirements
-                    if filename in old_files and filename != "main.py":
+                    if filename != "main.py":
                         # Use old code
-                        doc = await old_file_repo.get(filename=filename)
+                        doc = await src_file_repo.get(filename=filename)
                     # If the file is in the src workspace, skip it
                     else:
                         continue
-                    codes.insert(0, f"-----Now, {filename} to be rewritten\n```{doc.content}```\n=====")
+                    codes.insert(
+                        0, f"### The name of file to rewrite: `{filename}`\n```{code_block_type}\n{doc.content}```\n"
+                    )
+                    logger.info(f"Prepare to rewrite `{filename}`")
                 # The code snippets are generated from the src workspace
                 else:
                     doc = await src_file_repo.get(filename=filename)
                     # If the file does not exist in the src workspace, skip it
                     if not doc:
                         continue
-                    codes.append(f"----- {filename}\n```{doc.content}```")
+                    codes.append(f"### File Name: `{filename}`\n```{code_block_type}\n{doc.content}```\n\n")
 
         # Normal scenario
         else:
@@ -208,6 +222,7 @@ class WriteCode(Action):
                 doc = await src_file_repo.get(filename=filename)
                 if not doc:
                     continue
-                codes.append(f"----- {filename}\n```{doc.content}```")
+                code_block_type = get_markdown_code_block_type(filename)
+                codes.append(f"### File Name: `{filename}`\n```{code_block_type}\n{doc.content}```\n\n")
 
         return "\n".join(codes)

metagpt/actions/write_code_an_draft.py

@@ -578,7 +578,7 @@ class WriteCodeAN(Action):
 
     async def run(self, context):
         self.llm.system_prompt = "You are an outstanding engineer and can implement any code"
-        return await WRITE_MOVE_NODE.fill(context=context, llm=self.llm, schema="json")
+        return await WRITE_MOVE_NODE.fill(req=context, llm=self.llm, schema="json")
 
 
 async def main():

metagpt/actions/write_code_plan_and_change_an.py

@@ -5,15 +5,16 @@
 @Author  : mannaandpoem
 @File    : write_code_plan_and_change_an.py
 """
-import os
-from typing import List
+from typing import List, Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 
 from metagpt.actions.action import Action
 from metagpt.actions.action_node import ActionNode
 from metagpt.logs import logger
-from metagpt.schema import CodePlanAndChangeContext
+from metagpt.schema import CodePlanAndChangeContext, Document
+from metagpt.utils.common import get_markdown_code_block_type
+from metagpt.utils.project_repo import ProjectRepo
 
 DEVELOPMENT_PLAN = ActionNode(
     key="Development Plan",
@@ -162,9 +163,8 @@ Role: You are a professional engineer; The main goal is to complete incremental
 {task}
 
 ## Legacy Code
-```Code
 {code}
-```
+
 
 ## Debug logs
 ```text
@@ -179,9 +179,14 @@ Role: You are a professional engineer; The main goal is to complete incremental
 ```
 
 # Format example
-## Code: {filename}
+## Code: {demo_filename}.py
 ```python
-## {filename}
+## {demo_filename}.py
+...
+```
+## Code: {demo_filename}.js
+```javascript
+// {demo_filename}.js
 ...
 ```
 
@@ -206,13 +211,15 @@ WRITE_CODE_PLAN_AND_CHANGE_NODE = ActionNode.from_children("WriteCodePlanAndChan
 class WriteCodePlanAndChange(Action):
     name: str = "WriteCodePlanAndChange"
     i_context: CodePlanAndChangeContext = Field(default_factory=CodePlanAndChangeContext)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     async def run(self, *args, **kwargs):
         self.llm.system_prompt = "You are a professional software engineer, your primary responsibility is to "
         "meticulously craft comprehensive incremental development plan and deliver detailed incremental change"
-        prd_doc = await self.repo.docs.prd.get(filename=self.i_context.prd_filename)
-        design_doc = await self.repo.docs.system_design.get(filename=self.i_context.design_filename)
-        task_doc = await self.repo.docs.task.get(filename=self.i_context.task_filename)
+        prd_doc = await Document.load(filename=self.i_context.prd_filename)
+        design_doc = await Document.load(filename=self.i_context.design_filename)
+        task_doc = await Document.load(filename=self.i_context.task_filename)
         context = CODE_PLAN_AND_CHANGE_CONTEXT.format(
             requirement=f"```text\n{self.i_context.requirement}\n```",
             issue=f"```text\n{self.i_context.issue}\n```",
@@ -222,11 +229,12 @@ class WriteCodePlanAndChange(Action):
             code=await self.get_old_codes(),
         )
         logger.info("Writing code plan and change..")
-        return await WRITE_CODE_PLAN_AND_CHANGE_NODE.fill(context=context, llm=self.llm, schema="json")
+        return await WRITE_CODE_PLAN_AND_CHANGE_NODE.fill(req=context, llm=self.llm, schema="json")
 
     async def get_old_codes(self) -> str:
-        self.repo.old_workspace = self.repo.git_repo.workdir / os.path.basename(self.config.project_path)
-        old_file_repo = self.repo.git_repo.new_file_repository(relative_path=self.repo.old_workspace)
-        old_codes = await old_file_repo.get_all()
-        codes = [f"----- {code.filename}\n```{code.content}```" for code in old_codes]
+        old_codes = await self.repo.srcs.get_all()
+        codes = [
+            f"### File Name: `{code.filename}`\n```{get_markdown_code_block_type(code.filename)}\n{code.content}```\n"
+            for code in old_codes
+        ]
         return "\n".join(codes)

metagpt/actions/write_code_review.py

@@ -7,16 +7,22 @@
 @Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the
         WriteCode object, rather than passing them in when calling the run function.
 """
+import asyncio
+import os
+from pathlib import Path
+from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions import WriteCode
 from metagpt.actions.action import Action
-from metagpt.const import REQUIREMENT_FILENAME
 from metagpt.logs import logger
-from metagpt.schema import CodingContext
-from metagpt.utils.common import CodeParser
+from metagpt.schema import CodingContext, Document
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import CodeParser, aread, awrite
+from metagpt.utils.project_repo import ProjectRepo
+from metagpt.utils.report import EditorReporter
 
 PROMPT_TEMPLATE = """
 # System
@@ -119,34 +125,48 @@ LGTM
 
 REWRITE_CODE_TEMPLATE = """
 # Instruction: rewrite the `{filename}` based on the Code Review and Actions
-## Rewrite Code: CodeBlock. If it still has some bugs, rewrite {filename} with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Return all completed codes and prohibit the return of unfinished codes.
-```Code
+## Rewrite Code: CodeBlock. If it still has some bugs, rewrite {filename} using a Markdown code block, with the filename docstring preceding the code block. Do your utmost to optimize THIS SINGLE FILE. Return all completed codes and prohibit the return of unfinished codes.
+```python
 ## {filename}
 ...
 ```
+or
+```javascript
+// {filename}
+...
+```
 """
 
 
 class WriteCodeReview(Action):
     name: str = "WriteCodeReview"
     i_context: CodingContext = Field(default_factory=CodingContext)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
-    async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, filename):
+    async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, doc):
+        filename = doc.filename
         cr_rsp = await self._aask(context_prompt + cr_prompt)
         result = CodeParser.parse_block("Code Review Result", cr_rsp)
         if "LGTM" in result:
             return result, None
 
         # if LBTM, rewrite code
-        rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}"
-        code_rsp = await self._aask(rewrite_prompt)
-        code = CodeParser.parse_code(block="", text=code_rsp)
+        async with EditorReporter(enable_llm_stream=True) as reporter:
+            await reporter.async_report(
+                {"type": "code", "filename": filename, "src_path": doc.root_relative_path}, "meta"
+            )
+            rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}"
+            code_rsp = await self._aask(rewrite_prompt)
+            code = CodeParser.parse_code(text=code_rsp)
+            doc.content = code
+            await reporter.async_report(doc, "document")
         return result, code
 
     async def run(self, *args, **kwargs) -> CodingContext:
         iterative_code = self.i_context.code_doc.content
-        k = self.context.config.code_review_k_times or 1
+        k = self.context.config.code_validate_k_times or 1
 
         for i in range(k):
             format_example = FORMAT_EXAMPLE.format(filename=self.i_context.code_doc.filename)
@@ -154,7 +174,7 @@ class WriteCodeReview(Action):
             code_context = await WriteCode.get_codes(
                 self.i_context.task_doc,
                 exclude=self.i_context.filename,
-                project_repo=self.repo.with_src_path(self.context.src_workspace),
+                project_repo=self.repo,
                 use_inc=self.config.inc,
             )
 
@@ -164,7 +184,7 @@ class WriteCodeReview(Action):
                 "## Code Files\n" + code_context + "\n",
             ]
             if self.config.inc:
-                requirement_doc = await self.repo.docs.get(filename=REQUIREMENT_FILENAME)
+                requirement_doc = await Document.load(filename=self.input_args.requirements_filename)
                 insert_ctx_list = [
                     "## User New Requirements\n" + str(requirement_doc) + "\n",
                     "## Code Plan And Change\n" + str(self.i_context.code_plan_and_change_doc) + "\n",
@@ -187,7 +207,7 @@ class WriteCodeReview(Action):
                 f"len(self.i_context.code_doc.content)={len2}"
             )
             result, rewrited_code = await self.write_code_review_and_rewrite(
-                context_prompt, cr_prompt, self.i_context.code_doc.filename
+                context_prompt, cr_prompt, self.i_context.code_doc
             )
             if "LBTM" in result:
                 iterative_code = rewrited_code
@@ -199,3 +219,97 @@ class WriteCodeReview(Action):
         # 如果rewrited_code是None（原code perfect），那么直接返回code
         self.i_context.code_doc.content = iterative_code
         return self.i_context
+
+
+@register_tool(include_functions=["run"])
+class ValidateAndRewriteCode(Action):
+    """According to the design and task documents, validate the code to ensure it is complete and correct."""
+
+    name: str = "ValidateAndRewriteCode"
+
+    async def run(
+        self,
+        code_path: str,
+        system_design_input: str = "",
+        project_schedule_input: str = "",
+        code_validate_k_times: int = 2,
+    ) -> str:
+        """Validates the provided code based on the accompanying system design and project schedule documentation, return the complete and correct code.
+
+        Read the code from code_path, and write the final code to code_path.
+        If both system_design_input and project_schedule_input are absent, it will return and do nothing.
+
+        Args:
+            code_path (str): The file path of the code snippet to be validated. This should be a string containing the path to the source code file.
+            system_design_input (str): Content or file path of the design document associated with the code. This should describe the system architecture, used in the code. It helps provide context for the validation process.
+            project_schedule_input (str): Content or file path of the task document describing what the code is intended to accomplish. This should outline the functional requirements or objectives of the code.
+            code_validate_k_times (int, optional): The number of iterations for validating and potentially rewriting the code. Defaults to 2.
+
+        Returns:
+            str: The potentially corrected or approved code after validation.
+
+        Example Usage:
+            # Example of how to call the run method with a code snippet and documentation
+            await ValidateAndRewriteCode().run(
+                code_path="/tmp/game.js",
+                system_design_input="/tmp/system_design.json",
+                project_schedule_input="/tmp/project_task_list.json"
+            )
+        """
+        if not system_design_input and not project_schedule_input:
+            logger.info(
+                "Both `system_design_input` and `project_schedule_input` are absent, ValidateAndRewriteCode will do nothing."
+            )
+            return
+
+        code, design_doc, task_doc = await asyncio.gather(
+            aread(code_path), self._try_aread(system_design_input), self._try_aread(project_schedule_input)
+        )
+        code_doc = self._create_code_doc(code_path=code_path, code=code)
+        review_action = WriteCodeReview(i_context=CodingContext(filename=code_doc.filename))
+
+        context = "\n".join(
+            [
+                "## System Design\n" + design_doc + "\n",
+                "## Task\n" + task_doc + "\n",
+            ]
+        )
+
+        for i in range(code_validate_k_times):
+            context_prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=code_path)
+            cr_prompt = EXAMPLE_AND_INSTRUCTION.format(
+                format_example=FORMAT_EXAMPLE.format(filename=code_path),
+            )
+            logger.info(f"The {i+1}th time to CodeReview: {code_path}.")
+            result, rewrited_code = await review_action.write_code_review_and_rewrite(
+                context_prompt, cr_prompt, doc=code_doc
+            )
+
+            if "LBTM" in result:
+                code = rewrited_code
+            elif "LGTM" in result:
+                break
+
+        await awrite(filename=code_path, data=code)
+
+        return (
+            f"The review and rewriting of the code in the file '{os.path.basename(code_path)}' has been completed."
+            + code
+        )
+
+    @staticmethod
+    async def _try_aread(input: str) -> str:
+        """Try to read from the path if it's a file; return input directly if not."""
+
+        if os.path.exists(input):
+            return await aread(input)
+
+        return input
+
+    @staticmethod
+    def _create_code_doc(code_path: str, code: str) -> Document:
+        """Create a Document to represent the code doc."""
+
+        path = Path(code_path)
+
+        return Document(root_path=str(path.parent), filename=path.name, content=code)

metagpt/actions/write_prd.py

@@ -9,12 +9,16 @@
             2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality.
             3. Move the document storage operations related to WritePRD from the save operation of WriteDesign.
 @Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
+@Modified By: mashenquan, 2024/5/31. Implement Chapter 3 of RFC 236.
 """
 
 from __future__ import annotations
 
 import json
 from pathlib import Path
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, Field
 
 from metagpt.actions import Action, ActionOutput
 from metagpt.actions.action_node import ActionNode
@@ -33,10 +37,20 @@ from metagpt.const import (
     REQUIREMENT_FILENAME,
 )
 from metagpt.logs import logger
-from metagpt.schema import BugFixContext, Document, Documents, Message
-from metagpt.utils.common import CodeParser
+from metagpt.schema import AIMessage, Document, Documents, Message
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import (
+    CodeParser,
+    aread,
+    awrite,
+    rectify_pathname,
+    save_json_to_markdown,
+    to_markdown_code_block,
+)
 from metagpt.utils.file_repository import FileRepository
 from metagpt.utils.mermaid import mermaid_to_file
+from metagpt.utils.project_repo import ProjectRepo
+from metagpt.utils.report import DocsReporter, GalleryReporter
 
 CONTEXT_TEMPLATE = """
 ### Project Name
@@ -58,6 +72,7 @@ NEW_REQ_TEMPLATE = """
 """
 
 
+@register_tool(include_functions=["run"])
 class WritePRD(Action):
     """WritePRD deal with the following situations:
     1. Bugfix: If the requirement is a bugfix, the bugfix document will be generated.
@@ -65,10 +80,79 @@ class WritePRD(Action):
     3. Requirement update: If the requirement is an update, the PRD document will be updated.
     """
 
-    async def run(self, with_messages, *args, **kwargs) -> ActionOutput | Message:
-        """Run the action."""
-        req: Document = await self.repo.requirement
-        docs: list[Document] = await self.repo.docs.prd.get_all()
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
+
+    async def run(
+        self,
+        with_messages: List[Message] = None,
+        *,
+        user_requirement: str = "",
+        output_pathname: str = "",
+        legacy_prd_filename: str = "",
+        extra_info: str = "",
+        **kwargs,
+    ) -> Union[AIMessage, str]:
+        """
+        Write a Product Requirement Document.
+
+        Args:
+            user_requirement (str): A string detailing the user's requirements.
+            output_pathname (str, optional): The output file path of the document. Defaults to "".
+            legacy_prd_filename (str, optional): The file path of the legacy Product Requirement Document to use as a reference. Defaults to "".
+            extra_info (str, optional): Additional information to include in the document. Defaults to "".
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            str: The file path of the generated Product Requirement Document.
+
+        Example:
+            # Write a new PRD (Product Requirement Document)
+            >>> user_requirement = "Write a snake game"
+            >>> output_pathname = "snake_game/docs/prd.json"
+            >>> extra_info = "YOUR EXTRA INFO, if any"
+            >>> write_prd = WritePRD()
+            >>> result = await write_prd.run(user_requirement=user_requirement, output_pathname=output_pathname, extra_info=extra_info)
+            >>> print(result)
+            PRD filename: "/absolute/path/to/snake_game/docs/prd.json"
+
+            # Rewrite an existing PRD (Product Requirement Document) and save to a new path.
+            >>> user_requirement = "Write PRD for a snake game, include new features such as a web UI"
+            >>> legacy_prd_filename = "/absolute/path/to/snake_game/docs/prd.json"
+            >>> output_pathname = "/absolute/path/to/snake_game/docs/prd_new.json"
+            >>> extra_info = "YOUR EXTRA INFO, if any"
+            >>> write_prd = WritePRD()
+            >>> result = await write_prd.run(user_requirement=user_requirement, legacy_prd_filename=legacy_prd_filename, extra_info=extra_info)
+            >>> print(result)
+            PRD filename: "/absolute/path/to/snake_game/docs/prd_new.json"
+        """
+        if not with_messages:
+            return await self._execute_api(
+                user_requirement=user_requirement,
+                output_pathname=output_pathname,
+                legacy_prd_filename=legacy_prd_filename,
+                extra_info=extra_info,
+            )
+
+        self.input_args = with_messages[-1].instruct_content
+        if not self.input_args:
+            self.repo = ProjectRepo(self.context.kwargs.project_path)
+            await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[-1].content)
+            self.input_args = AIMessage.create_instruct_value(
+                kvs={
+                    "project_path": self.context.kwargs.project_path,
+                    "requirements_filename": str(self.repo.docs.workdir / REQUIREMENT_FILENAME),
+                    "prd_filenames": [str(self.repo.docs.prd.workdir / i) for i in self.repo.docs.prd.all_files],
+                },
+                class_name="PrepareDocumentsOutput",
+            )
+        else:
+            self.repo = ProjectRepo(self.input_args.project_path)
+        req = await Document.load(filename=self.input_args.requirements_filename)
+        docs: list[Document] = [
+            await Document.load(filename=i, project_path=self.repo.workdir) for i in self.input_args.prd_filenames
+        ]
+
         if not req:
             raise FileNotFoundError("No requirement document found.")
 
@@ -81,49 +165,80 @@ class WritePRD(Action):
         # if requirement is related to other documents, update them, otherwise create a new one
         if related_docs := await self.get_related_docs(req, docs):
             logger.info(f"Requirement update detected: {req.content}")
-            return await self._handle_requirement_update(req, related_docs)
+            await self._handle_requirement_update(req=req, related_docs=related_docs)
         else:
             logger.info(f"New requirement detected: {req.content}")
-            return await self._handle_new_requirement(req)
+            await self._handle_new_requirement(req)
 
-    async def _handle_bugfix(self, req: Document) -> Message:
+        kvs = self.input_args.model_dump()
+        kvs["changed_prd_filenames"] = [
+            str(self.repo.docs.prd.workdir / i) for i in list(self.repo.docs.prd.changed_files.keys())
+        ]
+        kvs["project_path"] = str(self.repo.workdir)
+        kvs["requirements_filename"] = str(self.repo.docs.workdir / REQUIREMENT_FILENAME)
+        self.context.kwargs.project_path = str(self.repo.workdir)
+        return AIMessage(
+            content="PRD is completed. "
+            + "\n".join(
+                list(self.repo.docs.prd.changed_files.keys())
+                + list(self.repo.resources.prd.changed_files.keys())
+                + list(self.repo.resources.competitive_analysis.changed_files.keys())
+            ),
+            instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WritePRDOutput"),
+            cause_by=self,
+        )
+
+    async def _handle_bugfix(self, req: Document) -> AIMessage:
         # ... bugfix logic ...
         await self.repo.docs.save(filename=BUGFIX_FILENAME, content=req.content)
         await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content="")
-        bug_fix = BugFixContext(filename=BUGFIX_FILENAME)
-        return Message(
-            content=bug_fix.model_dump_json(),
-            instruct_content=bug_fix,
-            role="",
+        return AIMessage(
+            content=f"A new issue is received: {BUGFIX_FILENAME}",
             cause_by=FixBug,
-            sent_from=self,
+            instruct_content=AIMessage.create_instruct_value(
+                {
+                    "project_path": str(self.repo.workdir),
+                    "issue_filename": str(self.repo.docs.workdir / BUGFIX_FILENAME),
+                    "requirements_filename": str(self.repo.docs.workdir / REQUIREMENT_FILENAME),
+                },
+                class_name="IssueDetail",
+            ),
             send_to="Alex",  # the name of Engineer
         )
 
+    async def _new_prd(self, requirement: str) -> ActionNode:
+        project_name = self.project_name
+        context = CONTEXT_TEMPLATE.format(requirements=requirement, project_name=project_name)
+        exclude = [PROJECT_NAME.key] if project_name else []
+        node = await WRITE_PRD_NODE.fill(
+            req=context, llm=self.llm, exclude=exclude, schema=self.prompt_schema
+        )  # schema=schema
+        return node
+
     async def _handle_new_requirement(self, req: Document) -> ActionOutput:
         """handle new requirement"""
-        project_name = self.project_name
-        context = CONTEXT_TEMPLATE.format(requirements=req, project_name=project_name)
-        exclude = [PROJECT_NAME.key] if project_name else []
-        node = await WRITE_PRD_NODE.fill(context=context, llm=self.llm, exclude=exclude)  # schema=schema
-        await self._rename_workspace(node)
-        new_prd_doc = await self.repo.docs.prd.save(
-            filename=FileRepository.new_filename() + ".json", content=node.instruct_content.model_dump_json()
-        )
-        await self._save_competitive_analysis(new_prd_doc)
-        await self.repo.resources.prd.save_pdf(doc=new_prd_doc)
-        return Documents.from_iterable(documents=[new_prd_doc]).to_action_output()
+        async with DocsReporter(enable_llm_stream=True) as reporter:
+            await reporter.async_report({"type": "prd"}, "meta")
+            node = await self._new_prd(req.content)
+            await self._rename_workspace(node)
+            new_prd_doc = await self.repo.docs.prd.save(
+                filename=FileRepository.new_filename() + ".json", content=node.instruct_content.model_dump_json()
+            )
+            await self._save_competitive_analysis(new_prd_doc)
+            md = await self.repo.resources.prd.save_pdf(doc=new_prd_doc)
+            await reporter.async_report(self.repo.workdir / md.root_relative_path, "path")
+            return Documents.from_iterable(documents=[new_prd_doc]).to_action_output()
 
     async def _handle_requirement_update(self, req: Document, related_docs: list[Document]) -> ActionOutput:
         # ... requirement update logic ...
         for doc in related_docs:
-            await self._update_prd(req, doc)
+            await self._update_prd(req=req, prd_doc=doc)
         return Documents.from_iterable(documents=related_docs).to_action_output()
 
     async def _is_bugfix(self, context: str) -> bool:
         if not self.repo.code_files_exists():
             return False
-        node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm)
+        node = await WP_ISSUE_TYPE_NODE.fill(req=context, llm=self.llm)
         return node.get("issue_type") == "BUG"
 
     async def get_related_docs(self, req: Document, docs: list[Document]) -> list[Document]:
@@ -133,33 +248,39 @@ class WritePRD(Action):
 
     async def _is_related(self, req: Document, old_prd: Document) -> bool:
         context = NEW_REQ_TEMPLATE.format(old_prd=old_prd.content, requirements=req.content)
-        node = await WP_IS_RELATIVE_NODE.fill(context, self.llm)
+        node = await WP_IS_RELATIVE_NODE.fill(req=context, llm=self.llm)
         return node.get("is_relative") == "YES"
 
     async def _merge(self, req: Document, related_doc: Document) -> Document:
         if not self.project_name:
             self.project_name = Path(self.project_path).name
         prompt = NEW_REQ_TEMPLATE.format(requirements=req.content, old_prd=related_doc.content)
-        node = await REFINED_PRD_NODE.fill(context=prompt, llm=self.llm, schema=self.prompt_schema)
+        node = await REFINED_PRD_NODE.fill(req=prompt, llm=self.llm, schema=self.prompt_schema)
         related_doc.content = node.instruct_content.model_dump_json()
         await self._rename_workspace(node)
         return related_doc
 
     async def _update_prd(self, req: Document, prd_doc: Document) -> Document:
-        new_prd_doc: Document = await self._merge(req, prd_doc)
-        await self.repo.docs.prd.save_doc(doc=new_prd_doc)
-        await self._save_competitive_analysis(new_prd_doc)
-        await self.repo.resources.prd.save_pdf(doc=new_prd_doc)
+        async with DocsReporter(enable_llm_stream=True) as reporter:
+            await reporter.async_report({"type": "prd"}, "meta")
+            new_prd_doc: Document = await self._merge(req=req, related_doc=prd_doc)
+            await self.repo.docs.prd.save_doc(doc=new_prd_doc)
+            await self._save_competitive_analysis(new_prd_doc)
+            md = await self.repo.resources.prd.save_pdf(doc=new_prd_doc)
+            await reporter.async_report(self.repo.workdir / md.root_relative_path, "path")
         return new_prd_doc
 
-    async def _save_competitive_analysis(self, prd_doc: Document):
+    async def _save_competitive_analysis(self, prd_doc: Document, output_filename: Path = None):
         m = json.loads(prd_doc.content)
         quadrant_chart = m.get(COMPETITIVE_QUADRANT_CHART.key)
         if not quadrant_chart:
             return
-        pathname = self.repo.workdir / COMPETITIVE_ANALYSIS_FILE_REPO / Path(prd_doc.filename).stem
+        pathname = output_filename or self.repo.workdir / COMPETITIVE_ANALYSIS_FILE_REPO / Path(prd_doc.filename).stem
         pathname.parent.mkdir(parents=True, exist_ok=True)
         await mermaid_to_file(self.config.mermaid.engine, quadrant_chart, pathname)
+        image_path = pathname.parent / f"{pathname.name}.svg"
+        if image_path.exists():
+            await GalleryReporter().async_report(image_path, "path")
 
     async def _rename_workspace(self, prd):
         if not self.project_name:
@@ -169,4 +290,36 @@ class WritePRD(Action):
                 ws_name = CodeParser.parse_str(block="Project Name", text=prd)
             if ws_name:
                 self.project_name = ws_name
-        self.repo.git_repo.rename_root(self.project_name)
+        if self.repo:
+            self.repo.git_repo.rename_root(self.project_name)
+
+    async def _execute_api(
+        self, user_requirement: str, output_pathname: str, legacy_prd_filename: str, extra_info: str
+    ) -> str:
+        content = "#### User Requirements\n{user_requirement}\n#### Extra Info\n{extra_info}\n".format(
+            user_requirement=to_markdown_code_block(val=user_requirement),
+            extra_info=to_markdown_code_block(val=extra_info),
+        )
+        async with DocsReporter(enable_llm_stream=True) as reporter:
+            await reporter.async_report({"type": "prd"}, "meta")
+            req = Document(content=content)
+            if not legacy_prd_filename:
+                node = await self._new_prd(requirement=req.content)
+                new_prd = Document(content=node.instruct_content.model_dump_json())
+            else:
+                content = await aread(filename=legacy_prd_filename)
+                old_prd = Document(content=content)
+                new_prd = await self._merge(req=req, related_doc=old_prd)
+
+            if not output_pathname:
+                output_pathname = self.config.workspace.path / "docs" / "prd.json"
+            elif not Path(output_pathname).is_absolute():
+                output_pathname = self.config.workspace.path / output_pathname
+            output_pathname = rectify_pathname(path=output_pathname, default_filename="prd.json")
+            await awrite(filename=output_pathname, data=new_prd.content)
+            competitive_analysis_filename = output_pathname.parent / f"{output_pathname.stem}-competitive-analysis"
+            await self._save_competitive_analysis(prd_doc=new_prd, output_filename=Path(competitive_analysis_filename))
+            md_output_filename = output_pathname.with_suffix(".md")
+            await save_json_to_markdown(content=new_prd.content, output_filename=md_output_filename)
+            await reporter.async_report(md_output_filename, "path")
+        return f'PRD filename: "{str(output_pathname)}". The  product requirement document (PRD) has been completed.'

metagpt/actions/write_prd_an.py

@@ -5,7 +5,7 @@
 @Author  : alexanderwu
 @File    : write_prd_an.py
 """
-from typing import List
+from typing import List, Union
 
 from metagpt.actions.action_node import ActionNode
 
@@ -19,8 +19,8 @@ LANGUAGE = ActionNode(
 PROGRAMMING_LANGUAGE = ActionNode(
     key="Programming Language",
     expected_type=str,
-    instruction="Python/JavaScript or other mainstream programming language.",
-    example="Python",
+    instruction="Mainstream programming language. If not specified in the requirements, use Vite, React, MUI, Tailwind CSS.",
+    example="Vite, React, MUI, Tailwind CSS",
 )
 
 ORIGINAL_REQUIREMENTS = ActionNode(
@@ -132,7 +132,7 @@ REQUIREMENT_ANALYSIS = ActionNode(
 
 REFINED_REQUIREMENT_ANALYSIS = ActionNode(
     key="Refined Requirement Analysis",
-    expected_type=List[str],
+    expected_type=Union[List[str], str],
     instruction="Review and refine the existing requirement analysis into a string list to align with the evolving needs of the project "
     "due to incremental development. Ensure the analysis comprehensively covers the new features and enhancements "
     "required for the refined project scope.",
@@ -165,7 +165,7 @@ ANYTHING_UNCLEAR = ActionNode(
     key="Anything UNCLEAR",
     expected_type=str,
     instruction="Mention any aspects of the project that are unclear and try to clarify them.",
-    example="",
+    example="Currently, all aspects of the project are clear.",
 )
 
 ISSUE_TYPE = ActionNode(

metagpt/actions/write_review.py

@@ -36,4 +36,4 @@ class WriteReview(Action):
     name: str = "WriteReview"
 
     async def run(self, context):
-        return await WRITE_REVIEW_NODE.fill(context=context, llm=self.llm, schema="json")
+        return await WRITE_REVIEW_NODE.fill(req=context, llm=self.llm, schema="json")

metagpt/actions/write_test.py

@@ -45,7 +45,7 @@ class WriteTest(Action):
         code_rsp = await self._aask(prompt)
 
         try:
-            code = CodeParser.parse_code(block="", text=code_rsp)
+            code = CodeParser.parse_code(text=code_rsp)
         except Exception:
             # Handle the exception if needed
             logger.error(f"Can't parse the code: {code_rsp}")

metagpt/base/__init__.py

@@ -0,0 +1,8 @@
+from metagpt.base.base_env import BaseEnvironment
+from metagpt.base.base_role import BaseRole
+
+
+__all__ = [
+    "BaseEnvironment",
+    "BaseRole",
+]

metagpt/base/base_env.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : base environment
+
+import typing
+from abc import abstractmethod
+from typing import Any, Optional
+
+from metagpt.base.base_env_space import BaseEnvAction, BaseEnvObsParams
+from metagpt.base.base_serialization import BaseSerialization
+
+if typing.TYPE_CHECKING:
+    from metagpt.schema import Message
+
+
+class BaseEnvironment(BaseSerialization):
+    """Base environment"""
+
+    @abstractmethod
+    def reset(
+        self,
+        *,
+        seed: Optional[int] = None,
+        options: Optional[dict[str, Any]] = None,
+    ) -> tuple[dict[str, Any], dict[str, Any]]:
+        """Implement this to get init observation"""
+
+    @abstractmethod
+    def observe(self, obs_params: Optional[BaseEnvObsParams] = None) -> Any:
+        """Implement this if you want to get partial observation from the env"""
+
+    @abstractmethod
+    def step(self, action: BaseEnvAction) -> tuple[dict[str, Any], float, bool, bool, dict[str, Any]]:
+        """Implement this to feed a action and then get new observation from the env"""
+
+    @abstractmethod
+    def publish_message(self, message: "Message", peekable: bool = True) -> bool:
+        """Distribute the message to the recipients."""
+
+    @abstractmethod
+    async def run(self, k=1):
+        """Process all task at once"""

metagpt/base/base_role.py

@@ -0,0 +1,36 @@
+from abc import abstractmethod
+from typing import Optional, Union
+
+from metagpt.base.base_serialization import BaseSerialization
+
+
+class BaseRole(BaseSerialization):
+    """Abstract base class for all roles."""
+
+    name: str
+
+    @property
+    def is_idle(self) -> bool:
+        raise NotImplementedError
+
+    @abstractmethod
+    def think(self):
+        """Consider what to do and decide on the next course of action."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def act(self):
+        """Perform the current action."""
+        raise NotImplementedError
+
+    @abstractmethod
+    async def react(self) -> "Message":
+        """Entry to one of three strategies by which Role reacts to the observed Message."""
+
+    @abstractmethod
+    async def run(self, with_message: Optional[Union[str, "Message", list[str]]] = None) -> Optional["Message"]:
+        """Observe, and think and act based on the results of the observation."""
+
+    @abstractmethod
+    def get_memories(self, k: int = 0) -> list["Message"]:
+        """Return the most recent k memories of this role."""

metagpt/base/base_serialization.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, model_serializer, model_validator
+
+
+class BaseSerialization(BaseModel, extra="forbid"):
+    """
+    PolyMorphic subclasses Serialization / Deserialization Mixin
+    - First of all, we need to know that pydantic is not designed for polymorphism.
+    - If Engineer is subclass of Role, it would be serialized as Role. If we want to serialize it as Engineer, we need
+        to add `class name` to Engineer. So we need Engineer inherit SerializationMixin.
+
+    More details:
+    - https://docs.pydantic.dev/latest/concepts/serialization/
+    - https://github.com/pydantic/pydantic/discussions/7008 discuss about avoid `__get_pydantic_core_schema__`
+    """
+
+    __is_polymorphic_base = False
+    __subclasses_map__ = {}
+
+    @model_serializer(mode="wrap")
+    def __serialize_with_class_type__(self, default_serializer) -> Any:
+        # default serializer, then append the `__module_class_name` field and return
+        ret = default_serializer(self)
+        ret["__module_class_name"] = f"{self.__class__.__module__}.{self.__class__.__qualname__}"
+        return ret
+
+    @model_validator(mode="wrap")
+    @classmethod
+    def __convert_to_real_type__(cls, value: Any, handler):
+        if isinstance(value, dict) is False:
+            return handler(value)
+
+        # it is a dict so make sure to remove the __module_class_name
+        # because we don't allow extra keywords but want to ensure
+        # e.g Cat.model_validate(cat.model_dump()) works
+        class_full_name = value.pop("__module_class_name", None)
+
+        # if it's not the polymorphic base we construct via default handler
+        if not cls.__is_polymorphic_base:
+            if class_full_name is None:
+                return handler(value)
+            elif str(cls) == f"<class '{class_full_name}'>":
+                return handler(value)
+            else:
+                # f"Trying to instantiate {class_full_name} but this is not the polymorphic base class")
+                pass
+
+        # otherwise we lookup the correct polymorphic type and construct that
+        # instead
+        if class_full_name is None:
+            raise ValueError("Missing __module_class_name field")
+
+        class_type = cls.__subclasses_map__.get(class_full_name, None)
+
+        if class_type is None:
+            # TODO could try dynamic import
+            raise TypeError(f"Trying to instantiate {class_full_name}, which has not yet been defined!")
+
+        return class_type(**value)
+
+    def __init_subclass__(cls, is_polymorphic_base: bool = False, **kwargs):
+        cls.__is_polymorphic_base = is_polymorphic_base
+        cls.__subclasses_map__[f"{cls.__module__}.{cls.__qualname__}"] = cls
+        super().__init_subclass__(**kwargs)

metagpt/config2.py

@@ -9,14 +9,17 @@ import os
 from pathlib import Path
 from typing import Dict, Iterable, List, Literal, Optional
 
-from pydantic import BaseModel, model_validator
+from pydantic import BaseModel, Field, model_validator
 
 from metagpt.configs.browser_config import BrowserConfig
 from metagpt.configs.embedding_config import EmbeddingConfig
-from metagpt.configs.file_parser_config import OmniParseConfig
+from metagpt.configs.exp_pool_config import ExperiencePoolConfig
 from metagpt.configs.llm_config import LLMConfig, LLMType
 from metagpt.configs.mermaid_config import MermaidConfig
+from metagpt.configs.omniparse_config import OmniParseConfig
 from metagpt.configs.redis_config import RedisConfig
+from metagpt.configs.role_custom_config import RoleCustomConfig
+from metagpt.configs.role_zero_config import RoleZeroConfig
 from metagpt.configs.s3_config import S3Config
 from metagpt.configs.search_config import SearchConfig
 from metagpt.configs.workspace_config import WorkspaceConfig
@@ -60,6 +63,7 @@ class Config(CLIParams, YamlModel):
 
     # Tool Parameters
     search: SearchConfig = SearchConfig()
+    enable_search: bool = False
     browser: BrowserConfig = BrowserConfig()
     mermaid: MermaidConfig = MermaidConfig()
 
@@ -70,10 +74,12 @@ class Config(CLIParams, YamlModel):
     # Misc Parameters
     repair_llm_output: bool = False
     prompt_schema: Literal["json", "markdown", "raw"] = "json"
-    workspace: WorkspaceConfig = WorkspaceConfig()
+    workspace: WorkspaceConfig = Field(default_factory=WorkspaceConfig)
     enable_longterm_memory: bool = False
-    code_review_k_times: int = 2
-    agentops_api_key: str = ""
+    code_validate_k_times: int = 2
+
+    # Experience Pool Parameters
+    exp_pool: ExperiencePoolConfig = Field(default_factory=ExperiencePoolConfig)
 
     # Will be removed in the future
     metagpt_tti_url: str = ""
@@ -86,6 +92,12 @@ class Config(CLIParams, YamlModel):
     azure_tts_region: str = ""
     _extra: dict = dict()  # extra config dict
 
+    # Role's custom configuration
+    roles: Optional[List[RoleCustomConfig]] = None
+
+    # RoleZero's configuration
+    role_zero: RoleZeroConfig = Field(default_factory=RoleZeroConfig)
+
     @classmethod
     def from_home(cls, path):
         """Load config from ~/.metagpt/config2.yaml"""
@@ -95,20 +107,20 @@ class Config(CLIParams, YamlModel):
         return Config.from_yaml_file(pathname)
 
     @classmethod
-    def default(cls):
+    def default(cls, reload: bool = False, **kwargs) -> "Config":
         """Load default config
         - Priority: env < default_config_paths
         - Inside default_config_paths, the latter one overwrites the former one
         """
-        default_config_paths: List[Path] = [
+        default_config_paths = (
             METAGPT_ROOT / "config/config2.yaml",
             CONFIG_ROOT / "config2.yaml",
-        ]
-
-        dicts = [dict(os.environ)]
-        dicts += [Config.read_yaml(path) for path in default_config_paths]
-        final = merge_dict(dicts)
-        return Config(**final)
+        )
+        if reload or default_config_paths not in _CONFIG_CACHE:
+            dicts = [dict(os.environ), *(Config.read_yaml(path) for path in default_config_paths), kwargs]
+            final = merge_dict(dicts)
+            _CONFIG_CACHE[default_config_paths] = Config(**final)
+        return _CONFIG_CACHE[default_config_paths]
 
     @classmethod
     def from_llm_config(cls, llm_config: dict):
@@ -166,4 +178,5 @@ def merge_dict(dicts: Iterable[Dict]) -> Dict:
     return result
 
 
+_CONFIG_CACHE = {}
 config = Config.default()

metagpt/configs/browser_config.py

@@ -5,12 +5,23 @@
 @Author  : alexanderwu
 @File    : browser_config.py
 """
+from enum import Enum
 from typing import Literal
 
-from metagpt.tools import WebBrowserEngineType
 from metagpt.utils.yaml_model import YamlModel
 
 
+class WebBrowserEngineType(Enum):
+    PLAYWRIGHT = "playwright"
+    SELENIUM = "selenium"
+    CUSTOM = "custom"
+
+    @classmethod
+    def __missing__(cls, key):
+        """Default type conversion"""
+        return cls.CUSTOM
+
+
 class BrowserConfig(YamlModel):
     """Config for Browser"""
 

metagpt/configs/compress_msg_config.py

@@ -0,0 +1,32 @@
+from enum import Enum
+
+
+class CompressType(Enum):
+    """
+    Compression Type for messages. Used to compress messages under token limit.
+    - "": No compression. Default value.
+    - "post_cut_by_msg": Keep as many latest messages as possible.
+    - "post_cut_by_token": Keep as many latest messages as possible and truncate the earliest fit-in message.
+    - "pre_cut_by_msg": Keep as many earliest messages as possible.
+    - "pre_cut_by_token": Keep as many earliest messages as possible and truncate the latest fit-in message.
+    """
+
+    NO_COMPRESS = ""
+    POST_CUT_BY_MSG = "post_cut_by_msg"
+    POST_CUT_BY_TOKEN = "post_cut_by_token"
+    PRE_CUT_BY_MSG = "pre_cut_by_msg"
+    PRE_CUT_BY_TOKEN = "pre_cut_by_token"
+
+    def __missing__(self, key):
+        return self.NO_COMPRESS
+
+    @classmethod
+    def get_type(cls, type_name):
+        for member in cls:
+            if member.value == type_name:
+                return member
+        return cls.NO_COMPRESS
+
+    @classmethod
+    def cut_types(cls):
+        return [member for member in cls if "cut" in member.value]

metagpt/configs/exp_pool_config.py

@@ -0,0 +1,25 @@
+from enum import Enum
+
+from pydantic import Field
+
+from metagpt.utils.yaml_model import YamlModel
+
+
+class ExperiencePoolRetrievalType(Enum):
+    BM25 = "bm25"
+    CHROMA = "chroma"
+
+
+class ExperiencePoolConfig(YamlModel):
+    enabled: bool = Field(
+        default=False,
+        description="Flag to enable or disable the experience pool. When disabled, both reading and writing are ineffective.",
+    )
+    enable_read: bool = Field(default=False, description="Enable to read from experience pool.")
+    enable_write: bool = Field(default=False, description="Enable to write to experience pool.")
+    persist_path: str = Field(default=".chroma_exp_data", description="The persist path for experience pool.")
+    retrieval_type: ExperiencePoolRetrievalType = Field(
+        default=ExperiencePoolRetrievalType.BM25, description="The retrieval type for experience pool."
+    )
+    use_llm_ranker: bool = Field(default=True, description="Use LLM Reranker to get better result.")
+    collection_name: str = Field(default="experience_pool", description="The collection name in chromadb")

metagpt/configs/llm_config.py

@@ -11,6 +11,7 @@ from typing import Optional
 
 from pydantic import field_validator
 
+from metagpt.configs.compress_msg_config import CompressType
 from metagpt.const import CONFIG_ROOT, LLM_API_TIMEOUT, METAGPT_ROOT
 from metagpt.utils.yaml_model import YamlModel
 
@@ -35,6 +36,9 @@ class LLMType(Enum):
     MOONSHOT = "moonshot"
     MISTRAL = "mistral"
     YI = "yi"  # lingyiwanwu
+    OPEN_ROUTER = "open_router"
+    DEEPSEEK = "deepseek"
+    SILICONFLOW = "siliconflow"
     OPENROUTER = "openrouter"
     OPENROUTER_REASONING = "openrouter_reasoning"
     BEDROCK = "bedrock"
@@ -98,6 +102,9 @@ class LLMConfig(YamlModel):
     # Cost Control
     calc_usage: bool = True
 
+    # Compress request messages under token limit
+    compress_type: CompressType = CompressType.NO_COMPRESS
+
     # For Messages Control
     use_system_prompt: bool = True
 

metagpt/configs/omniparse_config.py

@@ -4,3 +4,4 @@ from metagpt.utils.yaml_model import YamlModel
 class OmniParseConfig(YamlModel):
     api_key: str = ""
     base_url: str = ""
+    timeout: int = 600

metagpt/configs/role_custom_config.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/4/22 16:33
+@Author  : Justin
+@File    : role_custom_config.py
+"""
+from metagpt.configs.llm_config import LLMConfig
+from metagpt.utils.yaml_model import YamlModel
+
+
+class RoleCustomConfig(YamlModel):
+    """custom config for roles
+    role: role's className or role's role_id
+    To be expanded
+    """
+
+    role: str = ""
+    llm: LLMConfig

metagpt/configs/role_zero_config.py

@@ -0,0 +1,11 @@
+from pydantic import Field
+
+from metagpt.utils.yaml_model import YamlModel
+
+
+class RoleZeroConfig(YamlModel):
+    enable_longterm_memory: bool = Field(default=False, description="Whether to use long-term memory.")
+    longterm_memory_persist_path: str = Field(default=".role_memory_data", description="The directory to save data.")
+    memory_k: int = Field(default=200, description="The capacity of short-term memory.")
+    similarity_top_k: int = Field(default=5, description="The number of long-term memories to retrieve.")
+    use_llm_ranker: bool = Field(default=False, description="Whether to use LLM Reranker to get better result.")

metagpt/configs/search_config.py

@@ -5,17 +5,28 @@
 @Author  : alexanderwu
 @File    : search_config.py
 """
+from enum import Enum
 from typing import Callable, Optional
 
-from pydantic import Field
+from pydantic import ConfigDict, Field
 
-from metagpt.tools import SearchEngineType
 from metagpt.utils.yaml_model import YamlModel
 
 
+class SearchEngineType(Enum):
+    SERPAPI_GOOGLE = "serpapi"
+    SERPER_GOOGLE = "serper"
+    DIRECT_GOOGLE = "google"
+    DUCK_DUCK_GO = "ddg"
+    CUSTOM_ENGINE = "custom"
+    BING = "bing"
+
+
 class SearchConfig(YamlModel):
     """Config for Search"""
 
+    model_config = ConfigDict(extra="allow")
+
     api_type: SearchEngineType = SearchEngineType.DUCK_DUCK_GO
     api_key: str = ""
     cse_id: str = ""  # for google

metagpt/const.py

@@ -12,12 +12,6 @@ import metagpt
 def get_metagpt_package_root():
     """Get the root directory of the installed package."""
     package_root = Path(metagpt.__file__).parent.parent
-    for i in (".git", ".project_root", ".gitignore"):
-        if (package_root / i).exists():
-            break
-    else:
-        package_root = Path.cwd()
-
     logger.info(f"Package root set to {str(package_root)}")
     return package_root
 
@@ -32,6 +26,12 @@ def get_metagpt_root():
     else:
         # Fallback to package root if no environment variable is set
         project_root = get_metagpt_package_root()
+        for i in (".git", ".project_root", ".gitignore"):
+            if (project_root / i).exists():
+                break
+        else:
+            project_root = Path.cwd()
+
     return project_root
 
 
@@ -65,6 +65,11 @@ SKILL_DIRECTORY = SOURCE_ROOT / "skills"
 TOOL_SCHEMA_PATH = METAGPT_ROOT / "metagpt/tools/schemas"
 TOOL_LIBS_PATH = METAGPT_ROOT / "metagpt/tools/libs"
 
+# TEMPLATE PATH
+TEMPLATE_FOLDER_PATH = METAGPT_ROOT / "template"
+VUE_TEMPLATE_PATH = TEMPLATE_FOLDER_PATH / "vue_template"
+REACT_TEMPLATE_PATH = TEMPLATE_FOLDER_PATH / "react_template"
+
 # REAL CONSTS
 
 MEM_TTL = 24 * 30 * 3600
@@ -75,6 +80,8 @@ MESSAGE_ROUTE_CAUSE_BY = "cause_by"
 MESSAGE_META_ROLE = "role"
 MESSAGE_ROUTE_TO_ALL = "<all>"
 MESSAGE_ROUTE_TO_NONE = "<none>"
+MESSAGE_ROUTE_TO_SELF = "<self>"  # Add this tag to replace `ActionOutput`
+
 
 REQUIREMENT_FILENAME = "requirement.txt"
 BUGFIX_FILENAME = "bugfix.txt"
@@ -97,12 +104,13 @@ TEST_OUTPUTS_FILE_REPO = "test_outputs"
 CODE_SUMMARIES_FILE_REPO = "docs/code_summary"
 CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summary"
 RESOURCES_FILE_REPO = "resources"
-SD_OUTPUT_FILE_REPO = "resources/sd_output"
+SD_OUTPUT_FILE_REPO = DEFAULT_WORKSPACE_ROOT
 GRAPH_REPO_FILE_REPO = "docs/graph_repo"
 VISUAL_GRAPH_REPO_FILE_REPO = "resources/graph_db"
 CLASS_VIEW_FILE_REPO = "docs/class_view"
 
 YAPI_URL = "http://yapi.deepwisdomai.com/"
+SD_URL = "http://172.31.0.51:49094"
 
 DEFAULT_LANGUAGE = "English"
 DEFAULT_MAX_TOKENS = 1500
@@ -129,3 +137,28 @@ AGGREGATION = "Aggregate"
 # Timeout
 USE_CONFIG_TIMEOUT = 0  # Using llm.timeout configuration.
 LLM_API_TIMEOUT = 300
+
+# Assistant alias
+ASSISTANT_ALIAS = "response"
+
+# Markdown
+MARKDOWN_TITLE_PREFIX = "## "
+
+# Reporter
+METAGPT_REPORTER_DEFAULT_URL = os.environ.get("METAGPT_REPORTER_URL", "")
+
+# Metadata defines
+AGENT = "agent"
+IMAGES = "images"
+
+# SWE agent
+SWE_SETUP_PATH = get_metagpt_package_root() / "metagpt/tools/swe_agent_commands/setup_default.sh"
+
+# experience pool
+EXPERIENCE_MASK = "<experience>"
+
+# TeamLeader's name
+TEAMLEADER_NAME = "Mike"
+
+DEFAULT_MIN_TOKEN_COUNT = 10000
+DEFAULT_MAX_TOKEN_COUNT = 100000000

metagpt/context.py

@@ -5,11 +5,12 @@
 @Author  : alexanderwu
 @File    : context.py
 """
+from __future__ import annotations
+
 import os
-from pathlib import Path
 from typing import Any, Dict, Optional
 
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, Field
 
 from metagpt.config2 import Config
 from metagpt.configs.llm_config import LLMConfig, LLMType
@@ -20,8 +21,6 @@ from metagpt.utils.cost_manager import (
     FireworksCostManager,
     TokenCostManager,
 )
-from metagpt.utils.git_repository import GitRepository
-from metagpt.utils.project_repo import ProjectRepo
 
 
 class AttrDict(BaseModel):
@@ -62,11 +61,8 @@ class Context(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     kwargs: AttrDict = AttrDict()
-    config: Config = Config.default()
+    config: Config = Field(default_factory=Config.default)
 
-    repo: Optional[ProjectRepo] = None
-    git_repo: Optional[GitRepository] = None
-    src_workspace: Optional[Path] = None
     cost_manager: CostManager = CostManager()
 
     _llm: Optional[BaseLLM] = None
@@ -110,7 +106,6 @@ class Context(BaseModel):
             Dict[str, Any]: A dictionary containing serialized data.
         """
         return {
-            "workdir": str(self.repo.workdir) if self.repo else "",
             "kwargs": {k: v for k, v in self.kwargs.__dict__.items()},
             "cost_manager": self.cost_manager.model_dump_json(),
         }
@@ -123,13 +118,6 @@ class Context(BaseModel):
         """
         if not serialized_data:
             return
-        workdir = serialized_data.get("workdir")
-        if workdir:
-            self.git_repo = GitRepository(local_path=workdir, auto_init=True)
-            self.repo = ProjectRepo(self.git_repo)
-            src_workspace = self.git_repo.workdir / self.git_repo.workdir.name
-            if src_workspace.exists():
-                self.src_workspace = src_workspace
         kwargs = serialized_data.get("kwargs")
         if kwargs:
             for k, v in kwargs.items():

metagpt/environment/android/env_space.py

@@ -10,7 +10,7 @@ import numpy.typing as npt
 from gymnasium import spaces
 from pydantic import ConfigDict, Field, field_validator
 
-from metagpt.environment.base_env_space import (
+from metagpt.base.base_env_space import (
     BaseEnvAction,
     BaseEnvActionType,
     BaseEnvObsParams,

metagpt/environment/base_env.py

@@ -5,25 +5,25 @@
 import asyncio
 from abc import abstractmethod
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Set, Union
+from typing import Any, Dict, Iterable, Optional, Set, Union
 
 from gymnasium import spaces
 from gymnasium.core import ActType, ObsType
 from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, model_validator
 
+from metagpt.base import BaseEnvironment, BaseRole
+from metagpt.base.base_env_space import BaseEnvAction, BaseEnvObsParams
 from metagpt.context import Context
 from metagpt.environment.api.env_api import (
     EnvAPIAbstract,
     ReadAPIRegistry,
     WriteAPIRegistry,
 )
-from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams
 from metagpt.logs import logger
+from metagpt.memory import Memory
 from metagpt.schema import Message
 from metagpt.utils.common import get_function_schema, is_coroutine_func, is_send_to
-
-if TYPE_CHECKING:
-    from metagpt.roles.role import Role  # noqa: F401
+from metagpt.utils.git_repository import GitRepository
 
 
 class EnvType(Enum):
@@ -50,7 +50,7 @@ def mark_as_writeable(func):
     return func
 
 
-class ExtEnv(BaseModel):
+class ExtEnv(BaseEnvironment, BaseModel):
     """External Env to integrate actual game environment"""
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -129,9 +129,9 @@ class Environment(ExtEnv):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     desc: str = Field(default="")  # 环境描述
-    roles: dict[str, SerializeAsAny["Role"]] = Field(default_factory=dict, validate_default=True)
-    member_addrs: Dict["Role", Set] = Field(default_factory=dict, exclude=True)
-    history: str = ""  # For debug
+    roles: dict[str, SerializeAsAny[BaseRole]] = Field(default_factory=dict, validate_default=True)
+    member_addrs: Dict[BaseRole, Set] = Field(default_factory=dict, exclude=True)
+    history: Memory = Field(default_factory=Memory)  # For debug
     context: Context = Field(default_factory=Context, exclude=True)
 
     def reset(
@@ -153,20 +153,20 @@ class Environment(ExtEnv):
         self.add_roles(self.roles.values())
         return self
 
-    def add_role(self, role: "Role"):
+    def add_role(self, role: BaseRole):
         """增加一个在当前环境的角色
         Add a role in the current environment
         """
-        self.roles[role.profile] = role
+        self.roles[role.name] = role
         role.set_env(self)
         role.context = self.context
 
-    def add_roles(self, roles: Iterable["Role"]):
+    def add_roles(self, roles: Iterable[BaseRole]):
         """增加一批在当前环境的角色
         Add a batch of characters in the current environment
         """
         for role in roles:
-            self.roles[role.profile] = role
+            self.roles[role.name] = role
 
         for role in roles:  # setup system message with roles
             role.context = self.context
@@ -190,7 +190,7 @@ class Environment(ExtEnv):
                 found = True
         if not found:
             logger.warning(f"Message no recipients: {message.dump()}")
-        self.history += f"\n{message}"  # For debug
+        self.history.add(message)  # For debug
 
         return True
 
@@ -201,19 +201,22 @@ class Environment(ExtEnv):
         for _ in range(k):
             futures = []
             for role in self.roles.values():
+                if role.is_idle:
+                    continue
                 future = role.run()
                 futures.append(future)
 
-            await asyncio.gather(*futures)
+            if futures:
+                await asyncio.gather(*futures)
             logger.debug(f"is idle: {self.is_idle}")
 
-    def get_roles(self) -> dict[str, "Role"]:
+    def get_roles(self) -> dict[str, BaseRole]:
         """获得环境内的所有角色
         Process all Role runs at once
         """
         return self.roles
 
-    def get_role(self, name: str) -> "Role":
+    def get_role(self, name: str) -> BaseRole:
         """获得环境内的指定角色
         get all the environment roles
         """
@@ -239,14 +242,6 @@ class Environment(ExtEnv):
         self.member_addrs[obj] = addresses
 
     def archive(self, auto_archive=True):
-        if auto_archive and self.context.git_repo:
-            self.context.git_repo.archive()
-
-    @classmethod
-    def model_rebuild(cls, **kwargs):
-        from metagpt.roles.role import Role  # noqa: F401
-
-        super().model_rebuild(**kwargs)
-
-
-Environment.model_rebuild()
+        if auto_archive and self.context.kwargs.get("project_path"):
+            git_repo = GitRepository(self.context.kwargs.project_path)
+            git_repo.archive()

metagpt/environment/mgx/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

metagpt/environment/mgx/mgx_env.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from metagpt.const import AGENT, IMAGES, MESSAGE_ROUTE_TO_ALL, TEAMLEADER_NAME
+from metagpt.environment.base_env import Environment
+from metagpt.logs import get_human_input
+from metagpt.roles import Role
+from metagpt.schema import Message, SerializationMixin
+from metagpt.utils.common import extract_and_encode_images
+
+
+class MGXEnv(Environment, SerializationMixin):
+    """MGX Environment"""
+
+    direct_chat_roles: set[str] = set()  # record direct chat: @role_name
+
+    is_public_chat: bool = True
+
+    def _publish_message(self, message: Message, peekable: bool = True) -> bool:
+        if self.is_public_chat:
+            message.send_to.add(MESSAGE_ROUTE_TO_ALL)
+        message = self.move_message_info_to_content(message)
+        return super().publish_message(message, peekable)
+
+    def publish_message(self, message: Message, user_defined_recipient: str = "", publicer: str = "") -> bool:
+        """let the team leader take over message publishing"""
+        message = self.attach_images(message)  # for multi-modal message
+
+        tl = self.get_role(TEAMLEADER_NAME)  # TeamLeader's name is Mike
+
+        if user_defined_recipient:
+            # human user's direct chat message to a certain role
+            for role_name in message.send_to:
+                if self.get_role(role_name).is_idle:
+                    # User starts a new direct chat with a certain role, expecting a direct chat response from the role; Other roles including TL should not be involved.
+                    # If the role is not idle, it means the user helps the role with its current work, in this case, we handle the role's response message as usual.
+                    self.direct_chat_roles.add(role_name)
+
+            self._publish_message(message)
+            # # bypass team leader, team leader only needs to know but not to react (commented out because TL doesn't understand the message well in actual experiments)
+            # tl.rc.memory.add(self.move_message_info_to_content(message))
+
+        elif message.sent_from in self.direct_chat_roles:
+            # if chat is not public, direct chat response from a certain role to human user, team leader and other roles in the env should not be involved, no need to publish
+            self.direct_chat_roles.remove(message.sent_from)
+            if self.is_public_chat:
+                self._publish_message(message)
+
+        elif publicer == tl.profile:
+            if message.send_to == {"no one"}:
+                # skip the dummy message from team leader
+                return True
+            # message processed by team leader can be published now
+            self._publish_message(message)
+
+        else:
+            # every regular message goes through team leader
+            message.send_to.add(tl.name)
+            self._publish_message(message)
+
+        self.history.add(message)
+
+        return True
+
+    async def ask_human(self, question: str, sent_from: Role = None) -> str:
+        # NOTE: Can be overwritten in remote setting
+        rsp = await get_human_input(question)
+        return "Human response: " + rsp
+
+    async def reply_to_human(self, content: str, sent_from: Role = None) -> str:
+        # NOTE: Can be overwritten in remote setting
+        return "SUCCESS, human has received your reply. Refrain from resending duplicate messages.  If you no longer need to take action, use the command ‘end’ to stop."
+
+    def move_message_info_to_content(self, message: Message) -> Message:
+        """Two things here:
+        1. Convert role, since role field must be reserved for LLM API, and is limited to, for example, one of ["user", "assistant", "system"]
+        2. Add sender and recipient info to content, making TL aware, since LLM API only takes content as input
+        """
+        converted_msg = message.model_copy(deep=True)
+        if converted_msg.role not in ["system", "user", "assistant"]:
+            converted_msg.role = "assistant"
+        sent_from = converted_msg.metadata[AGENT] if AGENT in converted_msg.metadata else converted_msg.sent_from
+        # When displaying send_to, change it to those who need to react and exclude those who only need to be aware, e.g.:
+        # send_to={<all>} -> Mike; send_to={Alice} -> Alice; send_to={Alice, <all>} -> Alice.
+        if converted_msg.send_to == {MESSAGE_ROUTE_TO_ALL}:
+            send_to = TEAMLEADER_NAME
+        else:
+            send_to = ", ".join({role for role in converted_msg.send_to if role != MESSAGE_ROUTE_TO_ALL})
+        converted_msg.content = f"[Message] from {sent_from or 'User'} to {send_to}: {converted_msg.content}"
+        return converted_msg
+
+    def attach_images(self, message: Message) -> Message:
+        if message.role == "user":
+            images = extract_and_encode_images(message.content)
+            if images:
+                message.add_metadata(IMAGES, images)
+        return message
+
+    def __repr__(self):
+        return "MGXEnv()"

metagpt/environment/minecraft/minecraft_env.py

@@ -11,7 +11,7 @@ from typing import Any, Iterable
 from llama_index.vector_stores.chroma import ChromaVectorStore
 from pydantic import ConfigDict, Field
 
-from metagpt.config2 import config as CONFIG
+from metagpt.config2 import Config
 from metagpt.environment.base_env import Environment
 from metagpt.environment.minecraft.const import MC_CKPT_DIR
 from metagpt.environment.minecraft.minecraft_ext_env import MinecraftExtEnv
@@ -82,7 +82,7 @@ class MinecraftEnv(MinecraftExtEnv, Environment):
             persist_dir=f"{MC_CKPT_DIR}/skill/vectordb",
         )
 
-        if CONFIG.resume:
+        if Config.default().resume:
             logger.info(f"Loading Action Developer from {MC_CKPT_DIR}/action")
             self.chest_memory = read_json_file(f"{MC_CKPT_DIR}/action/chest_memory.json")
 

metagpt/environment/minecraft/minecraft_ext_env.py

@@ -10,8 +10,8 @@ from typing import Any, Optional
 import requests
 from pydantic import ConfigDict, Field, model_validator
 
+from metagpt.base.base_env_space import BaseEnvAction, BaseEnvObsParams
 from metagpt.environment.base_env import ExtEnv, mark_as_writeable
-from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvObsParams
 from metagpt.environment.minecraft.const import (
     MC_CKPT_DIR,
     MC_CORE_INVENTORY_ITEMS,

metagpt/environment/stanford_town/env_space.py

@@ -9,7 +9,7 @@ import numpy.typing as npt
 from gymnasium import spaces
 from pydantic import ConfigDict, Field, field_validator
 
-from metagpt.environment.base_env_space import (
+from metagpt.base.base_env_space import (
     BaseEnvAction,
     BaseEnvActionType,
     BaseEnvObsParams,

metagpt/environment/werewolf/env_space.py

@@ -5,7 +5,7 @@
 from gymnasium import spaces
 from pydantic import ConfigDict, Field
 
-from metagpt.environment.base_env_space import BaseEnvAction, BaseEnvActionType
+from metagpt.base.base_env_space import BaseEnvAction, BaseEnvActionType
 from metagpt.environment.werewolf.const import STEP_INSTRUCTIONS
 
 

metagpt/environment/werewolf/werewolf_ext_env.py

@@ -8,8 +8,8 @@ from typing import Any, Callable, Optional
 
 from pydantic import ConfigDict, Field
 
+from metagpt.base.base_env_space import BaseEnvObsParams
 from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable
-from metagpt.environment.base_env_space import BaseEnvObsParams
 from metagpt.environment.werewolf.const import STEP_INSTRUCTIONS, RoleState, RoleType
 from metagpt.environment.werewolf.env_space import EnvAction, EnvActionType
 from metagpt.logs import logger

metagpt/exp_pool/__init__.py

@@ -0,0 +1,6 @@
+"""Experience pool init."""
+
+from metagpt.exp_pool.manager import get_exp_manager
+from metagpt.exp_pool.decorator import exp_cache
+
+__all__ = ["get_exp_manager", "exp_cache"]

metagpt/exp_pool/context_builders/__init__.py

@@ -0,0 +1,7 @@
+"""Context builders init."""
+
+from metagpt.exp_pool.context_builders.base import BaseContextBuilder
+from metagpt.exp_pool.context_builders.simple import SimpleContextBuilder
+from metagpt.exp_pool.context_builders.role_zero import RoleZeroContextBuilder
+
+__all__ = ["BaseContextBuilder", "SimpleContextBuilder", "RoleZeroContextBuilder"]

metagpt/exp_pool/context_builders/action_node.py

@@ -0,0 +1,30 @@
+"""Action Node context builder."""
+
+from typing import Any
+
+from metagpt.exp_pool.context_builders.base import BaseContextBuilder
+
+ACTION_NODE_CONTEXT_TEMPLATE = """
+{req}
+
+### Experiences
+-----
+{exps}
+-----
+
+## Instruction
+Consider **Experiences** to generate a better answer.
+"""
+
+
+class ActionNodeContextBuilder(BaseContextBuilder):
+    async def build(self, req: Any) -> str:
+        """Builds the action node context string.
+
+        If there are no experiences, returns the original `req`;
+        otherwise returns context with `req` and formatted experiences.
+        """
+
+        exps = self.format_exps()
+
+        return ACTION_NODE_CONTEXT_TEMPLATE.format(req=req, exps=exps) if exps else req

metagpt/exp_pool/context_builders/base.py

@@ -0,0 +1,41 @@
+"""Base context builder."""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+from metagpt.exp_pool.schema import Experience
+
+EXP_TEMPLATE = """Given the request: {req}, We can get the response: {resp}, which scored: {score}."""
+
+
+class BaseContextBuilder(BaseModel, ABC):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    exps: list[Experience] = []
+
+    @abstractmethod
+    async def build(self, req: Any) -> Any:
+        """Build context from req.
+
+        Do not modify `req`. If modification is necessary, use copy.deepcopy to create a copy first.
+        """
+
+    def format_exps(self) -> str:
+        """Format experiences into a numbered list of strings.
+
+        Example:
+            1. Given the request: req1, We can get the response: resp1, which scored: 8.
+            2. Given the request: req2, We can get the response: resp2, which scored: 9.
+
+        Returns:
+            str: The formatted experiences as a string.
+        """
+
+        result = []
+        for i, exp in enumerate(self.exps, start=1):
+            score_val = exp.metric.score.val if exp.metric and exp.metric.score else "N/A"
+            result.append(f"{i}. " + EXP_TEMPLATE.format(req=exp.req, resp=exp.resp, score=score_val))
+
+        return "\n".join(result)

metagpt/exp_pool/context_builders/role_zero.py

@@ -0,0 +1,39 @@
+"""RoleZero context builder."""
+
+import copy
+from typing import Any
+
+from metagpt.const import EXPERIENCE_MASK
+from metagpt.exp_pool.context_builders.base import BaseContextBuilder
+
+
+class RoleZeroContextBuilder(BaseContextBuilder):
+    async def build(self, req: Any) -> list[dict]:
+        """Builds the role zero context string.
+
+        Note:
+            1. The expected format for `req`, e.g., [{...}, {"role": "user", "content": "context"}].
+            2. Returns the original `req` if it is empty.
+            3. Creates a copy of req and replaces the example content in the copied req with actual experiences.
+        """
+
+        if not req:
+            return req
+
+        exps = self.format_exps()
+        if not exps:
+            return req
+
+        req_copy = copy.deepcopy(req)
+
+        req_copy[-1]["content"] = self.replace_example_content(req_copy[-1].get("content", ""), exps)
+
+        return req_copy
+
+    def replace_example_content(self, text: str, new_example_content: str) -> str:
+        return self.fill_experience(text, new_example_content)
+
+    @staticmethod
+    def fill_experience(text: str, new_example_content: str) -> str:
+        replaced_text = text.replace(EXPERIENCE_MASK, new_example_content)
+        return replaced_text

metagpt/exp_pool/context_builders/simple.py

@@ -0,0 +1,26 @@
+"""Simple context builder."""
+
+
+from typing import Any
+
+from metagpt.exp_pool.context_builders.base import BaseContextBuilder
+
+SIMPLE_CONTEXT_TEMPLATE = """
+## Context
+
+### Experiences
+-----
+{exps}
+-----
+
+## User Requirement
+{req}
+
+## Instruction
+Consider **Experiences** to generate a better answer.
+"""
+
+
+class SimpleContextBuilder(BaseContextBuilder):
+    async def build(self, req: Any) -> str:
+        return SIMPLE_CONTEXT_TEMPLATE.format(req=req, exps=self.format_exps())

metagpt/exp_pool/decorator.py

@@ -0,0 +1,227 @@
+"""Experience Decorator."""
+
+import asyncio
+import functools
+from typing import Any, Callable, Optional, TypeVar
+
+from pydantic import BaseModel, ConfigDict, model_validator
+
+from metagpt.config2 import config
+from metagpt.exp_pool.context_builders import BaseContextBuilder, SimpleContextBuilder
+from metagpt.exp_pool.manager import ExperienceManager, get_exp_manager
+from metagpt.exp_pool.perfect_judges import BasePerfectJudge, SimplePerfectJudge
+from metagpt.exp_pool.schema import (
+    LOG_NEW_EXPERIENCE_PREFIX,
+    Experience,
+    Metric,
+    QueryType,
+    Score,
+)
+from metagpt.exp_pool.scorers import BaseScorer, SimpleScorer
+from metagpt.exp_pool.serializers import BaseSerializer, SimpleSerializer
+from metagpt.logs import logger
+from metagpt.utils.async_helper import NestAsyncio
+from metagpt.utils.exceptions import handle_exception
+
+ReturnType = TypeVar("ReturnType")
+
+
+def exp_cache(
+    _func: Optional[Callable[..., ReturnType]] = None,
+    query_type: QueryType = QueryType.SEMANTIC,
+    manager: Optional[ExperienceManager] = None,
+    scorer: Optional[BaseScorer] = None,
+    perfect_judge: Optional[BasePerfectJudge] = None,
+    context_builder: Optional[BaseContextBuilder] = None,
+    serializer: Optional[BaseSerializer] = None,
+    tag: Optional[str] = None,
+):
+    """Decorator to get a perfect experience, otherwise, it executes the function, and create a new experience.
+
+    Note:
+        1. This can be applied to both synchronous and asynchronous functions.
+        2. The function must have a `req` parameter, and it must be provided as a keyword argument.
+        3. If `config.exp_pool.enabled` is False, the decorator will just directly execute the function.
+        4. If `config.exp_pool.enable_write` is False, the decorator will skip evaluating and saving the experience.
+        5. If `config.exp_pool.enable_read` is False, the decorator will skip reading from the experience pool.
+
+
+    Args:
+        _func: Just to make the decorator more flexible, for example, it can be used directly with @exp_cache by default, without the need for @exp_cache().
+        query_type: The type of query to be used when fetching experiences.
+        manager: How to fetch, evaluate and save experience, etc. Default to `exp_manager`.
+        scorer: Evaluate experience. Default to `SimpleScorer()`.
+        perfect_judge: Determines if an experience is perfect. Defaults to `SimplePerfectJudge()`.
+        context_builder: Build the context from exps and the function parameters. Default to `SimpleContextBuilder()`.
+        serializer: Serializes the request and the function's return value for storage, deserializes the stored response back to the function's return value. Defaults to `SimpleSerializer()`.
+        tag: An optional tag for the experience. Default to `ClassName.method_name` or `function_name`.
+    """
+
+    def decorator(func: Callable[..., ReturnType]) -> Callable[..., ReturnType]:
+        @functools.wraps(func)
+        async def get_or_create(args: Any, kwargs: Any) -> ReturnType:
+            if not config.exp_pool.enabled:
+                rsp = func(*args, **kwargs)
+                return await rsp if asyncio.iscoroutine(rsp) else rsp
+
+            handler = ExpCacheHandler(
+                func=func,
+                args=args,
+                kwargs=kwargs,
+                query_type=query_type,
+                exp_manager=manager,
+                exp_scorer=scorer,
+                exp_perfect_judge=perfect_judge,
+                context_builder=context_builder,
+                serializer=serializer,
+                tag=tag,
+            )
+
+            await handler.fetch_experiences()
+
+            if exp := await handler.get_one_perfect_exp():
+                return exp
+
+            await handler.execute_function()
+
+            if config.exp_pool.enable_write:
+                await handler.process_experience()
+
+            return handler._raw_resp
+
+        return ExpCacheHandler.choose_wrapper(func, get_or_create)
+
+    return decorator(_func) if _func else decorator
+
+
+class ExpCacheHandler(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    func: Callable
+    args: Any
+    kwargs: Any
+    query_type: QueryType = QueryType.SEMANTIC
+    exp_manager: Optional[ExperienceManager] = None
+    exp_scorer: Optional[BaseScorer] = None
+    exp_perfect_judge: Optional[BasePerfectJudge] = None
+    context_builder: Optional[BaseContextBuilder] = None
+    serializer: Optional[BaseSerializer] = None
+    tag: Optional[str] = None
+
+    _exps: list[Experience] = None
+    _req: str = ""
+    _resp: str = ""
+    _raw_resp: Any = None
+    _score: Score = None
+
+    @model_validator(mode="after")
+    def initialize(self):
+        """Initialize default values for optional parameters if they are None.
+
+        This is necessary because the decorator might pass None, which would override the default values set by Field.
+        """
+
+        self._validate_params()
+
+        self.exp_manager = self.exp_manager or get_exp_manager()
+        self.exp_scorer = self.exp_scorer or SimpleScorer()
+        self.exp_perfect_judge = self.exp_perfect_judge or SimplePerfectJudge()
+        self.context_builder = self.context_builder or SimpleContextBuilder()
+        self.serializer = self.serializer or SimpleSerializer()
+        self.tag = self.tag or self._generate_tag()
+
+        self._req = self.serializer.serialize_req(**self.kwargs)
+
+        return self
+
+    async def fetch_experiences(self):
+        """Fetch experiences by query_type."""
+
+        self._exps = await self.exp_manager.query_exps(self._req, query_type=self.query_type, tag=self.tag)
+        logger.info(f"Found {len(self._exps)} experiences for tag '{self.tag}'")
+
+    async def get_one_perfect_exp(self) -> Optional[Any]:
+        """Get a potentially perfect experience, and resolve resp."""
+
+        for exp in self._exps:
+            if await self.exp_perfect_judge.is_perfect_exp(exp, self._req, *self.args, **self.kwargs):
+                logger.info(f"Got one perfect experience for req '{exp.req[:20]}...'")
+                return self.serializer.deserialize_resp(exp.resp)
+
+        return None
+
+    async def execute_function(self):
+        """Execute the function, and save resp."""
+
+        self._raw_resp = await self._execute_function()
+        self._resp = self.serializer.serialize_resp(self._raw_resp)
+
+    @handle_exception
+    async def process_experience(self):
+        """Process experience.
+
+        Evaluates and saves experience.
+        Use `handle_exception` to ensure robustness, do not stop subsequent operations.
+        """
+
+        await self.evaluate_experience()
+        self.save_experience()
+
+    async def evaluate_experience(self):
+        """Evaluate the experience, and save the score."""
+
+        self._score = await self.exp_scorer.evaluate(self._req, self._resp)
+
+    def save_experience(self):
+        """Save the new experience."""
+
+        exp = Experience(req=self._req, resp=self._resp, tag=self.tag, metric=Metric(score=self._score))
+        self.exp_manager.create_exp(exp)
+        self._log_exp(exp)
+
+    @staticmethod
+    def choose_wrapper(func, wrapped_func):
+        """Choose how to run wrapped_func based on whether the function is asynchronous."""
+
+        async def async_wrapper(*args, **kwargs):
+            return await wrapped_func(args, kwargs)
+
+        def sync_wrapper(*args, **kwargs):
+            NestAsyncio.apply_once()
+            return asyncio.get_event_loop().run_until_complete(wrapped_func(args, kwargs))
+
+        return async_wrapper if asyncio.iscoroutinefunction(func) else sync_wrapper
+
+    def _validate_params(self):
+        if "req" not in self.kwargs:
+            raise ValueError("`req` must be provided as a keyword argument.")
+
+    def _generate_tag(self) -> str:
+        """Generates a tag for the self.func.
+
+        "ClassName.method_name" if the first argument is a class instance, otherwise just "function_name".
+        """
+
+        if self.args and hasattr(self.args[0], "__class__"):
+            cls_name = type(self.args[0]).__name__
+            return f"{cls_name}.{self.func.__name__}"
+
+        return self.func.__name__
+
+    async def _build_context(self) -> str:
+        self.context_builder.exps = self._exps
+
+        return await self.context_builder.build(self.kwargs["req"])
+
+    async def _execute_function(self):
+        self.kwargs["req"] = await self._build_context()
+
+        if asyncio.iscoroutinefunction(self.func):
+            return await self.func(*self.args, **self.kwargs)
+
+        return self.func(*self.args, **self.kwargs)
+
+    def _log_exp(self, exp: Experience):
+        log_entry = exp.model_dump_json(include={"uuid", "req", "resp", "tag"})
+
+        logger.debug(f"{LOG_NEW_EXPERIENCE_PREFIX}{log_entry}")

metagpt/exp_pool/manager.py

@@ -0,0 +1,242 @@
+"""Experience Manager."""
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from metagpt.config2 import Config
+from metagpt.configs.exp_pool_config import ExperiencePoolRetrievalType
+from metagpt.exp_pool.schema import DEFAULT_SIMILARITY_TOP_K, Experience, QueryType
+from metagpt.logs import logger
+from metagpt.utils.exceptions import handle_exception
+
+if TYPE_CHECKING:
+    from metagpt.rag.engines import SimpleEngine
+
+
+class ExperienceManager(BaseModel):
+    """ExperienceManager manages the lifecycle of experiences, including CRUD and optimization.
+
+    Args:
+        config (Config): Configuration for managing experiences.
+        _storage (SimpleEngine): Engine to handle the storage and retrieval of experiences.
+        _vector_store (ChromaVectorStore): The actual place where vectors are stored.
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    config: Config = Field(default_factory=Config.default)
+
+    _storage: Any = None
+
+    @property
+    def storage(self) -> "SimpleEngine":
+        if self._storage is None:
+            logger.info(f"exp_pool config: {self.config.exp_pool}")
+
+            self._storage = self._resolve_storage()
+
+        return self._storage
+
+    @storage.setter
+    def storage(self, value):
+        self._storage = value
+
+    @property
+    def is_readable(self) -> bool:
+        return self.config.exp_pool.enabled and self.config.exp_pool.enable_read
+
+    @is_readable.setter
+    def is_readable(self, value: bool):
+        self.config.exp_pool.enable_read = value
+
+        # If set to True, ensure that enabled is also True.
+        if value:
+            self.config.exp_pool.enabled = True
+
+    @property
+    def is_writable(self) -> bool:
+        return self.config.exp_pool.enabled and self.config.exp_pool.enable_write
+
+    @is_writable.setter
+    def is_writable(self, value: bool):
+        self.config.exp_pool.enable_write = value
+
+        # If set to True, ensure that enabled is also True.
+        if value:
+            self.config.exp_pool.enabled = True
+
+    @handle_exception
+    def create_exp(self, exp: Experience):
+        """Adds an experience to the storage if writing is enabled.
+
+        Args:
+            exp (Experience): The experience to add.
+        """
+
+        self.create_exps([exp])
+
+    @handle_exception
+    def create_exps(self, exps: list[Experience]):
+        """Adds multiple experiences to the storage if writing is enabled.
+
+        Args:
+            exps (list[Experience]): A list of experiences to add.
+        """
+        if not self.is_writable:
+            return
+
+        self.storage.add_objs(exps)
+        self.storage.persist(self.config.exp_pool.persist_path)
+
+    @handle_exception(default_return=[])
+    async def query_exps(self, req: str, tag: str = "", query_type: QueryType = QueryType.SEMANTIC) -> list[Experience]:
+        """Retrieves and filters experiences.
+
+        Args:
+            req (str): The query string to retrieve experiences.
+            tag (str): Optional tag to filter the experiences by.
+            query_type (QueryType): Default semantic to vector matching. exact to same matching.
+
+        Returns:
+            list[Experience]: A list of experiences that match the args.
+        """
+
+        if not self.is_readable:
+            return []
+
+        nodes = await self.storage.aretrieve(req)
+        exps: list[Experience] = [node.metadata["obj"] for node in nodes]
+
+        # TODO: filter by metadata
+        if tag:
+            exps = [exp for exp in exps if exp.tag == tag]
+
+        if query_type == QueryType.EXACT:
+            exps = [exp for exp in exps if exp.req == req]
+
+        return exps
+
+    @handle_exception
+    def delete_all_exps(self):
+        """Delete the all experiences."""
+
+        if not self.is_writable:
+            return
+
+        self.storage.clear(persist_dir=self.config.exp_pool.persist_path)
+
+    def get_exps_count(self) -> int:
+        """Get the total number of experiences."""
+
+        return self.storage.count()
+
+    def _resolve_storage(self) -> "SimpleEngine":
+        """Selects the appropriate storage creation method based on the configured retrieval type."""
+
+        storage_creators = {
+            ExperiencePoolRetrievalType.BM25: self._create_bm25_storage,
+            ExperiencePoolRetrievalType.CHROMA: self._create_chroma_storage,
+        }
+
+        return storage_creators[self.config.exp_pool.retrieval_type]()
+
+    def _create_bm25_storage(self) -> "SimpleEngine":
+        """Creates or loads BM25 storage.
+
+        This function attempts to create a new BM25 storage if the specified
+        document store path does not exist. If the path exists, it loads the
+        existing BM25 storage.
+
+        Returns:
+            SimpleEngine: An instance of SimpleEngine configured with BM25 storage.
+
+        Raises:
+            ImportError: If required modules are not installed.
+        """
+
+        try:
+            from metagpt.rag.engines import SimpleEngine
+            from metagpt.rag.schema import BM25IndexConfig, BM25RetrieverConfig
+        except ImportError:
+            raise ImportError("To use the experience pool, you need to install the rag module.")
+
+        persist_path = Path(self.config.exp_pool.persist_path)
+        docstore_path = persist_path / "docstore.json"
+
+        ranker_configs = self._get_ranker_configs()
+
+        if not docstore_path.exists():
+            logger.debug(f"Path `{docstore_path}` not exists, try to create a new bm25 storage.")
+            exps = [Experience(req="req", resp="resp")]
+
+            retriever_configs = [BM25RetrieverConfig(create_index=True, similarity_top_k=DEFAULT_SIMILARITY_TOP_K)]
+
+            storage = SimpleEngine.from_objs(
+                objs=exps, retriever_configs=retriever_configs, ranker_configs=ranker_configs
+            )
+            return storage
+
+        logger.debug(f"Path `{docstore_path}` exists, try to load bm25 storage.")
+        retriever_configs = [BM25RetrieverConfig(similarity_top_k=DEFAULT_SIMILARITY_TOP_K)]
+        storage = SimpleEngine.from_index(
+            BM25IndexConfig(persist_path=persist_path),
+            retriever_configs=retriever_configs,
+            ranker_configs=ranker_configs,
+        )
+
+        return storage
+
+    def _create_chroma_storage(self) -> "SimpleEngine":
+        """Creates Chroma storage.
+
+        Returns:
+            SimpleEngine: An instance of SimpleEngine configured with Chroma storage.
+
+        Raises:
+            ImportError: If required modules are not installed.
+        """
+
+        try:
+            from metagpt.rag.engines import SimpleEngine
+            from metagpt.rag.schema import ChromaRetrieverConfig
+        except ImportError:
+            raise ImportError("To use the experience pool, you need to install the rag module.")
+
+        retriever_configs = [
+            ChromaRetrieverConfig(
+                persist_path=self.config.exp_pool.persist_path,
+                collection_name=self.config.exp_pool.collection_name,
+                similarity_top_k=DEFAULT_SIMILARITY_TOP_K,
+            )
+        ]
+        ranker_configs = self._get_ranker_configs()
+
+        storage = SimpleEngine.from_objs(retriever_configs=retriever_configs, ranker_configs=ranker_configs)
+
+        return storage
+
+    def _get_ranker_configs(self):
+        """Returns ranker configurations based on the configuration.
+
+        If `use_llm_ranker` is True, returns a list with one `LLMRankerConfig`
+        instance. Otherwise, returns an empty list.
+
+        Returns:
+            list: A list of `LLMRankerConfig` instances or an empty list.
+        """
+
+        from metagpt.rag.schema import LLMRankerConfig
+
+        return [LLMRankerConfig(top_n=DEFAULT_SIMILARITY_TOP_K)] if self.config.exp_pool.use_llm_ranker else []
+
+
+_exp_manager = None
+
+
+def get_exp_manager() -> ExperienceManager:
+    global _exp_manager
+    if _exp_manager is None:
+        _exp_manager = ExperienceManager()
+    return _exp_manager

metagpt/exp_pool/perfect_judges/__init__.py

@@ -0,0 +1,6 @@
+"""Perfect judges init."""
+
+from metagpt.exp_pool.perfect_judges.base import BasePerfectJudge
+from metagpt.exp_pool.perfect_judges.simple import SimplePerfectJudge
+
+__all__ = ["BasePerfectJudge", "SimplePerfectJudge"]

metagpt/exp_pool/perfect_judges/base.py

@@ -0,0 +1,20 @@
+"""Base perfect judge."""
+
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel, ConfigDict
+
+from metagpt.exp_pool.schema import Experience
+
+
+class BasePerfectJudge(BaseModel, ABC):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @abstractmethod
+    async def is_perfect_exp(self, exp: Experience, serialized_req: str, *args, **kwargs) -> bool:
+        """Determine whether the experience is perfect.
+
+        Args:
+            exp (Experience): The experience to evaluate.
+            serialized_req (str): The serialized request to compare against the experience's request.
+        """

metagpt/exp_pool/perfect_judges/simple.py

@@ -0,0 +1,27 @@
+"""Simple perfect judge."""
+
+
+from pydantic import ConfigDict
+
+from metagpt.exp_pool.perfect_judges.base import BasePerfectJudge
+from metagpt.exp_pool.schema import MAX_SCORE, Experience
+
+
+class SimplePerfectJudge(BasePerfectJudge):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    async def is_perfect_exp(self, exp: Experience, serialized_req: str, *args, **kwargs) -> bool:
+        """Determine whether the experience is perfect.
+
+        Args:
+            exp (Experience): The experience to evaluate.
+            serialized_req (str): The serialized request to compare against the experience's request.
+
+        Returns:
+            bool: True if the serialized request matches the experience's request and the experience's score is perfect, False otherwise.
+        """
+
+        if not exp.metric or not exp.metric.score:
+            return False
+
+        return serialized_req == exp.req and exp.metric.score.val == MAX_SCORE

metagpt/exp_pool/schema.py

@@ -0,0 +1,76 @@
+"""Experience schema."""
+import time
+from enum import Enum
+from typing import Optional
+from uuid import UUID, uuid4
+
+from pydantic import BaseModel, Field
+
+MAX_SCORE = 10
+
+DEFAULT_SIMILARITY_TOP_K = 2
+
+LOG_NEW_EXPERIENCE_PREFIX = "New experience: "
+
+
+class QueryType(str, Enum):
+    """Type of query experiences."""
+
+    EXACT = "exact"
+    SEMANTIC = "semantic"
+
+
+class ExperienceType(str, Enum):
+    """Experience Type."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    INSIGHT = "insight"
+
+
+class EntryType(Enum):
+    """Experience Entry Type."""
+
+    AUTOMATIC = "Automatic"
+    MANUAL = "Manual"
+
+
+class Score(BaseModel):
+    """Score in Metric."""
+
+    val: int = Field(default=1, description="Value of the score, Between 1 and 10, higher is better.")
+    reason: str = Field(default="", description="Reason for the value.")
+
+
+class Metric(BaseModel):
+    """Experience Metric."""
+
+    time_cost: float = Field(default=0.000, description="Time cost, the unit is milliseconds.")
+    money_cost: float = Field(default=0.000, description="Money cost, the unit is US dollars.")
+    score: Score = Field(default=None, description="Score, with value and reason.")
+
+
+class Trajectory(BaseModel):
+    """Experience Trajectory."""
+
+    plan: str = Field(default="", description="The plan.")
+    action: str = Field(default="", description="Action for the plan.")
+    observation: str = Field(default="", description="Output of the action.")
+    reward: int = Field(default=0, description="Measure the action.")
+
+
+class Experience(BaseModel):
+    """Experience."""
+
+    req: str = Field(..., description="")
+    resp: str = Field(..., description="The type is string/json/code.")
+    metric: Optional[Metric] = Field(default=None, description="Metric.")
+    exp_type: ExperienceType = Field(default=ExperienceType.SUCCESS, description="The type of experience.")
+    entry_type: EntryType = Field(default=EntryType.AUTOMATIC, description="Type of entry: Manual or Automatic.")
+    tag: str = Field(default="", description="Tagging experience.")
+    traj: Optional[Trajectory] = Field(default=None, description="Trajectory.")
+    timestamp: Optional[float] = Field(default_factory=time.time)
+    uuid: Optional[UUID] = Field(default_factory=uuid4)
+
+    def rag_key(self):
+        return self.req

metagpt/exp_pool/scorers/__init__.py

@@ -0,0 +1,6 @@
+"""Scorers init."""
+
+from metagpt.exp_pool.scorers.base import BaseScorer
+from metagpt.exp_pool.scorers.simple import SimpleScorer
+
+__all__ = ["BaseScorer", "SimpleScorer"]

metagpt/exp_pool/scorers/base.py

@@ -0,0 +1,15 @@
+"""Base scorer."""
+
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel, ConfigDict
+
+from metagpt.exp_pool.schema import Score
+
+
+class BaseScorer(BaseModel, ABC):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @abstractmethod
+    async def evaluate(self, req: str, resp: str) -> Score:
+        """Evaluates the quality of a response relative to a given request."""

metagpt/exp_pool/scorers/simple.py

@@ -0,0 +1,65 @@
+"""Simple scorer."""
+
+import json
+
+from pydantic import Field
+
+from metagpt.exp_pool.schema import Score
+from metagpt.exp_pool.scorers.base import BaseScorer
+from metagpt.llm import LLM
+from metagpt.provider.base_llm import BaseLLM
+from metagpt.utils.common import CodeParser
+
+SIMPLE_SCORER_TEMPLATE = """
+Role: You are a highly efficient assistant, tasked with evaluating a response to a given request. The response is generated by a large language model (LLM). 
+
+I will provide you with a request and a corresponding response. Your task is to assess this response and provide a score from a human perspective.
+
+## Context
+### Request
+{req}
+
+### Response
+{resp}
+
+## Format Example
+```json
+{{
+    "val": "the value of the score, int from 1 to 10, higher is better.",
+    "reason": "an explanation supporting the score."
+}}
+```
+
+## Instructions
+- Understand the request and response given by the user.
+- Evaluate the response based on its quality relative to the given request.
+- Provide a score from 1 to 10, where 10 is the best.
+- Provide a reason supporting your score.
+
+## Constraint
+Format: Just print the result in json format like **Format Example**.
+
+## Action
+Follow instructions, generate output and make sure it follows the **Constraint**.
+"""
+
+
+class SimpleScorer(BaseScorer):
+    llm: BaseLLM = Field(default_factory=LLM)
+
+    async def evaluate(self, req: str, resp: str) -> Score:
+        """Evaluates the quality of a response relative to a given request, as scored by an LLM.
+
+        Args:
+            req (str): The request.
+            resp (str): The response.
+
+        Returns:
+            Score: An object containing the score (1-10) and the reasoning.
+        """
+
+        prompt = SIMPLE_SCORER_TEMPLATE.format(req=req, resp=resp)
+        resp = await self.llm.aask(prompt)
+        resp_json = json.loads(CodeParser.parse_code(resp, lang="json"))
+
+        return Score(**resp_json)

metagpt/exp_pool/serializers/__init__.py

@@ -0,0 +1,9 @@
+"""Serializers init."""
+
+from metagpt.exp_pool.serializers.base import BaseSerializer
+from metagpt.exp_pool.serializers.simple import SimpleSerializer
+from metagpt.exp_pool.serializers.action_node import ActionNodeSerializer
+from metagpt.exp_pool.serializers.role_zero import RoleZeroSerializer
+
+
+__all__ = ["BaseSerializer", "SimpleSerializer", "ActionNodeSerializer", "RoleZeroSerializer"]

metagpt/exp_pool/serializers/action_node.py

@@ -0,0 +1,36 @@
+"""ActionNode Serializer."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Type
+
+# Import ActionNode only for type checking to avoid circular imports
+if TYPE_CHECKING:
+    from metagpt.actions.action_node import ActionNode
+
+from metagpt.exp_pool.serializers.simple import SimpleSerializer
+
+
+class ActionNodeSerializer(SimpleSerializer):
+    def serialize_resp(self, resp: ActionNode) -> str:
+        return resp.instruct_content.model_dump_json()
+
+    def deserialize_resp(self, resp: str) -> ActionNode:
+        """Customized deserialization, it will be triggered when a perfect experience is found.
+
+        ActionNode cannot be serialized, it throws an error 'cannot pickle 'SSLContext' object'.
+        """
+
+        class InstructContent:
+            def __init__(self, json_data):
+                self.json_data = json_data
+
+            def model_dump_json(self):
+                return self.json_data
+
+        from metagpt.actions.action_node import ActionNode
+
+        action_node = ActionNode(key="", expected_type=Type[str], instruction="", example="")
+        action_node.instruct_content = InstructContent(resp)
+
+        return action_node

metagpt/exp_pool/serializers/base.py

@@ -0,0 +1,29 @@
+"""Base serializer."""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+
+class BaseSerializer(BaseModel, ABC):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    @abstractmethod
+    def serialize_req(self, **kwargs) -> str:
+        """Serializes the request for storage.
+
+        Do not modify kwargs. If modification is necessary, use copy.deepcopy to create a copy first.
+        Note that copy.deepcopy may raise errors, such as TypeError: cannot pickle '_thread.RLock' object.
+        """
+
+    @abstractmethod
+    def serialize_resp(self, resp: Any) -> str:
+        """Serializes the function's return value for storage.
+
+        Do not modify resp. The rest is the same as `serialize_req`.
+        """
+
+    @abstractmethod
+    def deserialize_resp(self, resp: str) -> Any:
+        """Deserializes the stored response back to the function's return value"""

metagpt/exp_pool/serializers/role_zero.py

@@ -0,0 +1,58 @@
+"""RoleZero Serializer."""
+
+import copy
+import json
+
+from metagpt.exp_pool.serializers.simple import SimpleSerializer
+
+
+class RoleZeroSerializer(SimpleSerializer):
+    def serialize_req(self, **kwargs) -> str:
+        """Serialize the request for database storage, ensuring it is a string.
+
+        Only extracts the necessary content from `req` because `req` may be very lengthy and could cause embedding errors.
+
+        Args:
+            req (list[dict]): The request to be serialized. Example:
+                [
+                    {"role": "user", "content": "..."},
+                    {"role": "assistant", "content": "..."},
+                    {"role": "user", "content": "context"},
+                ]
+
+        Returns:
+            str: The serialized request as a JSON string.
+        """
+        req = kwargs.get("req", [])
+
+        if not req:
+            return ""
+
+        filtered_req = self._filter_req(req)
+
+        if state_data := kwargs.get("state_data"):
+            filtered_req.append({"role": "user", "content": state_data})
+
+        return json.dumps(filtered_req)
+
+    def _filter_req(self, req: list[dict]) -> list[dict]:
+        """Filter the `req` to include only necessary items.
+
+        Args:
+            req (list[dict]): The original request.
+
+        Returns:
+            list[dict]: The filtered request.
+        """
+
+        filtered_req = [copy.deepcopy(item) for item in req if self._is_useful_content(item["content"])]
+
+        return filtered_req
+
+    def _is_useful_content(self, content: str) -> bool:
+        """Currently, only the content of the file is considered, and more judgments can be added later."""
+
+        if "Command Editor.read executed: file_path" in content:
+            return True
+
+        return False

metagpt/exp_pool/serializers/simple.py

@@ -0,0 +1,22 @@
+"""Simple Serializer."""
+
+from typing import Any
+
+from metagpt.exp_pool.serializers.base import BaseSerializer
+
+
+class SimpleSerializer(BaseSerializer):
+    def serialize_req(self, **kwargs) -> str:
+        """Just use `str` to convert the request object into a string."""
+
+        return str(kwargs.get("req", ""))
+
+    def serialize_resp(self, resp: Any) -> str:
+        """Just use `str` to convert the response object into a string."""
+
+        return str(resp)
+
+    def deserialize_resp(self, resp: str) -> Any:
+        """Just return the string response as it is."""
+
+        return resp

metagpt/ext/android_assistant/roles/android_assistant.py

@@ -38,7 +38,6 @@ class AndroidAssistant(Role):
 
     def __init__(self, **data):
         super().__init__(**data)
-
         self._watch([UserRequirement, AndroidActionOutput])
         extra_config = config.extra
         self.task_desc = extra_config.get("task_desc", "Just explore any app in this phone!")

metagpt/ext/cr/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

metagpt/ext/cr/actions/__init__.py

@@ -0,0 +1 @@
+

metagpt/ext/cr/actions/code_review.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :
+
+import json
+import re
+from pathlib import Path
+
+import aiofiles
+from unidiff import PatchSet
+
+from metagpt.actions.action import Action
+from metagpt.ext.cr.utils.cleaner import (
+    add_line_num_on_patch,
+    get_code_block_from_patch,
+    rm_patch_useless_part,
+)
+from metagpt.ext.cr.utils.schema import Point
+from metagpt.logs import logger
+from metagpt.utils.common import parse_json_code_block
+from metagpt.utils.report import EditorReporter
+
+CODE_REVIEW_PROMPT_TEMPLATE = """
+NOTICE
+Let's think and work step by step.
+With the given pull-request(PR) Patch, and referenced Points(Code Standards), you should compare each point with the code one-by-one within 4000 tokens.
+
+The Patch code has added line number at the first character each line for reading, but the review should focus on new added code inside the `Patch` (lines starting with line number and '+').
+Each point is start with a line number and follows with the point description.
+
+## Patch
+```
+{patch}
+```
+
+## Points
+{points}
+
+## Output Format
+```json
+[
+    {{
+        "commented_file": "The file path which you give a comment from the patch",
+        "comment": "The chinese comment of code which do not meet point description and give modify suggestions",
+        "code_start_line": "the code start line number like `10` in the Patch of current comment,",
+        "code_end_line": "the code end line number like `15` in the Patch of current comment",
+        "point_id": "The point id which the `comment` references to"
+    }}
+]
+```
+
+CodeReview guidelines:
+- Generate code `comment` that do not meet the point description.
+- Each `comment` should be restricted inside the `commented_file`.
+- Try to provide diverse and insightful comments across different `commented_file`.
+- Don't suggest to add docstring unless it's necessary indeed.
+- If the same code error occurs multiple times, it cannot be omitted, and all places need to be identified.But Don't duplicate at the same place with the same comment!
+- Every line of code in the patch needs to be carefully checked, and laziness cannot be omitted. It is necessary to find out all the places.
+- The `comment` and `point_id` in the Output must correspond to and belong to the same one `Point`.
+
+Strictly Observe:
+Just print the PR Patch comments in json format like **Output Format**.
+And the output JSON must be able to be parsed by json.loads() without any errors.
+"""
+
+CODE_REVIEW_COMFIRM_SYSTEM_PROMPT = """
+You are a professional engineer with {code_language} stack, and good at code review comment result judgement.Let's think and work step by step.
+"""
+
+CODE_REVIEW_COMFIRM_TEMPLATE = """
+## Code
+```
+{code}
+```
+## Code Review Comments
+{comment}
+
+## Description of Defects
+{desc}
+
+## Reference Example for Judgment
+{example}
+
+## Your Task:
+1. First, check if the code meets the requirements and does not violate any defects. If it meets the requirements and does not violate any defects, print `False` and do not proceed with further judgment.
+2. Based on the `Reference Example for Judgment` provided, determine if the `Code` and `Code Review Comments` match. If they match, print `True`; otherwise, print `False`.
+
+Note: Your output should only be `True` or `False` without any explanations.
+"""
+
+
+class CodeReview(Action):
+    name: str = "CodeReview"
+
+    def format_comments(self, comments: list[dict], points: list[Point], patch: PatchSet):
+        new_comments = []
+        logger.debug(f"original comments: {comments}")
+        for cmt in comments:
+            try:
+                if cmt.get("commented_file").endswith(".py"):
+                    points = [p for p in points if p.language == "Python"]
+                elif cmt.get("commented_file").endswith(".java"):
+                    points = [p for p in points if p.language == "Java"]
+                else:
+                    continue
+                for p in points:
+                    point_id = int(cmt.get("point_id", -1))
+                    if point_id == p.id:
+                        code_start_line = cmt.get("code_start_line")
+                        code_end_line = cmt.get("code_end_line")
+                        code = get_code_block_from_patch(patch, code_start_line, code_end_line)
+
+                        new_comments.append(
+                            {
+                                "commented_file": cmt.get("commented_file"),
+                                "code": code,
+                                "code_start_line": code_start_line,
+                                "code_end_line": code_end_line,
+                                "comment": cmt.get("comment"),
+                                "point_id": p.id,
+                                "point": p.text,
+                                "point_detail": p.detail,
+                            }
+                        )
+                        break
+            except Exception:
+                pass
+
+        logger.debug(f"new_comments: {new_comments}")
+        return new_comments
+
+    async def confirm_comments(self, patch: PatchSet, comments: list[dict], points: list[Point]) -> list[dict]:
+        points_dict = {point.id: point for point in points}
+        new_comments = []
+        for cmt in comments:
+            try:
+                point = points_dict[cmt.get("point_id")]
+
+                code_start_line = cmt.get("code_start_line")
+                code_end_line = cmt.get("code_end_line")
+                # 如果代码位置为空的话，那么就将这条记录丢弃掉
+                if not code_start_line or not code_end_line:
+                    logger.info("False")
+                    continue
+
+                # 代码增加上下文，提升confirm的准确率
+                code = get_code_block_from_patch(
+                    patch, str(max(1, int(code_start_line) - 3)), str(int(code_end_line) + 3)
+                )
+                pattern = r"^[ \t\n\r(){}[\];,]*$"
+                if re.match(pattern, code):
+                    code = get_code_block_from_patch(
+                        patch, str(max(1, int(code_start_line) - 5)), str(int(code_end_line) + 5)
+                    )
+                code_language = "Java"
+                code_file_ext = cmt.get("commented_file", ".java").split(".")[-1]
+                if code_file_ext == ".java":
+                    code_language = "Java"
+                elif code_file_ext == ".py":
+                    code_language = "Python"
+                prompt = CODE_REVIEW_COMFIRM_TEMPLATE.format(
+                    code=code,
+                    comment=cmt.get("comment"),
+                    desc=point.text,
+                    example=point.yes_example + "\n" + point.no_example,
+                )
+                system_prompt = [CODE_REVIEW_COMFIRM_SYSTEM_PROMPT.format(code_language=code_language)]
+                resp = await self.llm.aask(prompt, system_msgs=system_prompt)
+                if "True" in resp or "true" in resp:
+                    new_comments.append(cmt)
+            except Exception:
+                logger.info("False")
+        logger.info(f"original comments num: {len(comments)}, confirmed comments num: {len(new_comments)}")
+        return new_comments
+
+    async def cr_by_points(self, patch: PatchSet, points: list[Point]):
+        comments = []
+        valid_patch_count = 0
+        for patched_file in patch:
+            if not patched_file:
+                continue
+            if patched_file.path.endswith(".py"):
+                points = [p for p in points if p.language == "Python"]
+                valid_patch_count += 1
+            elif patched_file.path.endswith(".java"):
+                points = [p for p in points if p.language == "Java"]
+                valid_patch_count += 1
+            else:
+                continue
+            group_points = [points[i : i + 3] for i in range(0, len(points), 3)]
+            for group_point in group_points:
+                points_str = "id description\n"
+                points_str += "\n".join([f"{p.id} {p.text}" for p in group_point])
+                prompt = CODE_REVIEW_PROMPT_TEMPLATE.format(patch=str(patched_file), points=points_str)
+                resp = await self.llm.aask(prompt)
+                json_str = parse_json_code_block(resp)[0]
+                comments_batch = json.loads(json_str)
+                if comments_batch:
+                    patched_file_path = patched_file.path
+                    for c in comments_batch:
+                        c["commented_file"] = patched_file_path
+                    comments.extend(comments_batch)
+
+        if valid_patch_count == 0:
+            raise ValueError("Only code reviews for Python and Java languages are supported.")
+
+        return comments
+
+    async def run(self, patch: PatchSet, points: list[Point], output_file: str):
+        patch: PatchSet = rm_patch_useless_part(patch)
+        patch: PatchSet = add_line_num_on_patch(patch)
+
+        result = []
+        async with EditorReporter(enable_llm_stream=True) as reporter:
+            log_cr_output_path = Path(output_file).with_suffix(".log")
+            await reporter.async_report(
+                {"src_path": str(log_cr_output_path), "filename": log_cr_output_path.name}, "meta"
+            )
+            comments = await self.cr_by_points(patch=patch, points=points)
+            log_cr_output_path.parent.mkdir(exist_ok=True, parents=True)
+            async with aiofiles.open(log_cr_output_path, "w", encoding="utf-8") as f:
+                await f.write(json.dumps(comments, ensure_ascii=False, indent=2))
+            await reporter.async_report(log_cr_output_path)
+
+        if len(comments) != 0:
+            comments = self.format_comments(comments, points, patch)
+            comments = await self.confirm_comments(patch=patch, comments=comments, points=points)
+            for comment in comments:
+                if comment["code"]:
+                    if not (comment["code"].isspace()):
+                        result.append(comment)
+
+        async with EditorReporter() as reporter:
+            src_path = output_file
+            cr_output_path = Path(output_file)
+            await reporter.async_report(
+                {"type": "CodeReview", "src_path": src_path, "filename": cr_output_path.name}, "meta"
+            )
+            async with aiofiles.open(cr_output_path, "w", encoding="utf-8") as f:
+                await f.write(json.dumps(comments, ensure_ascii=False, indent=2))
+            await reporter.async_report(cr_output_path)
+        return result

metagpt/ext/cr/actions/modify_code.py

@@ -0,0 +1,112 @@
+import datetime
+import itertools
+import re
+from pathlib import Path
+from typing import Optional
+
+from unidiff import PatchSet
+
+from metagpt.actions.action import Action
+from metagpt.ext.cr.utils.cleaner import (
+    add_line_num_on_patch,
+    get_code_block_from_patch,
+    rm_patch_useless_part,
+)
+from metagpt.utils.common import CodeParser
+from metagpt.utils.report import EditorReporter
+
+SYSTEM_MSGS_PROMPT = """
+You're an adaptive software developer who excels at refining code based on user inputs. You're proficient in creating Git patches to represent code modifications.
+"""
+
+MODIFY_CODE_PROMPT = """
+NOTICE
+With the given pull-request(PR) Patch, and referenced Comments(Code Standards), you should modify the code according the Comments.
+
+The Patch code has added line no at the first character each line for reading, but the modification should focus on new added code inside the `Patch` (lines starting with line no and '+').
+
+## Patch
+```
+{patch}
+```
+
+## Comments
+{comments}
+
+## Output Format
+<the standard git patch>
+
+
+Code Modification guidelines:
+- Look at `point_detail`, modify the code by `point_detail`, use `code_start_line` and `code_end_line` to locate the problematic code, fix the problematic code by `point_detail` in Comments.Strictly,must handle the fix plan given by `point_detail` in every comment.
+- Create a patch that satifies the git patch standard and your fixes need to be marked with '+' and '-',but notice:don't change the hunk header!
+- Do not print line no in the new patch code.
+
+Just print the Patch in the format like **Output Format**.
+"""
+
+
+class ModifyCode(Action):
+    name: str = "Modify Code"
+    pr: str
+
+    async def run(self, patch: PatchSet, comments: list[dict], output_dir: Optional[str] = None) -> str:
+        patch: PatchSet = rm_patch_useless_part(patch)
+        patch: PatchSet = add_line_num_on_patch(patch)
+
+        #
+        for comment in comments:
+            code_start_line = comment.get("code_start_line")
+            code_end_line = comment.get("code_end_line")
+            # 如果代码位置为空的话，那么就将这条记录丢弃掉
+            if code_start_line and code_end_line:
+                code = get_code_block_from_patch(
+                    patch, str(max(1, int(code_start_line) - 3)), str(int(code_end_line) + 3)
+                )
+                pattern = r"^[ \t\n\r(){}[\];,]*$"
+                if re.match(pattern, code):
+                    code = get_code_block_from_patch(
+                        patch, str(max(1, int(code_start_line) - 5)), str(int(code_end_line) + 5)
+                    )
+                # 代码增加上下文，提升代码修复的准确率
+                comment["code"] = code
+            # 去掉CR时LLM给的comment的影响，应该使用既定的修复方案
+            comment.pop("comment")
+
+        # 按照 commented_file 字段进行分组
+        comments.sort(key=lambda x: x["commented_file"])
+        grouped_comments = {
+            key: list(group) for key, group in itertools.groupby(comments, key=lambda x: x["commented_file"])
+        }
+        resp = None
+        for patched_file in patch:
+            patch_target_file_name = str(patched_file.path).split("/")[-1]
+            if patched_file.path not in grouped_comments:
+                continue
+            comments_prompt = ""
+            index = 1
+            for grouped_comment in grouped_comments[patched_file.path]:
+                comments_prompt += f"""
+                    <comment{index}>
+                    {grouped_comment}
+                    </comment{index}>\n
+                """
+                index += 1
+            prompt = MODIFY_CODE_PROMPT.format(patch=patched_file, comments=comments_prompt)
+            output_dir = (
+                Path(output_dir)
+                if output_dir
+                else self.config.workspace.path / "modify_code" / str(datetime.date.today()) / self.pr
+            )
+            patch_file = output_dir / f"{patch_target_file_name}.patch"
+            patch_file.parent.mkdir(exist_ok=True, parents=True)
+            async with EditorReporter(enable_llm_stream=True) as reporter:
+                await reporter.async_report(
+                    {"type": "Patch", "src_path": str(patch_file), "filename": patch_file.name}, "meta"
+                )
+                resp = await self.llm.aask(msg=prompt, system_msgs=[SYSTEM_MSGS_PROMPT])
+                resp = CodeParser.parse_code(resp, "diff")
+                with open(patch_file, "w", encoding="utf-8") as file:
+                    file.writelines(resp)
+                await reporter.async_report(patch_file)
+        return resp

metagpt/ext/cr/points.json

@@ -0,0 +1,656 @@
+[
+    {
+        "id": 1,
+        "text": "Avoid unused temporary variables",
+        "language": "Java",
+        "detail": "Defect type: Avoid unused temporary variables; Corresponding Fixer: UnusedLocalVariableFixer; Fix solution: Delete unused temporary variables",
+        "yes_example": "Examples of being judged as 'avoid unused temporary variables'",
+        "no_example": "Examples that cannot be judged as 'avoiding unused temporary variables'\n<Example1>\npublic void setTransientVariablesLocal(Map<String, Object> transientVariables) {\n    throw new UnsupportedOperationException(\"No execution active, no variables can be set\");\n}\nThis code's 'transientVariables' is a function parameter rather than a temporary variable. Although 'transientVariables' is not used or referenced, this cannot be judged as 'avoiding unused temporary variables'\n</Example1>\n\n<Example2>\npublic class TriggerCmd extends NeedsActiveExecutionCmd<Object> {\n  protected Map<String, Object> transientVariables;\n  public TriggerCmd(Map<String, Object> transientVariables) {\n    this.transientVariables = transientVariables;\n  }\n}\nIn the above code, 'transientVariables' is not a temporary variable; it is a class attribute and is used in the constructor, so this cannot be judged as 'avoiding unused temporary variables'\n</Example2>"
+    },
+    {
+        "id": 2,
+        "text": "Do not use System.out.println to print",
+        "language": "Java",
+        "detail": "Defect type: Do not use System.out.println to print; Corresponding Fixer: SystemPrintlnFixer; Fixing solution: Comment out the System.out.println code",
+        "yes_example": "Example of being judged as 'Do not use System.out.println for printing'",
+        "no_example": "Examples that cannot be judged as 'Do not use System.out.println to print'\n<Example1>\nthrow new IllegalStateException(\"There is no authenticated user, we need a user authenticated to find tasks\");\nThe above code is throwing an exception, not using 'System.out.print', so this cannot be judged as 'Do not use System.out.println to print'\n</Example1>"
+    },
+    {
+        "id": 3,
+        "text": "Avoid unused formal parameters in functions",
+        "language": "Java",
+        "detail": "Defect type: Avoid unused formal parameters in functions; Fix solution: Ignore",
+        "yes_example": "Examples of being judged as 'avoiding unused formal parameters' in functions\n\n<Example1>\npublic void setTransientVariablesLocal(Map<String, Object> transientVariables) {\n    throw new UnsupportedOperationException(\"No execution active, no variables can be set\");\n}In this code, the formal parameter \"transientVariables\" does not appear in the function body, so this is judged as 'avoiding unused formal parameters'\n</Example1>\n\n<Example2>\nprotected void modifyFetchPersistencePackageRequest(PersistencePackageRequest ppr, Map<String, String> pathVars) {}\nIn this code, the formal parameters \"ppr\" and \"pathVars\" do not appear in the function body, so this is judged as 'avoiding unused formal parameters'\n</Example2>",
+        "no_example": "Examples that cannot be judged as 'avoiding unused parameters in functions'\n<Example1>\npublic String processFindForm(@RequestParam(value = \"pageNo\", defaultValue = \"1\") int pageNo) {\n\tlastName = owner.getLastName();\n\treturn addPaginationModel(pageNo, paginationModel, lastName, ownersResults);\n}In this code, the parameter 'pageNo' is used within the current function 'processFindForm' in the statement 'return addPaginationModel(pageNo, paginationModel, lastName, ownersResults);', although pageNo is not used for logical calculations, it is used as a parameter in a function call to another function, so this cannot be judged as 'avoiding unused parameters in functions'\n</Example1>\n<Example2>\npublic void formatDate(Date date) {\n\tSimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd\");\n\tSystem.out.println(\"Formatted date: \" + sdf.format(date));\n}In this code, the parameter 'date' is referenced in the statement 'System.out.println(\"Formatted date: \" + sdf.format(date))', so this cannot be judged as 'avoiding unused parameters in functions'\n</Example2>"
+    },
+    {
+        "id": 4,
+        "text": "if statement block cannot be empty",
+        "language": "Java",
+        "detail": "Defect type: if statement block cannot be empty; Corresponding Fixer: EmptyIfStmtFixer; Fixing solution: delete the if statement block or handle the logic appropriately or comment to explain why it is empty",
+        "yes_example": "Examples of being judged as 'if statement block cannot be empty'\n<Example1>\npublic void emptyIfStatement() {\n\tif (getSpecialties().isEmpty()) {\n\t}\n}\nThis code's if statement block is empty, so it is judged as 'if statement block cannot be empty'\n</Example1>\n\n<Example2>\npublic void judgePersion() {\n\tif (persion != null) {\n\t\t// judge persion if not null\n\t}\n}\nAlthough this code's if statement block has content, the '// judge persion if not null' is just a code comment, and there is no actual logic code inside the if statement block, so it is judged as 'if statement block cannot be empty'\n</Example2>",
+        "no_example": "Example that cannot be judged as 'if statement block cannot be empty'"
+    },
+    {
+        "id": 5,
+        "text": "Loop body cannot be empty",
+        "language": "Java",
+        "detail": "Defect type: loop body cannot be empty; Corresponding Fixer: EmptyStatementNotInLoopFixer; Repair solution: delete the corresponding while, for, foreach loop body or add appropriate logical processing or comment explaining why it is empty",
+        "yes_example": "Examples of being judged as 'Loop body cannot be empty'\n<Example1>\npublic void emptyLoopBody() {\n\tfor (Specialty specialty : getSpecialties()) {\n\t}\n}\nThis code's for loop body is empty, so it is judged as 'Loop body cannot be empty'\n</Example1>\n\n<Example2>\npublic void emptyLoopBody() {\n\twhile (True) {\n\t\t// this is a code example\n\t}\n}\nThe while loop body in this code is not empty, but the content is just a code comment with no logical content, so it is judged as 'Loop body cannot be empty'\n</Example2>\n\n<Example3>\npublic void emptyLoopBody() {\n\twhile (True) {\n\t\t\n\t}\n}\nThe while loop body in this code is empty, so it is judged as 'Loop body cannot be empty'\n</Example3>",
+        "no_example": "Example that cannot be judged as 'loop body cannot be empty'\n<Example1>\npublic void emptyLoopBody() {\n\tfor (Specialty specialty : getSpecialties()) {\n\t\ta = 1;\n\t\tif (a == 1) {\n\t\t\tretrun a;\n\t\t}\n\t}\n}\nThe content of the for loop in the above code is not empty, and the content is not entirely code comments, so this cannot be judged as 'loop body cannot be empty'\n</Example1>"
+    },
+    {
+        "id": 6,
+        "text": "Avoid using printStackTrace(), and instead use logging to record.",
+        "language": "Java",
+        "detail": "Defect type: Avoid using printStackTrace(), should use logging to record; Repair solution: Use logging to record",
+        "yes_example": "Example of being judged as 'Avoid using printStackTrace(), should use logging to record'",
+        "no_example": "### Example that cannot be judged as 'avoid using printStackTrace(), should use logging to record'\n<Example1>\npublic void usePrintStackTrace() {\n\ttry {\n\t\tthrow new Exception(\"Fake exception\");\n\t} catch (Exception e) {\n\t\tlogging.info(\"info\");\n\t}\n}\nThis code uses logging in the catch statement, so it cannot be judged as 'avoid using printStackTrace(), should use logging to record'\n</Example1>"
+    },
+    {
+        "id": 7,
+        "text": "The catch block cannot be empty",
+        "language": "Java",
+        "detail": "Defect type: catch block cannot be empty; Corresponding Fixer: EmptyCatchBlockFixer; Fix solution: Add a comment inside the catch block",
+        "yes_example": "Examples of being judged as 'catch block cannot be empty'\n\n<Example1>\n\ntry {\n    int[] array = new int[5];\n    int number = array[10];\n} catch (ArrayIndexOutOfBoundsException e) {\n    \n}\nThis code has an empty catch block, so it is judged as 'catch block cannot be empty'\n</Example1>\n\n<Example2>\n\ntry {\n    String str = null;\n    str.length();\n} catch (NullPointerException e) {\n   \n}\nThis code has an empty catch block, so it is judged as 'catch block cannot be empty'\n</Example2>\n\n<Example3>\npublic class EmptyCatchExample {\n    public static void main(String[] args) {\n        try {\n            // Attempt to divide by zero to trigger an exception\n            int result = 10 / 0;\n        } catch (ArithmeticException e) {\n         \n        }\n    }\n}\nThis code has an empty catch block, so it is judged as 'catch block cannot be empty'\n</Example3>\n\n<Example4>\n\ntry {\n    FileReader file = new FileReader(\"nonexistentfile.txt\");\n} catch (FileNotFoundException e) {\n    \n}\nThis code has an empty catch block, so it is judged as 'catch block cannot be empty'\n</Example4>\n\n<Example5>\n\ntry {\n    Object obj = \"string\";\n    Integer num = (Integer) obj;\n} catch (ClassCastException e) {\n\t\n}\nThis code has an empty catch block, so it is judged as 'catch block cannot be empty'\n</Example5>",
+        "no_example": "Examples that cannot be judged as 'catch block cannot be empty'\n<Example1>\npersionNum = 1\ntry {\n\treturn True;\n} catch (Exception e) {\n\t// If the number of people is 1, return false\n\tif (persionNum == 1){\n\t\treturn False;\n\t}\n}This catch statement is not empty, so it cannot be judged as 'catch block cannot be empty'\n</Example1>\n\n<Example2>\ntry {\n\tthrow new Exception(\"Fake exception\");\n} catch (Exception e) {\n\te.printStackTrace();\n}Although this catch statement only has 'e.printStackTrace();', it is indeed not empty, so it cannot be judged as 'catch block cannot be empty'\n</Example2>"
+    },
+    {
+        "id": 8,
+        "text": "Avoid unnecessary tautologies/contradictions",
+        "language": "Java",
+        "detail": "Defect type: Avoid unnecessary true/false judgments; Corresponding Fixer: UnconditionalIfStatement Fixer; Fixing solution: Delete true/false judgment logic",
+        "yes_example": "Examples of being judged as 'avoiding unnecessary always true/always false judgments'",
+        "no_example": "Examples that cannot be judged as 'avoiding unnecessary always true/always false judgments'"
+    },
+    {
+        "id": 9,
+        "text": "In a switch statement, default must be placed at the end",
+        "language": "Java",
+        "detail": "Defect type: The default in switch must be placed at the end; Corresponding Fixer: DefaultLabelNotLastInSwitchStmtFixer; Fixing solution: Place default at the end in switch",
+        "yes_example": "Example of being judged as 'default in switch must be placed at the end'",
+        "no_example": "Example that cannot be judged as 'the default in switch must be placed at the end'"
+    },
+    {
+        "id": 10,
+        "text": "Comparison of String without using equals() function",
+        "language": "Java",
+        "detail": "Defect type: Not using the equals() function to compare Strings; Corresponding Fixer: UnSynStaticDateFormatter Fixer; Fix solution: Use the equals() function to compare Strings",
+        "yes_example": "Examples of being judged as 'not using the equals() function to compare Strings'\n\n<Example1>\nif (existingPet != null && existingPet.getName() == petName) {\n    result.rejectValue(\"name\", \"duplicate\", \"already exists\");\n}\nIn this code, both existingPet.getName() and petName are strings, but the comparison in the if statement uses == instead of equals() to compare the strings, so this is judged as 'not using the equals() function to compare Strings'.\n</Example1>\n\n<Example2>\nString isOk = \"ok\";\nif (\"ok\" == isOk) {\n    result.rejectValue(\"name\", \"duplicate\", \"already exists\");\n}\nIn this code, isOk is a string, but in the if statement, it is compared with \"ok\" using ==, not using equals() to compare the strings, it should use \"ok\".equals(isOk), so this is judged as 'not using the equals() function to compare Strings'.\n</Example2>\n\n<Example3>\nString str1 = \"Hello\";\nString str2 = \"Hello\";\nif (str1 == str2) {\n    System.out.println(\"str1 and str2 reference the same object\");\n} else {\n    System.out.println(\"str1 and str2 reference different objects\");\n}\nIn this code, if (str1 == str2) uses == to compare str1 and str2, not using equals() to compare the strings, it should use str1.equals(str2), so this is judged as 'not using the equals() function to compare Strings'.\n</Example3>\n\n<Example4>\nString str = \"This is string\";\nif (str == \"This is not str\") {\n    return str;\n}\nIn this code, if (str == \"This is not str\") uses == to compare the strings, not using equals() to compare the strings, it should use \"This is not str\".equals(str), so this is judged as 'not using the equals() function to compare Strings'.\n</Example4>",
+        "no_example": "Examples that cannot be judged as 'not using the equals() function to compare Strings'\n\n<Example1>\nif (PROPERTY_VALUE_YES.equalsIgnoreCase(readWriteReqNode))\n  formProperty.setRequired(true);\nIn this code, both PROPERTY_VALUE_YES and readWriteReqNode are strings. The comparison between PROPERTY_VALUE_YES and readWriteReqNode in the if statement uses equalsIgnoreCase (case-insensitive string comparison), which is also in line with using the equals() function to compare Strings. Therefore, this cannot be judged as 'not using the equals() function to compare Strings'\n</Example1>\n\n<Example2>\nString isOk = \"ok\";\nif (\"ok\".equals(isOk)) {\n\tresult.rejectValue(\"name\", \"duplicate\", \"already exists\");\n}In this code, isOk is a string. In the if statement, the comparison with \"ok\" uses the equals() function to compare Strings, so this cannot be judged as 'not using the equals() function to compare Strings'\n</Example2>"
+    },
+    {
+        "id": 11,
+        "text": "Prohibit the direct use of string output for exceptions in logs, please use placeholders to pass the exception object",
+        "language": "Java",
+        "detail": "Defect type: Do not directly output exceptions as strings in logs, use placeholders to pass the exception object; Corresponding Fixer: ConcatExceptionFixer; Fix solution: Use placeholders to pass the exception object",
+        "yes_example": "Example of being judged as 'Prohibited to directly output exceptions using string in logs, please use placeholders to pass exception objects'\n<Example1>\ntry {\n  listenersNode = objectMapper.readTree(listenersNode.asText());\n} catch (Exception e) {\n  LOGGER.info(\"Listeners node can not be read\", e);\n}In this code, the log output content is directly concatenated using the string \"Listeners node can not be read\". When outputting exceptions in logs, placeholders should be used to output exception information, rather than directly concatenating strings. Therefore, this is judged as 'Prohibited to directly output exceptions using string in logs, please use placeholders to pass exception objects'.\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'Prohibited to directly output exceptions using string in logs, please use placeholders to pass exception objects':\n\n<Example1>\nPerson person = personService.getPerson(1);\nif (person == null) {\n    LOGGER.error(PERSION_NOT_EXIT);\n}\nIn this code, PERSION_NOT_EXIT is a user-defined exception constant representing that the person does not exist, and it does not directly use the string 'person not exit' for concatenation, so this cannot be judged as 'Prohibited to directly output exceptions using string in logs, please use placeholders to pass exception objects'.\n<Example1>\n\n<Example2>\ntry {\n  a = a + 1;\n} catch (Exception e) {\n  Person person = personService.getPerson(1);\n  LOGGER.info(person);\n}\nIn this code, the log output does not directly use string concatenation, but rather uses the Person object for output, so this cannot be judged as 'Prohibited to directly output exceptions using string in logs, please use placeholders to pass exception objects'.\n</Example2>"
+    },
+    {
+        "id": 12,
+        "text": "The finally block cannot be empty",
+        "language": "Java",
+        "detail": "Defect type: finally block cannot be empty; Corresponding Fixer: EmptyFinallyBlockFixer; Fix solution: Delete the empty finally block",
+        "yes_example": "Examples of being judged as 'finally block cannot be empty'\n\n<Example1>\n\ntry {\n    Persion persion = persionService.getPersion(1);\n    return persion;\n} finally {\n    \n}\nThis code has an empty finally block, so it is judged as 'finally block cannot be empty'\n\n</Example1>\n\n<Example2>\n\ntry {\n    System.out.println(\"Inside try block\");\n} finally {\n    // Empty finally block with no statements, this is a defect\n}\nThis code has an empty finally block, so it is judged as 'finally block cannot be empty'\n\n</Example2>\n\n<Example3>\n\ntry {\n    int result = 10 / 0;\n} catch (ArithmeticException e) {\n    e.printStackTrace();\n} finally {\n    \n}\nThis code has an empty finally block, so it is judged as 'finally block cannot be empty'\n\n</Example3>\n\n<Example4>\n\ntry {\n    String str = null;\n    System.out.println(str.length());\n} catch (NullPointerException e) {\n    e.printStackTrace();\n} finally {\n    \n}\nThis code has an empty finally block, so it is judged as 'finally block cannot be empty'\n\n</Example4>\n\n<Example5>\n\ntry {\n    int[] array = new int[5];\n    int number = array[10];\n} catch (ArrayIndexOutOfBoundsException e) {\n    e.printStackTrace();\n} finally {\n    // Finally block with only comments\n    // This is an empty finally block\n}\nThis code has an empty finally block, so it is judged as 'finally block cannot be empty'\n\n</Example5>\n\n<Example6>\n\ntry {\n    FileReader file = new FileReader(\"nonexistentfile.txt\");\n} catch (FileNotFoundException e) {\n    e.printStackTrace();\n} finally {\n    // Finally block with only empty lines\n    \n}\nThis code has an empty finally block, so it is judged as 'finally block cannot be empty'\n\n</Example6>",
+        "no_example": "Example that cannot be judged as 'finally block cannot be empty'\n<Example1>\npublic void getPersion() {\n\ttry {\n\t\tPersion persion = persionService.getPersion(1);\n\t\tif (persion != null){\n\t\t\treturn persion;\n\t\t}\n\t} finally {\n\t\treturn null;\n\t}\n}\nThis code's finally block contains non-comment content 'return null;', so this cannot be judged as 'finally block cannot be empty'\n</Example1>"
+    },
+    {
+        "id": 13,
+        "text": "try block cannot be empty",
+        "language": "Java",
+        "detail": "Defect type: try block cannot be empty; Corresponding Fixer: EmptyTryBlockFixer; Fix solution: Delete the entire try statement",
+        "yes_example": "Examples of being judged as 'try block cannot be empty'\n<Example1>\npublic void getPersion() {\n\ttry {\n\n\t}\n\treturn null;\n}This code's try block is empty, so it is judged as 'try block cannot be empty'\n</Example1>\n\n<Example2>\npublic void demoFinallyBlock() {\n\ttry {\n\n\t} finally {\n\t\treturn null;\n\t}\n}This code's try block is empty, so it is judged as 'try block cannot be empty'\n</Example2>\n\n<Example3>\ntry {\n    \n} catch (Exception e) {\n    e.printStackTrace();\n}This code's try block is empty, so it is judged as 'try block cannot be empty'\n</Example3>\n\n<Example4>\ntry {\n    // try block with only comments\n\t\n} catch (Exception e) {\n    e.printStackTrace();\n}This code's try block contains only comments and blank lines, which can also be considered as having no content in the try block, so it is judged as 'try block cannot be empty'\n</Example4>",
+        "no_example": "### Example that cannot be judged as 'try block cannot be empty'\n<Example1>\ntry {\n\ta = a + 1;\n} catch (Exception e) {\n\te.printStackTrace();\n}\nThis code snippet contains non-comment content 'return null;' in the try block, so it cannot be judged as 'try block cannot be empty'\n</Example1>"
+    },
+    {
+        "id": 14,
+        "text": "Avoid unnecessary NULL or null checks on objects",
+        "language": "Java",
+        "detail": "Defect type: Avoid unnecessary NULL or null checks on objects; Corresponding Fixer: LogicalOpNpeFixer; Fix solution: Remove the logic of unnecessary NULL checks on objects",
+        "yes_example": "Examples of being judged as 'avoiding unnecessary NULL or null checks':",
+        "no_example": "Example that cannot be judged as 'avoiding unnecessary NULL or null checks'\n<Example1>\nCat cat = catService.get(1);\nif (cat != null){\n\tretrun cat;\n}In this code, the object 'cat' is obtained through the service and it is uncertain whether it is null or not, so the condition 'cat != null' in the if statement is necessary, therefore this cannot be judged as 'avoiding unnecessary NULL or null checks'\n</Example1>"
+    },
+    {
+        "id": 15,
+        "text": "Avoid return in finally block",
+        "language": "Java",
+        "detail": "Defect type: Avoid return in finally block; Repair solution: No need for repair",
+        "yes_example": "Example judged as 'avoid return in finally block'",
+        "no_example": "Example that cannot be judged as 'avoiding return in finally block'\n<Example1>\npublic void getPersion() {\n\ttry {\n\t\tPersion persion = persionService.getPersion(1);\n\t\tif (persion != null){ \n\t\t\treturn persion;\n\t\t}\n\t} finally {\n\t\tLOGGER.info(PERSION_NOT_EXIT);\n\t}\n}\nThis code's finally block does not contain 'return', so it cannot be judged as 'avoiding return in finally block'\n</Example1>"
+    },
+    {
+        "id": 16,
+        "text": "Avoid empty static initialization",
+        "language": "Java",
+        "detail": "Defect type: Avoid empty static initialization; Corresponding Fixer: EmptyInitializerFixer; Fix solution: Delete the entire empty initialization block",
+        "yes_example": "Examples of being judged as 'Avoid empty static initialization'",
+        "no_example": "Example that cannot be judged as 'avoiding empty static initialization'\n<Example1>\npublic class Cat {\n\tstatic {\n\t\t// Static initialization block\n\t\tcat = null;\n\t}\n}\nThis code has a static block with content, not empty, and the static initialization block contains non-commented code with actual logic, so this cannot be judged as 'avoiding empty static initialization'\n</Example1>"
+    },
+    {
+        "id": 17,
+        "text": "Avoid risks of improper use of calendar",
+        "language": "Java",
+        "detail": "Defect type: Avoid improper usage risks of calendar classes; Fix solution: Use LocalDate from the java.time package in Java 8 and above",
+        "yes_example": "Examples of being judged as 'avoiding improper use of calendar class risks'\n<Example1>\nprivate static final Calendar calendar = new GregorianCalendar(2020, Calendar.JANUARY, 1);\nThe Calendar and GregorianCalendar in this code are not thread-safe, so this is judged as 'avoiding improper use of calendar class risks'\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'avoiding improper use of calendar class risks'"
+    },
+    {
+        "id": 18,
+        "text": "To convert a collection to an array, you must use the toArray(T[] array) method of the collection, passing in an array of the exact same type, with a size equal to list.size()",
+        "language": "Java",
+        "detail": "Defect type: When converting a collection to an array, you must use the toArray(T[] array) method of the collection, passing an array of the exact same type, with a size equal to list.size(); Corresponding Fixer: ClassCastExpWithToArrayFixer; Repair solution: Use the toArray(T[] array) method of the collection, and pass an array of the exact same type",
+        "yes_example": "Example judged as 'When converting a collection to an array, you must use the collection's toArray(T[] array) method, passing an array of exactly the same type, with the size being list.size()'",
+        "no_example": "Example that cannot be judged as 'using the method of converting a collection to an array, you must use the toArray(T[] array) of the collection, passing in an array of exactly the same type, and the size is list.size()':"
+    },
+    {
+        "id": 19,
+        "text": "Prohibit the use of NULL or null for comparison in equals()",
+        "language": "Java",
+        "detail": "Defect type: Prohibit using NULL or null for comparison in equals(); Corresponding Fixer: EqualsNullFixer; Fixing solution: Use Object's null check function for comparison",
+        "yes_example": "Examples of being judged as 'Prohibited to use NULL or null for comparison in equals()'",
+        "no_example": "Examples that cannot be judged as 'prohibiting the use of NULL or null for comparison in equals()'"
+    },
+    {
+        "id": 20,
+        "text": "switch statement block cannot be empty",
+        "language": "Java",
+        "detail": "Defect type: switch statement block cannot be empty; Corresponding Fixer: EmptySwitchStatementsFix; Fix solution: Delete the entire empty switch statement block",
+        "yes_example": "Examples of being judged as 'switch statement block cannot be empty'\n<Example1>\nswitch (number) {\n    \n}This code is a switch statement block, but it contains no content, so it is judged as 'switch statement block cannot be empty'\n</Example1>\n\n<Example2>\nswitch (number) {\n    // This is a switch statement block\n}This code is a switch statement block, which contains content, but the content is only comments without actual logic, so it is judged as 'switch statement block cannot be empty'\n</Example2>",
+        "no_example": "Example that cannot be judged as 'switch statement block cannot be empty'\n<Example1>\nswitch (number) {\n\tcase 1:\n\t\tSystem.out.println(\"Number one\");\n\t\tbreak;\n\tdefault:\n\t\tSystem.out.println(\"This is the default block, which is incorrectly placed here.\");\n\t\tbreak;\n}\nThis code is a switch statement block that contains content, and the content includes non-commented code with actual logic, so it cannot be judged as 'switch statement block cannot be empty'.\n</Example1>"
+    },
+    {
+        "id": 21,
+        "text": "When performing type coercion, no spaces are needed between the right parenthesis and the coercion value.",
+        "detail": "Defect type: When performing type coercion, no space is required between the right parenthesis and the coercion value; Fix solution: When performing type coercion, no space is required between the right parenthesis and the coercion value.",
+        "language": "Java",
+        "yes_example": "Examples judged as 'When performing type casting, no space is needed between the closing parenthesis and the cast value'",
+        "no_example": "Examples that cannot be judged as 'When performing type coercion, no spaces are required between the right parenthesis and the coercion value'"
+    },
+    {
+        "id": 22,
+        "text": "Method parameters must have a space after the comma when defined and passed",
+        "detail": "Defect type: In the definition and passing of method parameters, a space must be added after the comma for multiple parameters; Repair solution: In the definition and passing of method parameters, a space must be added after the comma for multiple parameters.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'Method parameters must have a space after the comma when defined and passed'",
+        "no_example": "Examples that cannot be judged as 'Method parameters must have a space after the comma both in definition and when passed'"
+    },
+    {
+        "id": 23,
+        "text": "Prohibit the use of the BigDecimal(double) constructor to convert a double value to a BigDecimal object",
+        "detail": "Defect type: Do not use the constructor BigDecimal(double) to convert a double value to a BigDecimal object; Repair solution: It is recommended to use the valueOf method of BigDecimal.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'Prohibited to use the constructor BigDecimal(double) to convert a double value to a BigDecimal object'",
+        "no_example": "Examples that cannot be considered as 'prohibiting the use of the BigDecimal(double) constructor to convert a double value to a BigDecimal object'"
+    },
+    {
+        "id": 24,
+        "text": "No extra semicolons allowed",
+        "detail": "Defect type: extra semicolon; Fix solution: remove extra semicolon",
+        "yes_example": "Example of being judged as 'cannot have extra semicolons'",
+        "no_example": "Examples that cannot be judged as 'cannot have extra semicolons'\n<Example1>\nwhile (True) {\n\ta = a + 1;\n\tbreak;\n}This code requires every semicolon, so it can be judged as 'cannot have extra semicolons'\n</Example1>"
+    },
+    {
+        "id": 25,
+        "text": "Non-thread-safe SimpleDateFormat usage must be synchronized at the function or code block level",
+        "detail": "Defect type: Non-thread-safe SimpleDateFormat usage; Fix solution: Add synchronized modifier at the function or code block level or use other thread-safe methods",
+        "yes_example": "Example of 'Non-thread-safe SimpleDateFormat usage, must be used with synchronized at the function or block level'",
+        "no_example": "Example that cannot be judged as 'Unsafe use of SimpleDateFormat, which must be used at the function or code block level with synchronized':\n<Example1>\npublic synchronized void formatDate(Date date) {\n\tSimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd\");\n\tSystem.out.println(\"Formatted date: \" + sdf.format(date));\n}\nThis code is protected by a synchronized block on the function 'formatDate', ensuring thread safety, so it cannot be judged as 'Unsafe use of SimpleDateFormat, which must be used at the function or code block level with synchronized'.\n</Example1>"
+    },
+    {
+        "id": 26,
+        "text": "Naming does not follow the camel case specification. Class names should use UpperCamelCase style, while method names, parameter names, member variables, and local variables should all use lowerCamelCase style.",
+        "detail": "Defect type: Not following camel case naming convention; Fix solution: Class names should use UpperCamelCase style, method names, parameter names, member variables, and local variables should use lowerCamelCase style.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as 'not following the camel case naming convention'\n<Example1>\npublic class myClass {\n    private int MyVariable;\n    public void MyMethod() {}\n}\nThis code does not follow the camel case naming convention for class names, member variables, and method names, so it is judged as a naming convention issue.\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'not following the camel case naming convention'\n<Example1>\npublic class MyClass {\n    private int myVariable;\n    public void myMethod() {}\n}\nThe class name, member variable, and method name in this code all follow the camel case naming convention, so it cannot be judged as a naming convention issue.\n</Example1>"
+    },
+    {
+        "id": 27,
+        "text": "Abstract class names start with Abstract or Base; exception class names end with Exception; test class names begin with the name of the class they are testing and end with Test",
+        "detail": "Defect type: Naming convention; Solution: Abstract class names should start with Abstract or Base, exception class names should end with Exception, and test class names should start with the name of the class they are testing and end with Test.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as 'naming conventions'\n<Example1>\npublic class MyAbstractClass {}\npublic class MyExceptionClass {}\npublic class TestMyClass {}\nThe naming of the abstract class, exception class, and test class in this code does not conform to the conventions, so it is judged as a naming convention issue.\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'naming conventions'"
+    },
+    {
+        "id": 28,
+        "text": "Avoid adding the 'is' prefix to any boolean type variables in POJO classes",
+        "detail": "Defect type: Naming convention; Fix solution: Do not prefix boolean variables in POJO classes with 'is'.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as 'naming convention' issues\n<Example1>\npublic class User {\n    private boolean isActive;\n}\nIn this code, the boolean type variable has the 'is' prefix, so it is judged as a naming convention issue.\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'naming conventions'"
+    },
+    {
+        "id": 29,
+        "text": "Eliminate completely non-standard English abbreviations to avoid confusion when interpreting them.",
+        "detail": "Defect type: Naming conventions; Solution: Avoid using non-standard English abbreviations to ensure code readability.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as 'naming conventions'\n<Example1>\npublic class CfgMgr {\n    private int cnt;\n}\nIn this code, the class name and variable name use non-standard English abbreviations, so they are judged as naming convention issues.\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'naming conventions'"
+    },
+    {
+        "id": 30,
+        "text": "Avoid using magic characters and numbers, they should be declared as constants",
+        "detail": "Defect type: Avoid using magic characters and numbers, they should be declared as constants; Fix solution: Define magic values as constants.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as 'avoiding magic characters and numbers, should be declared as constants'",
+        "no_example": "Examples that cannot be judged as 'avoiding magic characters and numbers, should be declared as constants'"
+    },
+    {
+        "id": 31,
+        "text": "When assigning values to long or Long, use uppercase L after the number, not lowercase l. The suffix for floating-point numbers should be uppercase D or F.",
+        "detail": "Defect type: Code specification; Repair solution: Use uppercase L when assigning values to long or Long, and use uppercase D or F as suffixes for floating-point type values.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as 'code specification'",
+        "no_example": "Examples that cannot be judged as 'code specification'"
+    },
+    {
+        "id": 32,
+        "text": "If the curly braces are empty, simply write {} without line breaks or spaces inside the braces; if it is a non-empty code block, then: 1) Do not line break before the left curly brace. 2) Line break after the left curly brace. 3) Line break before the right curly brace. 4) Do not line break after the right curly brace if there is code like 'else' following it; the right curly brace indicating termination must be followed by a line break.",
+        "detail": "Defect type: code formatting; Fix solution: follow the curly brace usage standard.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'code format'",
+        "no_example": "Examples that cannot be judged as 'code format' issues\n<Example 1>\npublic class BracketExample {\n    public void method() {\n        if (true) {\n            // do something\n        }\n    }\n}\nThe use of curly braces in this code is in accordance with the standards, so it cannot be judged as a code format issue.\n</Example 1>"
+    },
+    {
+        "id": 33,
+        "text": "No space is needed between the left parenthesis and the adjacent character; no space is needed between the right parenthesis and the adjacent character; and a space is required before the left brace.",
+        "detail": "Defect type: code formatting; Fix solution: follow the usage rules for brackets and spaces.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'code format'\n<Example1>\npublic class SpaceExample {\n    public void method (){\n    }\n}\nThe use of brackets and spaces in this code does not conform to the standard, so it is judged as a code format issue.\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'code specification'\n<Example1>\npublic class SpaceExample {\n    public void method() {}\n}\nThis code uses brackets and spaces in accordance with the specification, so it cannot be judged as a code format issue.\n</Example1>"
+    },
+    {
+        "id": 34,
+        "text": "Reserved words such as if / for / while / switch / do must be separated from the parentheses on both sides by spaces.",
+        "detail": "Defect type: code format; Fix solution: add spaces between reserved words and parentheses.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'code specification'\n<Example1>\npublic class KeywordExample {\n    public void method() {\n        if(true) {\n        }\n    }\n}\nIn this code, there is no space between the if keyword and the parentheses, so it is judged as a code formatting issue.\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'code specification'"
+    },
+    {
+        "id": 35,
+        "text": "All value comparisons between integer wrapper class objects should be done using the equals method",
+        "detail": "Defect type: Code specification; Repair solution: Use the equals method for value comparison between integer wrapper class objects.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as 'code specification'",
+        "no_example": "### Example that cannot be judged as 'code specification'\n<Example1>\npublic class IntegerComparison {\n    public void compare() {\n        Integer a = 100;\n        Integer b = 100;\n        if (a.equals(b)) {\n        }\n    }\n}\nIn this code, the equals method is used to compare integer wrapper class objects, so it cannot be judged as a code specification issue.\n</Example1>"
+    },
+    {
+        "id": 36,
+        "text": "For comparing BigDecimal values, the compareTo() method should be used instead of the equals() method.",
+        "detail": "Defect type: The equality comparison of BigDecimal should use the compareTo() method instead of the equals() method; Fix solution: Use the compareTo() method for comparison.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'For BigDecimal equality comparison, the compareTo() method should be used instead of the equals() method'\n<Example1>\nBigDecimal a = new BigDecimal(\"1.0\");\nBigDecimal b = new BigDecimal(\"1.00\");\nif (a.equals(b)) {\n    // This code will return false because the equals() method compares precision\n}\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'For BigDecimal equality comparison, the compareTo() method should be used instead of the equals() method'"
+    },
+    {
+        "id": 37,
+        "text": "Prohibit having both isXxx() and getXxx() methods for the same attribute xxx in a POJO class.",
+        "detail": "Defect type: Duplicate getter methods in POJO class; Fix solution: Ensure only one getter method exists.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'Prohibited to have both isXxx() and getXxx() methods for the corresponding attribute xxx in a POJO class'",
+        "no_example": "Examples that cannot be judged as 'Prohibiting the existence of both isXxx() and getXxx() methods for the corresponding attribute xxx in a POJO class'"
+    },
+    {
+        "id": 38,
+        "text": "When formatting dates, use the lowercase 'y' uniformly to represent the year in the pattern.",
+        "detail": "Defect type: date formatting error; Fix solution: use lowercase y to represent the year.",
+        "language": "Java",
+        "yes_example": "Example judged as 'When formatting dates, use lowercase y for the year in the pattern'",
+        "no_example": "Examples that cannot be judged as 'When formatting dates, use lowercase y for the year in the pattern'"
+    },
+    {
+        "id": 39,
+        "text": "Prohibited from using in any part of the program: 1) java.sql.Date 2) java.sql.Time 3) java.sql.Timestamp.",
+        "detail": "Defect type: used date classes from the java.sql package; Fix solution: use date classes from the java.time package.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as \"Prohibited from using in any part of the program: 1) java.sql.Date 2) java.sql.Time 3) java.sql.Timestamp\"",
+        "no_example": "Examples that cannot be judged as 'Prohibited to use in any part of the program: 1) java.sql.Date 2) java.sql.Time 3) java.sql.Timestamp'"
+    },
+    {
+        "id": 40,
+        "text": "Determine if all elements within a collection are empty using the isEmpty() method, rather than using the size() == 0 approach.",
+        "detail": "Defect type: Incorrect method for checking empty collection; Fix solution: Use isEmpty() method.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'To determine if all elements within a collection are empty, use the isEmpty() method instead of the size() == 0 approach'\n<Example1>\nList<String> list = new ArrayList<>();\nif (list.size() == 0) {\n    // Empty logic\n}\n</Example1>",
+        "no_example": "Examples that cannot be considered as 'judging whether all elements within a set are empty using the isEmpty() method instead of the size() == 0 approach'"
+    },
+    {
+        "id": 41,
+        "text": "Whenever you override equals, you must also override hashCode.",
+        "detail": "Defect type: hashCode method not overridden; Fix solution: Override both equals and hashCode methods.",
+        "language": "Java",
+        "yes_example": "An example where it is judged that 'if you override equals, you must also override hashCode'",
+        "no_example": "An example where it cannot be judged as 'Whenever you override equals, you must also override hashCode'"
+    },
+    {
+        "id": 42,
+        "text": "When using the Map methods keySet() / values() / entrySet() to return a collection object, you cannot perform element addition operations on it, otherwise a UnsupportedOperationException will be thrown.",
+        "detail": "Defect type: Adding operations to the collections returned by keySet() / values() / entrySet() of a Map; Repair solution: Avoid adding operations to these collections.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'When using the Map methods keySet() / values() / entrySet() to return a collection object, you cannot perform element addition operations on it, otherwise a UnsupportedOperationException exception will be thrown'",
+        "no_example": "Example that cannot be judged as 'When using the methods keySet() / values() / entrySet() of Map to return a collection object, it is not allowed to perform element addition operations on it, otherwise a UnsupportedOperationException will be thrown'"
+    },
+    {
+        "id": 43,
+        "text": "Do not perform element removal / addition operations within a foreach loop. Use the iterator method for removing elements. If concurrent operations are required, the iterator must be synchronized.",
+        "detail": "Defect type: performing remove / add operations on elements within a foreach loop; Repair solution: use iterator to perform remove operations on elements.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'Do not perform element remove / add operations within a foreach loop. Use the iterator method for removing elements; if concurrent operations are required, the iterator must be synchronized.'",
+        "no_example": "Example that cannot be judged as 'Do not perform element remove / add operations inside a foreach loop. Use the iterator method for removing elements. If concurrent operations are required, the iterator should be synchronized.'\n<Example1>\nList<String> list = new ArrayList<>(Arrays.asList(\"a\", \"b\", \"c\"));\nIterator<String> iterator = list.iterator();\nwhile (iterator.hasNext()) {\n    String s = iterator.next();\n    if (s.equals(\"a\")) {\n        iterator.remove();\n    }\n}\n</Example1>"
+    },
+    {
+        "id": 44,
+        "text": "Class, class attributes, and class methods must use Javadoc specifications for comments, using the format /** content */, and must not use the // xxx format.",
+        "detail": "Defect type: Comments do not conform to Javadoc standards; Solution: Use Javadoc-compliant comment format.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as 'class, class attribute, class method annotations must use Javadoc specification, using the format /** content */, not using the // xxx method'",
+        "no_example": "Examples that cannot be judged as 'Class, class attribute, and class method comments must follow the Javadoc specification, using the /** content */ format, not the // xxx format'"
+    },
+    {
+        "id": 45,
+        "text": "All abstract methods (including methods in interfaces) must be annotated with Javadoc comments",
+        "detail": "Defect type: All abstract methods (including methods in interfaces) must be annotated with Javadoc; Repair solution: Add Javadoc comments to all abstract methods (including methods in interfaces), in addition to the return value, parameter exception description, it must also indicate what the method does and what function it implements.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'All abstract methods (including methods in interfaces) must be annotated with Javadoc'",
+        "no_example": "Example that cannot be judged as 'all abstract methods (including methods in interfaces) must be annotated with Javadoc comments'"
+    },
+    {
+        "id": 46,
+        "text": "Usage guidelines for single-line and multi-line comments within methods",
+        "detail": "Defect type: Improper use of comments; Repair solution: Single-line comments inside the method, start a new line above the commented statement, use // for comments. Multi-line comments inside the method use /* */ comments, and pay attention to aligning with the code.",
+        "language": "Java",
+        "yes_example": "### Examples of being judged as 'Improper Use of Comments'\n<Example1>\npublic void exampleMethod() {\n    int a = 1; // Initialize variable a\n    int b = 2; /* Initialize variable b */\n}\nThe single-line and multi-line comments in this code are not used according to the standard, so they are judged as improper use of comments.\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'improper use of comments'\n<Example1>\npublic void exampleMethod() {\n    // Initialize variable a\n    int a = 1;\n    /*\n     * Initialize variable b\n     */\n    int b = 2;\n}\nThis code uses single-line and multi-line comments according to the standard, so it cannot be judged as improper use of comments.\n</Example1>"
+    },
+    {
+        "id": 47,
+        "text": "All enumeration type fields must have comments",
+        "detail": "Defect type: Enumeration type field lacks comments; Fix plan: Add comments to all enumeration type fields to explain the purpose of each data item.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'Enumeration type field lacks comments'\n<Example1>\npublic enum Status {\n    ACTIVE,\n    INACTIVE\n}\nThe enumeration type fields in this code are not commented, so they are judged as lacking comments for enumeration type fields.\n</Example1>",
+        "no_example": "Examples that cannot be judged as 'missing comments for enum fields'\n<Example1>\npublic enum Status {\n    /**\n     * Active status\n     */\n    ACTIVE,\n    /**\n     * Inactive status\n     */\n    INACTIVE\n}\nThis code has comments for the enum fields, so it cannot be judged as missing comments for enum fields.\n</Example1>"
+    },
+    {
+        "id": 48,
+        "text": "The finally block must close resource objects and stream objects.",
+        "detail": "Defect type: resource objects and stream objects are not closed in the finally block; Fix solution: Close resource objects and stream objects in the finally block, and use try-catch for exceptions.",
+        "language": "Java",
+        "yes_example": "Example of being judged as 'resource object, stream object not closed in finally block'",
+        "no_example": "Examples that cannot be judged as 'resource objects, stream objects not closed in the finally block'"
+    },
+    {
+        "id": 49,
+        "text": "Constant names should be in all uppercase, with words separated by underscores.",
+        "detail": "Defect type: Constant naming is not standardized; Fix solution: Constant names should be all uppercase, words separated by underscores, and strive for complete and clear semantic expression, do not be afraid of long names.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as 'Constant names should be in all uppercase, with words separated by underscores'",
+        "no_example": "Examples that cannot be judged as 'constant names should be all uppercase, with words separated by underscores'"
+    },
+    {
+        "id": 50,
+        "text": "Spaces are required on both sides of any binary or ternary operator.",
+        "detail": "Defect type: Lack of space around operators; Fix solution: Any binary or ternary operator should have a space on both sides.",
+        "language": "Java",
+        "yes_example": "Examples of being judged as 'Any binary or ternary operator must have spaces on both sides'",
+        "no_example": "Examples that cannot be judged as 'any binary, ternary operator needs a space on both sides'"
+    },
+    {
+        "id": 51,
+        "text": "Avoid using from <module> import *",
+        "detail": "Defect type: Avoid using 'from <module> import *', importing everything can cause naming conflicts; Solution: Each sub-dependency used should be imported separately.",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'avoid using from <module> import *'",
+        "no_example": "Examples that cannot be judged as 'avoid using from <module> import *'"
+    },
+    {
+        "id": 52,
+        "text": "Avoid using the __import__() function to dynamically import modules",
+        "detail": "Defect type: Avoid using __import__() function to dynamically import modules; Repair solution: Use standard import statements.",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'dynamically importing modules using the __import__() function'",
+        "no_example": "Examples that cannot be judged as 'dynamically importing modules using the __import__() function'"
+    },
+    {
+        "id": 53,
+        "text": "Import statements are not grouped in the order of standard library imports, related third-party imports, and local application/library specific imports.",
+        "detail": "Defect type: Import statements are not grouped in the order of standard library imports, related third-party imports, and local application/library specific imports; Solution: Group import statements in order.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'import statements not grouped in the order of standard library imports, related third-party imports, and local application/library specific imports'",
+        "no_example": "Example that cannot be judged as 'import statements not grouped in the order of standard library imports, related third-party imports, local application/library specific imports'"
+    },
+    {
+        "id": 54,
+        "text": "Avoid unused function parameters",
+        "detail": "Defect type: Avoid unused function parameters; Fix solution: Remove unused function parameters.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'avoid unused function parameters'",
+        "no_example": "Examples that cannot be judged as 'avoiding unused function parameters'"
+    },
+    {
+        "id": 55,
+        "text": "Use is not None to check if a variable is not None",
+        "detail": "Defect type: Not using 'is not None' to check if a variable is not None; Fix solution: Use 'is not None' to check.",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'not using is not None to check if a variable is not None'",
+        "no_example": "Examples that cannot be judged as 'not using is not None to check if a variable is not None'"
+    },
+    {
+        "id": 56,
+        "text": "Avoid using == or != to compare the equivalence of object instances",
+        "detail": "Defect type: Using == or != to compare object instances for equivalence; Fix solution: Should use equals for comparison.",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'using == or != to compare the equivalence of object instances'",
+        "no_example": "Examples that cannot be judged as 'using == or != to compare the equivalence of object instances'"
+    },
+    {
+        "id": 57,
+        "text": "Avoid using single-letter variable names, use descriptive variable names",
+        "detail": "Defect type: Avoid using single-letter variable names, use descriptive variable names; Fix solution: Use descriptive variable names.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'avoid using single-letter variable names, use descriptive variable names'",
+        "no_example": "Examples that cannot be judged as 'avoid using single-letter variable names, use descriptive variable names'"
+    },
+    {
+        "id": 58,
+        "text": "Constant names use all uppercase letters and separate words with underscores",
+        "detail": "Defect type: Constant naming does not use all uppercase letters or does not use underscores to separate; Repair solution: Use all uppercase letters for constant naming and separate with underscores.",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'Constant naming not using all uppercase letters and separated by underscores'",
+        "no_example": "Examples that cannot be judged as 'constant naming not using all uppercase letters and separated by underscores'"
+    },
+    {
+        "id": 59,
+        "text": "Class names should use camel case (CamelCase)",
+        "detail": "Defect type: Class name not using camel case; Repair solution: Use camel case for class names.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'class name not using CamelCase'",
+        "no_example": "Examples that cannot be judged as 'class name not using CamelCase'"
+    },
+    {
+        "id": 60,
+        "text": "Try to use the with statement to manage resources as much as possible",
+        "detail": "Defect type: Not using the with statement to manage resources; Fix solution: Use the with statement to manage resources.",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'not using the with statement to manage resources'",
+        "no_example": "Examples that cannot be judged as 'not using the with statement to manage resources'"
+    },
+    {
+        "id": 61,
+        "text": "Avoid using except or generic Exception to catch all exceptions, specify the exception type instead.",
+        "detail": "Defect type: catch all exceptions; Fix solution: specify specific exception types.",
+        "language": "Python",
+        "yes_example": "Examples judged as 'catching all exceptions using except:' and 'throwing a generic Exception exception'",
+        "no_example": "Example that cannot be judged as 'using except: to catch all exceptions'"
+    },
+    {
+        "id": 62,
+        "text": "Avoid manual string concatenation whenever possible",
+        "detail": "Defect type: manual string concatenation; Fix solution: use formatted strings or join method.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'manual string concatenation'",
+        "no_example": "Examples that cannot be judged as 'manual string concatenation'"
+    },
+    {
+        "id": 63,
+        "text": "Avoid using magic characters and numbers, should be declared as constants",
+        "detail": "Defect type: Using magic characters and numbers; Fix solution: Declare them as constants.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'having magic characters and numbers'",
+        "no_example": "Examples that cannot be judged as 'containing magic characters and numbers'"
+    },
+    {
+        "id": 64,
+        "text": "Boolean variable judgment does not require explicit comparison",
+        "detail": "Defect type: explicit comparison of boolean variables; fix solution: directly use boolean variables for judgment.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'explicit comparison of boolean variables'",
+        "no_example": "Examples that cannot be judged as 'explicit comparison of boolean variables'"
+    },
+    {
+        "id": 65,
+        "text": "Avoid using type() to check object types",
+        "detail": "Defect type: Avoid using type() to check object type; Fix solution: Use isinstance() function.",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'avoid using type() to check object type'",
+        "no_example": "Examples that cannot be judged as 'avoid using type() to check object type'"
+    },
+    {
+        "id": 66,
+        "text": "Avoid using os.system() to call external commands",
+        "detail": "Defect type: Using os.system() to call external commands; Fix solution: Use the subprocess module.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'using os.system() to call external commands'\n<Example1>os.system('ls -l')</Example1>\n<Example2>os.system('ls -l')</Example2>",
+        "no_example": "Examples that cannot be judged as 'using os.system() to call external commands'"
+    },
+    {
+        "id": 67,
+        "text": "Create read-only properties using the @property decorator instead of modifying properties",
+        "detail": "Defect type: Creating modifiable properties using the @property decorator; Fix solution: Only use the @property decorator to create read-only properties.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'using the @property decorator to create modifiable attributes'",
+        "no_example": "Examples that cannot be judged as 'using the @property decorator to create a modifiable attribute'"
+    },
+    {
+        "id": 68,
+        "text": "When using indexing or slicing, do not add spaces inside the brackets or colons.",
+        "detail": "Defect type: adding spaces inside brackets or colons for indexing or slicing; Repair solution: remove spaces inside brackets or colons.",
+        "language": "Python",
+        "yes_example": "Examples judged as 'using spaces inside brackets or colons when using indexing or slicing'",
+        "no_example": "Examples that cannot be judged as 'adding spaces inside brackets or colons when using indexes or slices'"
+    },
+    {
+        "id": 69,
+        "text": "Do not add a space before a comma, semicolon, or colon, but add a space after them",
+        "detail": "Defect type: adding a space before a comma, semicolon, or colon, or not adding a space after them; Fix solution: do not add a space before a comma, semicolon, or colon, but add a space after them.",
+        "language": "Python",
+        "yes_example": "Examples judged as 'adding a space before a comma, semicolon, or colon, or not adding a space after them'",
+        "no_example": "Examples that cannot be judged as 'adding a space before a comma, semicolon, or colon, or not adding a space after them'"
+    },
+    {
+        "id": 70,
+        "text": "For binary operators, there should be spaces on both sides",
+        "detail": "Defect type: no spaces around binary operators; Fix solution: add spaces around binary operators",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'no space around binary operator'",
+        "no_example": "Examples that cannot be judged as 'no space on both sides of the binary operator'"
+    },
+    {
+        "id": 71,
+        "text": "Avoid using Python keywords as variable or function names",
+        "detail": "Defect type: Using Python keywords as variable names or function names; Repair solution: Use non-keyword names.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'using Python keywords as variable names or function names'",
+        "no_example": "Examples that cannot be judged as 'using Python keywords as variable names or function names'\n<Example1>def my_function():\n    pass</Example1>\n<Example2>number = 5</Example2>"
+    },
+    {
+        "id": 72,
+        "text": "Avoid using special characters as variable names/method names/class names, such as $ or @",
+        "detail": "Defect type: Using special characters as variable names/method names/class names; Repair solution: Use legal variable names.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'using special characters as variable names/method names/class names, such as $ or @'",
+        "no_example": "Examples that cannot be judged as 'using special characters as variable names/method names/class names, such as $ or @'"
+    },
+    {
+        "id": 73,
+        "text": "Avoid using raise to rethrow the current exception, as it will lose the original stack trace.",
+        "detail": "Defect type: Re-raise the current exception using raise; Fix solution: Use the raise ... from ... syntax.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'avoid using raise to rethrow the current exception, as it will lose the original stack trace'",
+        "no_example": "Examples that cannot be judged as 'avoid using raise to rethrow the current exception, as it will lose the original stack trace'"
+    },
+    {
+        "id": 74,
+        "text": "Avoid using pass in except block, as it will catch and ignore the exception",
+        "detail": "Defect type: using pass in except block; Fix solution: handle the exception or log the error.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'using pass in except block'",
+        "no_example": "Examples that cannot be judged as 'using pass in an except block'"
+    },
+    {
+        "id": 75,
+        "text": "Avoid using assert statements to perform important runtime checks",
+        "detail": "Defect type: Using assert statements for important runtime checks; Fix solution: Use explicit condition checks and exception handling.",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'using assert statements to perform important runtime checks'",
+        "no_example": "Examples that cannot be judged as 'using assert statements to perform important runtime checks'"
+    },
+    {
+        "id": 76,
+        "text": "Avoid using eval() and exec(), these functions may bring security risks",
+        "detail": "Defect type: Use of eval() and exec() functions; Repair solution: Use secure alternatives.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'using eval() and exec()'\n<Example1>\n eval('print(1)') \n</Example1>\n<Example2> \n exec('a = 1') \n</Example2>",
+        "no_example": "Examples that cannot be judged as 'using eval() and exec()'\n<Example1>\ncompiled_code = compile('print(1)', '<string>', 'exec')\nexec(compiled_code)\n</Example1>"
+    },
+    {
+        "id": 77,
+        "text": "Avoid using sys.exit(), use exceptions to control program exit instead.",
+        "detail": "Defect type: Avoid using sys.exit(), should use exceptions to control program exit; Repair solution: Use exceptions to control program exit.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'avoid using sys.exit(), should use exceptions to control program exit'",
+        "no_example": "Examples that cannot be judged as 'avoid using sys.exit(), should use exceptions to control program exit'"
+    },
+    {
+        "id": 78,
+        "text": "Avoid using time.sleep() for thread synchronization, and instead use synchronization primitives such as locks or events.",
+        "detail": "Defect type: Using time.sleep() for thread synchronization; Fix solution: Use synchronization primitives.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'using time.sleep() for thread synchronization'",
+        "no_example": "Examples that cannot be judged as 'using time.sleep() for thread synchronization'"
+    },
+    {
+        "id": 79,
+        "text": "Avoid exceeding 79 characters per line of code",
+        "detail": "Defect type: Avoid exceeding 79 characters per line of code; Fix solution: Format long lines of code into multiple lines.",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'avoiding more than 79 characters per line of code'",
+        "no_example": "Examples that cannot be judged as 'each line of code should not exceed 79 characters'"
+    },
+    {
+        "id": 80,
+        "text": "Functions and class definitions at the module level are separated by two blank lines, and method definitions within a class are separated by one blank line",
+        "detail": "Defect type: There is no separation of two blank lines between function and class definitions at the module level, and no separation of one blank line between method definitions within the class; Solution: Add blank lines according to the specification.",
+        "language": "Python",
+        "yes_example": "Example of being judged as 'Functions at the module level are not separated by two blank lines, and method definitions within a class are not separated by one blank line'",
+        "no_example": "Examples that cannot be judged as 'There is no two blank lines between module-level function and class definitions, and no one blank line between method definitions inside a class'"
+    },
+    {
+        "id": 81,
+        "text": "Use lowercase letters and underscores to separate variable and function names",
+        "detail": "Defect type: Variable and function naming do not conform to the lowercase letters and underscore separation method; Repair solution: Use lowercase letters and underscore separation method for naming.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'not using lowercase letters and underscores to separate variable and function names'",
+        "no_example": "Examples that cannot be judged as 'naming variables and functions without using lowercase letters and underscores to separate them'"
+    },
+    {
+        "id": 82,
+        "text": "It is not allowed to use the print() function to record logs, use the logging module, etc. to record logs",
+        "detail": "Defect type: Using the print() function to log; Fix solution: Use the logging module to log.",
+        "language": "Python",
+        "yes_example": "Examples of being judged as 'using the print() function to log'",
+        "no_example": "Examples that cannot be considered as 'using the print() function to log'"
+    }
+]

metagpt/ext/cr/points_cn.json

@@ -0,0 +1,656 @@
+[
+  {
+    "id": 1,
+    "text": "避免未使用的临时变量",
+    "language": "Java",
+    "detail": "缺陷类型：避免未使用的临时变量；对应Fixer：UnusedLocalVariableFixer；修复方案：删除未使用的临时变量",
+    "yes_example": "### 被判定为\"避免未使用的临时变量\"的例子\n<例子1>\npublic String initCreationForm(Map<String, Object> model) {\n\t\tOwner owner = new Owner();\n\t\tmodel.put(\"owner\", owner);\n\t\tint unusedVar = 10;\n\t\treturn VIEWS_OWNER_CREATE_OR_UPDATE_FORM;\n\t}\n上述代码中unusedVar变量未被使用，所以这个被判定为\"避免未使用的临时变量\"\n</例子1>\n<例子2>\nint unusedVariable = 10;\nSystem.out.println(\"Hello, World!\");\n这段代码的变量\"unusedVariable\"未被使用或者引用，所以这个不能判定为\"避免未使用的临时变量\"\n</例子2>",
+    "no_example": "### 不能被判定为\"避免未使用的临时变量\"的例子\n<例子1>\npublic void setTransientVariablesLocal(Map<String, Object> transientVariables) {\nthrow new UnsupportedOperationException(\"No execution active, no variables can be set\");\n}\n这段代码的\"transientVariables\"是函数参数而不是临时变量，虽然transientVariables没有被使用或者引用，但是这个也不能判定为\"避免未使用的临时变量\"\n</例子1>\n\n<例子2>\npublic class TriggerCmd extends NeedsActiveExecutionCmd<Object> {\n  protected Map<String, Object> transientVariables;\n  public TriggerCmd(Map<String, Object> transientVariables) {\n    this.transientVariables = transientVariables;\n  }\n}\n上述代码中transientVariables不属于临时变量，它是类属性，且它在构造函数中被使用，所以这个不能被判定为\"避免未使用的临时变量\"\n</例子2>"
+  },
+  {
+    "id": 2,
+    "text": "不要使用 System.out.println 去打印",
+    "language": "Java",
+    "detail": "缺陷类型：不要使用 System.out.println 去打印；对应Fixer：SystemPrintlnFixer；修复方案：注释System.out.println代码",
+    "yes_example": "### 被判定为\"不要使用 System.out.println 去打印\"的例子\n<例子1>\nSystem.out.println(\"Initializing new owner form.\");\n上述代码使用了\"System.out.println\"进行打印，所以这个被判定为\"不要使用 System.out.println 去打印\"\n</例子1>",
+    "no_example": "### 不能被判定为\"不要使用 System.out.println 去打印\"的例子\n<例子1>\nthrow new IllegalStateException(\"There is no authenticated user, we need a user authenticated to find tasks\");\n上述代码是抛出异常的代码，没有使用\"System.out.print\"，所以这个不能被判定为\"不要使用 System.out.println 去打印\"\n</例子1>"
+  },
+  {
+    "id": 3,
+    "text": "避免函数中未使用的形参",
+    "language": "Java",
+    "detail": "缺陷类型：避免函数中未使用的形参；修复方案：忽略",
+    "yes_example": "### 被判定为\"避免函数中未使用的形参\"的例子\n<例子1>\npublic void setTransientVariablesLocal(Map<String, Object> transientVariables) {\n    throw new UnsupportedOperationException(\"No execution active, no variables can be set\");\n}这段代码中的形参\"transientVariables\"未在函数体内出现，所以这个被判定为\"避免函数中未使用的形参\"\n</例子1>\n\n<例子2>\nprotected void modifyFetchPersistencePackageRequest(PersistencePackageRequest ppr, Map<String, String> pathVars) {}\n这段代码中的形参\"ppr\"和\"pathVars\"未在函数体内出现，所以这个被判定为\"避免函数中未使用的形参\"\n</例子2>",
+    "no_example": "### 不能被判定为\"避免函数中未使用的形参\"的例子\n<例子1>\npublic String processFindForm(@RequestParam(value = \"pageNo\", defaultValue = \"1\") int pageNo) {\n\tlastName = owner.getLastName();\n\treturn addPaginationModel(pageNo, paginationModel, lastName, ownersResults);\n}这段代码中的形参\"pageNo\"在当前函数'processFindForm'内被'return addPaginationModel(pageNo, paginationModel, lastName, ownersResults);'这一句被使用，虽然pageNo没有被用于逻辑计算，但作为了函数调用其他函数的参数使用了，所以这个不能被判定为\"避免函数中未使用的形参\"\n</例子1>\n<例子2>\npublic void formatDate(Date date) {\n\tSimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd\");\n\tSystem.out.println(\"Formatted date: \" + sdf.format(date));\n}这段代码中的形参date在System.out.println(\"Formatted date: \" + sdf.format(date))这一句中被引用到，所以这个不能被判定为\"避免函数中未使用的形参\"\n</例子2>"
+  },
+  {
+    "id": 4,
+    "text": "if语句块不能为空",
+    "language": "Java",
+    "detail": "缺陷类型：if 语句块不能为空；对应Fixer：EmptyIfStmtFixer；修复方案：删除if语句块 或 适当的逻辑处理 或 注释说明为何为空",
+    "yes_example": "### 被判定为\"if语句块不能为空\"的例子\n<例子1>\npublic void emptyIfStatement() {\n\tif (getSpecialties().isEmpty()) {\n\t}\n}这段代码中的if语句块内容是空的，所以这个被判定为\"if语句块不能为空\"\n</例子1>\n\n<例子2>\npublic void judgePersion() {\n\tif (persion != null) {\n\t\t// judge persion if not null\n\t}\n}\n这段代码中的if语句块虽然有内容，但是\"// judge persion if not null\"只是代码注释，if语句块内并没有实际的逻辑代码，所以这个被判定为\"if语句块不能为空\"\n</例子2>",
+    "no_example": "### 不能被判定为\"if语句块不能为空\"的例子\n<例子1>\npublic void judgePersion() {\n\tif (persion != null) {\n\t\treturn 0;\n\t}\n}这段代码中的if语句块里有内容，且里面有非注释代码的逻辑代码\"return 0;\"，所以这个不能被判定为\"if语句块不能为空\"\n</例子1>"
+  },
+  {
+    "id": 5,
+    "text": "循环体不能为空",
+    "language": "Java",
+    "detail": "缺陷类型：循环体不能为空；对应Fixer：EmptyStatementNotInLoopFixer；修复方案：删除对应while、for、foreach 循环体 或 添加适当的逻辑处理或者注释说明为何为空",
+    "yes_example": "### 被判定为\"循环体不能为空\"的例子\n<例子1>\npublic void emptyLoopBody() {\n\tfor (Specialty specialty : getSpecialties()) {\n\t}\n}这段代码中的for循环体的内容是空的，所以这个被判定为\"循环体不能为空\"\n</例子1>\n\n<例子2>\npublic void emptyLoopBody() {\n\twhile (True) {\n\t\t// this is a code example\n\t}\n}这段代码中的while循环体的内容虽然不是空的，但内容只是代码注释，无逻辑内容，所以这个被判定为\"循环体不能为空\"\n</例子2>\n\n<例子3>\npublic void emptyLoopBody() {\n\twhile (True) {\n\t\t\n\t}\n}这段代码中的while循环体内容是空的，所以这个被判定为\"循环体不能为空\"\n</例子3>",
+    "no_example": "### 不能被判定为\"循环体不能为空\"的例子\n<例子1>\npublic void emptyLoopBody() {\n\tfor (Specialty specialty : getSpecialties()) {\n\t\ta = 1;\n\t\tif (a == 1) {\n\t\t\tretrun a;\n\t\t}\n\t}\n}上述代码的for循环体的内容不为空，且内容不全是代码注释，所以这个不能被判定为\"循环体不能为空\"\n</例子1>"
+  },
+  {
+    "id": 6,
+    "text": "避免使用 printStackTrace()，应该使用日志的方式去记录",
+    "language": "Java",
+    "detail": "缺陷类型：避免使用 printStackTrace()，应该使 用日志的方式去记录；修复方案：用日志的方式去记录",
+    "yes_example": "### 被判定为\"避免使用 printStackTrace()，应该使用日志的方式去记录\"的例子\n<例子1>\npublic void usePrintStackTrace() {\n\ttry {\n\t\tthrow new Exception(\"Fake exception\");\n\t} catch (Exception e) {\n\t\te.printStackTrace();\n\t}\n}这段代码中的catch语句中使用了printStackTrace()，所以这个被判定为\"避免使用 printStackTrace()，应该使用日志的方式去记录\"\n</例子1>",
+    "no_example": "### 不能被判定为\"避免使用 printStackTrace()，应该使用日志的方式去记录\"的例子\n<例子1>\npublic void usePrintStackTrace() {\n\ttry {\n\t\tthrow new Exception(\"Fake exception\");\n\t} catch (Exception e) {\n\t\tlogging.info(\"info\");\n\t}\n}这段代码的catch语句中使用的是日志记录的方式，所以这个不能被判定为\"避免使用 printStackTrace()，应该使用日志的方式去记录\"\n</例子1>"
+  },
+  {
+    "id": 7,
+    "text": "catch 语句块不能为空",
+    "language": "Java",
+    "detail": "缺陷类型：catch 语句块不能为空；对应Fixer：EmptyCatchBlockFixer；修复方案：在catch里面添加注释",
+    "yes_example": "### 被判定为\"catch语句块不能为空\"的例子\n<例子1>\ntry {\n    int[] array = new int[5];\n    int number = array[10];\n} catch (ArrayIndexOutOfBoundsException e) {\n    \n}\n这段代码中的catch语句中没有内容，所以这个被判定为\"catch语句块不能为空\"\n</例子1>\n\n<例子2>\ntry {\n    String str = null;\n    str.length();\n} catch (NullPointerException e) {\n   \n}这段代码中的catch语句中没有内容，所以这个被判定为\"catch语句块不能为空\"\n</例子2>\n\n<例子3>\npublic class EmptyCatchExample {\n    public static void main(String[] args) {\n        try {\n            // 尝试除以零引发异常\n            int result = 10 / 0;\n        } catch (ArithmeticException e) {\n         \n        }\n    }\n}这段代码中的catch语句中没有内容，所以这个被判定为\"catch语句块不能为空\"\n</例子3>\n<例子4>\ntry {\n    FileReader file = new FileReader(\"nonexistentfile.txt\");\n} catch (FileNotFoundException e) {\n    \n}这段代码中的catch语句中没有内容，所以这个被判定为\"catch语句块不能为空\"\n</例子4>\n<例子5>\ntry {\n    Object obj = \"string\";\n    Integer num = (Integer) obj;\n} catch (ClassCastException e) {\n\t\n}这段代码中的catch语句中没有内容，所以这个被判定为\"catch语句块不能为空\"\n</例子5>",
+    "no_example": "### 不能被判定为\"catch语句块不能为空\"的例子\n<例子1>\npersionNum = 1\ntry {\n\treturn True;\n} catch (Exception e) {\n\t// 如果人数为1则返回false\n\tif (persionNum == 1){\n\t\treturn False;\n\t}\n}这段代码的catch语句中不为空，所以不能把这个被判定为\"catch语句块不能为空\"\n</例子1>\n\n<例子2>\ntry {\n\tthrow new Exception(\"Fake exception\");\n} catch (Exception e) {\n\te.printStackTrace();\n}这段代码的catch语句中虽然只有\"e.printStackTrace();\"但确实不为空，所以不能把这个被判定为\"catch语句块不能为空\"\n</例子2>"
+  },
+  {
+    "id": 8,
+    "text": "避免不必要的永真/永假判断",
+    "language": "Java",
+    "detail": "缺陷类型：避免不必要的永真/永假判断；对应Fixer：UnconditionalIfStatement Fixer；修复方案：删除永真/永假判断逻辑",
+    "yes_example": "### 被判定为\"避免不必要的永真/永假判断\"的例子\n<例子1>\npublic void someMethod() {\n\twhile (true) {\n\t}\n}这段代码中的\"while (true)\"是一个使用true做判断条件，但是没有循环结束标记，所以这个被判定为\"避免不必要的永真/永假判断\"\n</例子1>\n\n<例子2>\nif (true) {\n\tSystem.out.println(\"This is always true\");\n}这段代码中的\"if (true)\"是一个使用true条件做条件，但是没有循环结束标记，所以这个被判定为\"避免不必要的永真/永假判断\"\n</例子2>\n\n<例子3>\na = 1;\nwhile(a > 0){\n\ta = a + 1\n}这段代码初始化a=1，是大于0的，while循环体的逻辑是每次加1，那么判断条件a > 0会永远是真的，不会退出循环，所以这个被判定为\"避免不必要的永真/永假判断\"\n<例子3>",
+    "no_example": "### 不能被判定为\"避免不必要的永真/永假判断\"的例子\n<例子1>\na = 0;\nwhile (a < 5) {\n\ta = a + 1;\n}这段代码中的a<5是一个判断，当执行了5次while语句中的逻辑a=a+1之后，a会满足a < 5，就会退出循环，所以这个能被判定为\"避免不必要的永真/永假判断\"\n</例子1>"
+  },
+  {
+    "id": 9,
+    "text": "switch 中 default 必须放在最后",
+    "language": "Java",
+    "detail": "缺陷类型：switch 中 default 必须放在最后；对应Fixer：DefaultLabelNotLastInSwitchStmtFixer；修复方案：switch 中 default 放在最后",
+    "yes_example": "### 被判定为\"switch 中 default 必须放在最后\"的例子\n<例子1>\nswitch (number) {\n\tdefault:\n\t\tSystem.out.println(\"This is the default block, which is incorrectly placed here.\");\n\t\tbreak;\n\tcase 1:\n\t\tSystem.out.println(\"Number one\");\n\t\tbreak;\n\tcase 2:\n\t\tSystem.out.println(\"Number two\");\n\t\tbreak;\n}这段代码是一个switch语句，但是里面的default没有放在最后，所以这个被判定为\"switch 中 default 必须放在最后\"\n</例子1>",
+    "no_example": "### 不能被判定为\"switch 中 default 必须放在最后\"的例子\n<例子1>\nswitch (number) {\ncase 3:\n\tSystem.out.println(\"Number one\");\n\tbreak;\ncase 4:\n\tSystem.out.println(\"Number two\");\n\tbreak;\ndefault:\n\tSystem.out.println(\"This is the default block, which is incorrectly placed here.\");\n\tbreak;\n}这段代码是一个switch语句且里面的default放在了最后，所以这个不能被判定为\"switch 中 default 必须放在最后\"\n</例子1>"
+  },
+  {
+    "id": 10,
+    "text": "未使用equals()函数对 String 作比较",
+    "language": "Java",
+    "detail": "缺陷类型：未使用equals()函数对 String 作比较；对应Fixer：UnSynStaticDateFormatter Fixer；修复方案：使用equals()函数对 String 作比较",
+    "yes_example": "### 被判定为\"未使用equals()函数对 String 作比较\"的例子\n<例子1>\nif (existingPet != null && existingPet.getName() == petName) {\n\tresult.rejectValue(\"name\", \"duplicate\", \"already exists\");\n}这段代码中所涉及的existingPet.getName()和petName均是字符串，但是在if语句里做比较的时候使用了==而没有使用equals()对string做比较，所以这个被判定为\"未使用equals()函数对 String 作比较\"\n</例子1>\n\n<例子2>\nString isOk = \"ok\";\nif (\"ok\" == isOk) {\n\tresult.rejectValue(\"name\", \"duplicate\", \"already exists\");\n}这段代码中的isOk是个字符串，但在if判断中与\"ok\"比较的时候使用的是==，未使用equals()对string做比较，应该使用\"ok\".equals(isOk)，所以这个被判定为\"未使用equals()函数对 String 作比较\"\n</例子2>\n\n<例子3>\nString str1 = \"Hello\";\nString str2 = \"Hello\";\nif (str1 == str2) {\n\tSystem.out.println(\"str1 和 str2 引用相同\");\n} else {\n\tSystem.out.println(\"str1 和 str2 引用不同\");\n}\n这段代码中的if (str1 == str2) 使用了==进行str1和str2的比较，未使用equals()对string做比较，应该使用str1.equals(str2)，所以这个被判定为\"未使用equals()函数对 String 作比较\"\n</例子3>\n\n<例子4>\nString str = \"This is string\";\nif (str == \"This is not str\") {\n\treturn str;\n}这段代码中的if (str == \"This is not str\")使用了==进行字符串比较，未使用equals()对string做比较，\"This is not str\".equals(str)，所以这个被判定为\"未使用equals()函数对 String 作比较\"\n</例子4>",
+    "no_example": "### 不能被判定为\"未使用equals()函数对 String 作比较\"的例子\n<例子1>\nif (PROPERTY_VALUE_YES.equalsIgnoreCase(readWriteReqNode))\n  formProperty.setRequired(true);\n这段代码中的PROPERTY_VALUE_YES和readWriteReqNode均是字符串，在if语句里比较PROPERTY_VALUE_YES和readWriteReqNode的使用的是equalsIgnoreCase（字符串比较忽略大小写），所以equalsIgnoreCase也是符合使用equals()函数对 String 作比较的，所以这个不能被判定为\"未使用equals()函数对 String 作比较\"\n</例子1>\n\n<例子2>\nString isOk = \"ok\";\nif (\"ok\".equals(isOk)) {\n\tresult.rejectValue(\"name\", \"duplicate\", \"already exists\");\n}这段代码中的isOk是个字符串，在if判断中与\"ok\"比较的时候使用的是equals()对string做比较，所以这个不能被判定为\"未使用equals()函数对 String 作比较\"\n</例子2>"
+  },
+  {
+    "id": 11,
+    "text": "禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象",
+    "language": "Java",
+    "detail": "缺陷类型：禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象 输出异常；对应Fixer：ConcatExceptionFixer；修复方案：使用占位符传递异常对象",
+    "yes_example": "### 被判定为\"禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象\"的例子\n<例子1>\ntry {\n  listenersNode = objectMapper.readTree(listenersNode.asText());\n} catch (Exception e) {\n  LOGGER.info(\"Listeners node can not be read\", e);\n}这段代码中日志输出内容内容是直接使用字符串\"Listeners node can not be read\"拼接，日志输出异常时，应使用占位符输出异常信息，而不是直接使用字符串拼接，所以这个被判定为\"禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象\"\n</例子1>",
+    "no_example": "### 不能被判定为\"禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象\"的例子\n<例子1>\nPersion persion = persionService.getPersion(1);\nif (persion == null){\n\tLOGGER.error(PERSION_NOT_EXIT);\n}这段代码中的PERSION_NOT_EXIT是一个用户自定义的异常常量，代表persion不存在，没有直接使用字符串\"persion not exit\"拼接，所以这个不能被判定为\"禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象\"\n<例子1>\n\n<例子2>\ntry {\n  a = a + 1;\n} catch (Exception e) {\n  Persion persion = persionService.getPersion(1);\n  LOGGER.info(persion);\n}这段代码中输出日志没有直接使用字符串拼接，而是使用的Persion对象输出，所以这个不能被判定为\"禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象\"\n</例子2>"
+  },
+  {
+    "id": 12,
+    "text": "finally 语句块不能为空",
+    "language": "Java",
+    "detail": "缺陷类型：finally 语句块不能为空；对应Fixer：EmptyFinallyBlockFixer；修复方案：删除空 finally 语句块",
+    "yes_example": "### 被判定为\"finally 语句块不能为空\"的例子\n<例子1>\ntry {\n\tPersion persion = persionService.getPersion(1);\n\treturn persion;\n} finally {\n\t\n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n</例子1>\n\n<例子2>\ntry {\n\tSystem.out.println(\"Inside try block\");\n} finally {\n\t// 空的finally块，没有任何语句，这是一个缺陷\n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n</例子2>\n\n<例子3>\ntry {\n    int result = 10 / 0;\n} catch (ArithmeticException e) {\n    e.printStackTrace();\n} finally {\n    \n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n</例子3>\n\n<例子4>\ntry {\n    String str = null;\n    System.out.println(str.length());\n} catch (NullPointerException e) {\n    e.printStackTrace();\n} finally {\n    \n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n</例子4>\n\n<例子5>\ntry {\n    int[] array = new int[5];\n    int number = array[10];\n} catch (ArrayIndexOutOfBoundsException e) {\n    e.printStackTrace();\n} finally {\n    // 只有注释的 finally 语句块\n    // 这是一个空的 finally 块\n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n</例子5>\n\n<例子6>\ntry {\n    FileReader file = new FileReader(\"nonexistentfile.txt\");\n} catch (FileNotFoundException e) {\n    e.printStackTrace();\n} finally {\n    // 只有空行的 finally 语句块\n    \n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n</例子6>",
+    "no_example": "### 不能被判定为\"finally 语句块不能为空\"的例子\n<例子1>\npublic void getPersion() {\n\ttry {\n\t\tPersion persion = persionService.getPersion(1);\n\t\tif (persion != null){ \n\t\t\treturn persion;\n\t\t}\n\t} finally {\n\t\treturn null;\n\t}\n}这段代码中的finally语句块中有非注释意外的内容\"return null;\"，所以这个不能被判定为\"finally 语句块不能为空\"\n</例子1>"
+  },
+  {
+    "id": 13,
+    "text": "try 语句块不能为空",
+    "language": "Java",
+    "detail": "缺陷类型：try 语句块不能为空；对应Fixer：EmptyTryBlockFixer；修复方案：删除整个 try 语句",
+    "yes_example": "### 被判定为\"try 语句块不能为空\"的例子\n<例子1>\npublic void getPersion() {\n\ttry {\n\n\t}\n\treturn null;\n}这段代码中的try语句块内没有内容，所以这个被判定为\"try 语句块不能为空\"\n</例子1>\n\n<例子2>\npublic void demoFinallyBlock() {\n\ttry {\n\n\t} finally {\n\t\treturn null;\n\t}\n}这段代码中的try语句块内没有内容，所以这个被判定为\"try 语句块不能为空\"\n</例子2>\n\n<例子3>\ntry {\n    \n} catch (Exception e) {\n    e.printStackTrace();\n}这段代码中的try语句块内没有内容，所以这个被判定为\"try 语句块不能为空\"\n</例子3>\n\n<例子4>\ntry {\n    // 只有注释的 try 语句块\n\t\n} catch (Exception e) {\n    e.printStackTrace();\n}这段代码中的try语句块内只有注释和空行，也可以认定为这种情况是try语句块内没有内容，所以这个被判定为\"try 语句块不能为空\"\n</例子4>",
+    "no_example": "### 不能被判定为\"try 语句块不能为空\"的例子\n<例子1>\ntry {\n\ta = a + 1;\n} catch (Exception e) {\n\te.printStackTrace();\n}\n这段代码中的try语句块中有非注释意外的内容\"return null;\"，所以这个不能被判定为\"try 语句块不能为空\"\n</例子1>"
+  },
+  {
+    "id": 14,
+    "text": "避免对象进行不必要的 NULL或者null 检查",
+    "language": "Java",
+    "detail": "缺陷类型：避免对象进行不必要的 NULL或者null 检查；对应Fixer：LogicalOpNpeFixer；修复方案：删除对对象不必要的 NULL 检查的逻辑",
+    "yes_example": "### 被判定为\"避免对象进行不必要的 NULL或者null 检查\"的例子\n<例子1>\na = \"dog\";\nif (a != null){\n\treturn a;\n}这段代码中的对象a已经是确定的值\"dog\"，所以if条件句的判断\"a != null\"是不必要的，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n</例子1>\n\n<例子2>\nif (authenticatedUserId != null && !authenticatedUserId.isEmpty() && userGroupManager!=null){\n\treturn authenticatedUserId;\n}这段代码中的\"authenticatedUserId != null\"和\"!authenticatedUserId.isEmpty()\"都是对\"authenticatedUserId\"的空判断，重复了，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n</例子2>\n\n<例子3>\nList<Integer> list = new ArrayList<>();\nif (list != null) {\n    list.add(1);\n}这段代码中的list已经被初始化，不需要进行 null 检查，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n</例子3>\n\n<例子4>\nif (this.type != null && this.type.getName() != null) {\n\tSystem.out.println(\"Type name is not null\");\n}这段代码中的对象type已经检查过非null，再次检查getName()是否为null是不必要的，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n\n</例子4>\n\n<例子5>\nif (\"dog\".equals(null)){\n\treturn a;\n}这段代码中的\"dog\"是个确定的字符串，不需要进行null 检查，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n</例子5>\n\n<例子6>\nInteger num = 10;\nif (num != null) {\n    System.out.println(num);\n}这段代码中的num 已经被初始化，不需要进行 null 检查，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n</例子6>",
+    "no_example": "### 不能被判定为\"避免对象进行不必要的 NULL或者null 检查\"的例子\n<例子1>\nCat cat = catService.get(1);\nif (cat != null){\n\tretrun cat;\n}这段代码中的对象\"cat\"是通过service获取到的，不确定是否为空，所以if条件句的判断的\"cat != null\"是必要的，所以这个不能被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n</例子1>"
+  },
+  {
+    "id": 15,
+    "text": "避免 finally 块中出现 return",
+    "language": "Java",
+    "detail": "缺陷类型：避免 finally 块中出现 return；修复方案：无需修复",
+    "yes_example": "### 被判定为\"避免 finally 块中出现 return\"的例子\n<例子1>\npublic void getPersion() {\n\ttry {\n\t\tPersion persion = persionService.getPersion(1);\n\t\tif (persion != null){ \n\t\t\treturn persion;\n\t\t}\n\t} finally {\n\t\treturn null;\n\t}\n}这段代码中的finally语句块内容包含\"return\"，所以这个被判定为\"避免 finally 块中出现 return\"\n</例子1>",
+    "no_example": "### 不能被判定为\"避免 finally 块中出现 return\"的例子\n<例子1>\npublic void getPersion() {\n\ttry {\n\t\tPersion persion = persionService.getPersion(1);\n\t\tif (persion != null){ \n\t\t\treturn persion;\n\t\t}\n\t} finally {\n\t\tLOGGER.info(PERSION_NOT_EXIT);\n\t}\n}这段代码中的finally语句块中内容不包含\"return\"，所以这个不能被判定为\"避免 finally 块中出现 return\"\n</例子1>"
+  },
+  {
+    "id": 16,
+    "text": "避免空的 static 初始化",
+    "language": "Java",
+    "detail": "缺陷类型：避免空的 static 初始化；对应Fixer：EmptyInitializerFixer；修复方案：删除整个空初始化块",
+    "yes_example": "### 被判定为\"避免空的 static 初始化\"的例子\n<例子1>\npublic class PetValidator implements Validator {\n\tstatic {\n\n\t}\n}这段代码中的static语句块没有内容，是空的，所以这个被判定为\"避免空的 static 初始化\"\n</例子1>\n\n<例子2>\npublic class Persion {\n\tstatic {\n\t\t// 初始化的静态块\n\t}\n}这段代码中的static语句块是有内容的，不是空的，但是static初始化语句块中只有注释代码，没有实际的逻辑，所以这个被判定为\"避免空的 static 初始化\"\n</例子2>",
+    "no_example": "### 不能被判定为\"避免空的 static 初始化\"的例子\n<例子1>\npublic class Cat {\n\tstatic {\n\t\t// 初始化的静态块\n\t\tcat = null;\n\t}\n}这段代码中的static语句块是有内容的，不是空的，且static初始化语句块中有非注释代码，有实际的逻辑，所以这个不能被判定为\"避免空的 static 初始化\"\n</例子1>"
+  },
+  {
+    "id": 17,
+    "text": "避免日历类用法不当风险",
+    "language": "Java",
+    "detail": "缺陷类型：避免日历类用法不当风险；修复方案：使用Java 8 及以上版本中的 java.time 包的LocalDate",
+    "yes_example": "### 被判定为\"避免日历类用法不当风险\"的例子\n<例子1>\nprivate static final Calendar calendar = new GregorianCalendar(2020, Calendar.JANUARY, 1);\n这段代码中的Calendar和GregorianCalendar是线程不安全的，所以这个被判定为\"避免日历类用法不当风险\"\n</例子1>",
+    "no_example": "### 不能被判定为\"避免日历类用法不当风险\"的例子\n<例子1>\nprivate static final LocalDate calendar = LocalDate.of(2020, 1, 1);\n这段代码中的LocalDate使用的是Java 8 及以上版本中的 java.time 包，LocalDate 是不可变的并且是线程安全的，不会有线程安全和性能方面的问题，所以这个不能被判定为\"避免日历类用法不当风险\"\n</例子1>"
+  },
+  {
+    "id": 18,
+    "text": "使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()",
+    "language": "Java",
+    "detail": "缺陷类型：使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()；对应Fixer：ClassCastExpWithToArrayF ixer；修复方案：使用集合的toArray(T[]array)，且传入的是类型完全一样的数组",
+    "yes_example": "### 被判定为\"使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()\"的例子\n<例子1>\nList<String> stringList = new ArrayList<>();\nstringList.add(\"Apple\");\nstringList.add(\"Banana\");\nObject[] objectArray = stringList.toArray(new Object[5]);\n这段代码使用集合转数组的方法的时候使用了toArray(new Object[5])，但是传入的数组类型不一致，所以这个被判定为\"使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()\"\n</例子1>",
+    "no_example": "### 不能被判定为\"使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()\"的例子\n<例子1>\nList<String> stringList = new ArrayList<>();\nstringList.add(\"Apple\");\nstringList.add(\"Banana\");\nString[] stringArray = stringList.toArray(new String[stringList.size()]);\n这段代码使用集合转数组的方法的时候使用了toArray(new String[stringList.size()])，传入的是类型完全一样的数组，所以这个不能被判定为\"使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()\"\n</例子1>"
+  },
+  {
+    "id": 19,
+    "text": "禁止在 equals()中使用 NULL或者null 做比较",
+    "language": "Java",
+    "detail": "缺陷类型：禁止在 equals()中使用 NULL或者null 做比较；对应Fixer：EqualsNullFixer；修复方案：使用Object的判空函数 做比较",
+    "yes_example": "### 被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"的例子\n<例子1>\nif (\"test\".equals(null)) {\n\tSystem.out.println(\"test\");\n}这段代码中if条件中的代码\"test\".equals(null)使用equals()函数与null进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n</例子1>\n\n<例子2>\nif (!rangeValues[1].equals(\"null\")) {\n\tmaxValue = new BigDecimal(rangeValues[1]);\n}这段代码中if条件中的代码!rangeValues[1].equals(\"null\")使用equals()函数与Nnull进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n</例子2>\n\n<例子3>\nString str1 = \"example\";\nif (str1.equals(\"null\")) {\n    System.out.println(\"str1 is null\");\n}这段代码中if条件中的代码str1.equals(null)使用equals()函数与null进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n</例子3>\n\n<例子4>\nString str3 = \"example\";\nif (str3 != null && str3.equals(\"null\")) {\n    System.out.println(\"str3 is null\");\n}这段代码中if条件中的代码str3.equals(\"null\")使用equals()函数与\"null\"进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n</例子4>\n\n<例子5>\nInteger num1 = 10;\nif (num1.equals(null)) {\n    System.out.println(\"num1 is null\");\n}这段代码中if条件中的代码num1.equals(null)使用equals()函数与\"null\"进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n</例子5>\n\n<例子6>\nObject obj = new Object();\nif (obj.equals(null)) {\n    System.out.println(\"obj is null\");\n}这段代码中if条件中的代码obj.equals(null)使用equals()函数与\"null\"进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n</例子6>",
+    "no_example": "### 不能被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"的例子\n<例子1>\na = \"test\";\nif (a.equals(\"test\")) {\n\tSystem.out.println(\"test\");\n}这段代码中if条件中的代码a.equals(\"test\")使用equals()函数与\"test\"进行了比较，所以这个不能被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n</例子1>"
+  },
+  {
+    "id": 20,
+    "text": "switch 语句块不能为空",
+    "language": "Java",
+    "detail": "缺陷类型：switch 语句块不能为空；对应Fixer：EmptySwitchStatementsFix；修复方案：删除整个空 switch 语句块",
+    "yes_example": "### 被判定为\"switch 语句块不能为空\"的例子\n<例子1>\nswitch (number) {\n\t\n}这段代码是一个switch语句块，但是里面没有内容，所以这个被判定为\"switch 语句块不能为空\"\n</例子1>\n\n<例子2>\nswitch (number) {\n\t// 这是一个switch语句块\n}这段代码是一个switch语句块，里面虽然有内容，但是内容仅仅是注释内容，没有实际的逻辑，所以这个被判定为\"switch 语句块不能为空\"\n</例子2>",
+    "no_example": "### 不能被判定为\"switch 语句块不能为空\"的例子\n<例子1>\nswitch (number) {\n\tcase 1:\n\t\tSystem.out.println(\"Number one\");\n\t\tbreak;\n\tdefault:\n\t\tSystem.out.println(\"This is the default block, which is incorrectly placed here.\");\n\t\tbreak;\n}这段代码是一个switch语句块，里面有内容，而且内容里有非注释的代码，有实际的逻辑，所以这个不能被判定为\"switch 语句块不能为空\"\n</例子1>"
+  },
+  {
+    "id": 21,
+    "text": "在进行类型强制转换时，右括号与强制转换值之间不需要任何空格隔开",
+    "detail": "缺陷类型：在进行类型强制转换时，右括号与强制转换值之间不需要任何空格隔开；修复方案：在进行类型强制转换时，右括号与强制转换值之间不需要任何空格隔开。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"在进行类型强制转换时，右括号与强制转换值之间不需要任何空格隔开\"的例子\n<例子1>\nint a = (int) 3.0;\n</例子1>\n<例子2>\nint b = (int) 4.0;\n</例子2>\n<例子3>\nlong a = (long) 5;\n</例子3>\n<例子4>\nstring a = (string) 3.5;\n</例子4>\n<例子5>\nPersion a = (Persion) \"zhangsan\";\n</例子5>",
+    "no_example": "### 不能被判定为\"在进行类型强制转换时，右括号与强制转换值之间不需要任何空格隔开\"的例子\n<例子1>\nint a = (int)3.0;\n</例子1>"
+  },
+  {
+    "id": 22,
+    "text": "方法参数在定义和传入时，多个参数逗号后面必须加空格",
+    "detail": "缺陷类型：方法参数在定义和传入时，多个参数逗号后面必须加空格；修复方案：方法参数在定义和传入时，多个参数逗号后面必须加空格。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"方法参数在定义和传入时，多个参数逗号后面必须加空格\"的例子\n<例子1>\npublic void exampleMethod(int a,int b,int c) {}\n</例子1>",
+    "no_example": "### 不能被判定为\"方法参数在定义和传入时，多个参数逗号后面必须加空格\"的例子\n<例子1>\npublic void exampleMethod(int a, int b, int c) {}\n</例子1>"
+  },
+  {
+    "id": 23,
+    "text": "禁止使用构造方法 BigDecimal(double) 的方式把 double 值转化为 BigDecimal 对象",
+    "detail": "缺陷类型：禁止使用构造方法 BigDecimal(double) 的方式把 double 值转化为 BigDecimal 对象；修复方案：推荐使用 BigDecimal 的 valueOf 方法。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"禁止使用构造方法 BigDecimal(double) 的方式把 double 值转化为 BigDecimal 对象\"的例子\n<例子1>\nBigDecimal bd = new BigDecimal(0.1);\n</例子1>",
+    "no_example": "### 不能被判定为\"禁止使用构造方法 BigDecimal(double) 的方式把 double 值转化为 BigDecimal 对象\"的例子\n<例子1>\nBigDecimal bd = BigDecimal.valueOf(0.1);\n</例子1>"
+  },
+  {
+    "id": 24,
+    "text": "不能有多余的分号",
+    "detail": "缺陷类型：多余的分号；修复方案：删除多余的分号",
+    "yes_example": "### 被判定为\"不能有多余的分号\"的例子\n<例子1>\npublic void trigger(String executionId, Map<String, Object> processVariables) {\n    commandExecutor.execute(new TriggerCmd(executionId, processVariables));\n}\n;\na = 1;\nb = 2;\nsum = a + b;\n这段代码中包含一个多余的分号\";\"，所以这个被判定为\"不能有多余的分号\"\n</例子1>",
+    "no_example": "### 不能被判定为\"不能有多余的分号\"的例子\n<例子1>\nwhile (True) {\n\ta = a + 1;\n\tbreak;\n}这段代码每个分号都是必须要的，所以这个能被判定为\"不能有多余的分号\"\n</例子1>"
+  },
+  {
+    "id": 25,
+    "text": "非线程安全的 SimpleDateFormat 使用，必须在函数或代码块级别使用synchronized",
+    "detail": "缺陷类型：非线程安全的 SimpleDateFormat 使用；修复方案：在函数或代码块级别加上synchronized修饰 或 使用其他线程安全的方式",
+    "yes_example": "### 被判定为\"非线程安全的 SimpleDateFormat 使用，必须在函数或代码块级别使用synchronized\"的例子\n<例子1>\npublic void formatDate(Date date) {\n\tSimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd\");\n\tSystem.out.println(\"Formatted date: \" + sdf.format(date));\n}这段代码中的函数formatDate在未使用synchronized同步修饰的情况下使用了SimpleDateFormat，这是线程不安全的，所以这个被判定为\"非线程安全的 SimpleDateFormat 使用，必须在函数或代码块级别使用synchronized\"\n</例子1>",
+    "no_example": "### 不能被判定为\"非线程安全的 SimpleDateFormat 使用，必须在函数或代码块级别使用synchronized\"的例子\n<例子1>\npublic synchronized void formatDate(Date date) {\n\tSimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd\");\n\tSystem.out.println(\"Formatted date: \" + sdf.format(date));\n}这段代码是在synchronized同步块对函数'formatDate'进行保护，保证了线程安全，所以这个不能被判定为\"非线程安全的 SimpleDateFormat 使用，必须在函数或代码块级别使用synchronized\"\n</例子1>"
+  },
+  {
+    "id": 26,
+    "text": "未按驼峰命名规范进行命名，类名使用驼峰式UpperCamelCase风格, 方法名、参数名、成员变量、局部变量都统一使用lowerCamelCase风格",
+    "detail": "缺陷类型：未按驼峰命名规范进行命名；修复方案：类名使用UpperCamelCase风格，方法名、参数名、成员变量、局部变量使用lowerCamelCase风格。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"未按驼峰命名规范进行命名\"的例子\n<例子1>\npublic class myClass {\n    private int MyVariable;\n    public void MyMethod() {}\n}\n这段代码中的类名、成员变量和方法名没有遵循驼峰命名法，所以被判定为命名规范问题。\n</例子1>",
+    "no_example": "### 不能被判定为\"未按驼峰命名规范进行命名\"的例子\n<例子1>\npublic class MyClass {\n    private int myVariable;\n    public void myMethod() {}\n}\n这段代码中的类名、成员变量和方法名都遵循了驼峰命名法，所以不能被判定为命名规范问题。\n</例子1>"
+  },
+  {
+    "id": 27,
+    "text": "抽象类命名使用 Abstract 或 Base 开头；异常类命名使用 Exception 结尾，测试类命名以它要测试的类的名称开始，以 Test 结尾",
+    "detail": "缺陷类型：命名规范；修复方案：抽象类命名使用 Abstract 或 Base 开头，异常类命名使用 Exception 结尾，测试类命名以它要测试的类的名称开始，以 Test 结尾。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"命名规范\"的例子\n<例子1>\npublic class MyAbstractClass {}\npublic class MyExceptionClass {}\npublic class TestMyClass {}\n这段代码中的抽象类、异常类和测试类的命名不符合规范，所以被判定为命名规范问题。\n</例子1>",
+    "no_example": "### 不能被判定为\"命名规范\"的例子\n<例子1>\npublic abstract class AbstractMyClass {}\npublic class MyCustomException extends Exception {}\npublic class MyClassTest {}\n这段代码中的抽象类、异常类和测试类的命名都符合规范，所以不能被判定为命名规范问题。\n</例子1>"
+  },
+  {
+    "id": 28,
+    "text": "POJO 类中的任何布尔类型的变量，避免加\"is\" 前缀",
+    "detail": "缺陷类型：命名规范；修复方案：POJO 类中的布尔类型变量不要加 is 前缀。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"命名规范\"的例子\n<例子1>\npublic class User {\n    private boolean isActive;\n}\n这段代码中的布尔类型变量加了 is 前缀，所以被判定为命名规范问题。\n</例子1>",
+    "no_example": "### 不能被判定为\"命名规范\"的例子\n<例子1>\npublic class User {\n    private boolean active;\n}\n这段代码中的布尔类型变量没有加 is 前缀，所以不能被判定为命名规范问题。\n</例子1>"
+  },
+  {
+    "id": 29,
+    "text": "杜绝完全不规范的英文缩写，避免望文不知义。",
+    "detail": "缺陷类型：命名规范；修复方案：避免使用不规范的英文缩写，确保代码可读性。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"命名规范\"的例子\n<例子1>\npublic class CfgMgr {\n    private int cnt;\n}\n这段代码中的类名和变量名使用了不规范的英文缩写，所以被判定为命名规范问题。\n</例子1>",
+    "no_example": "### 不能被判定为\"命名规范\"的例子\n<例子1>\npublic class ConfigManager {\n    private int count;\n}\n这段代码中的类名和变量名没有使用不规范的英文缩写，所以不能被判定为命名规范问题。\n</例子1>"
+  },
+  {
+    "id": 30,
+    "text": "避免出现魔法字符和数字，应声明为常量",
+    "detail": "缺陷类型：避免出现魔法字符和数字，应声明为常量；修复方案：将魔法值定义为常量。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"避免出现魔法字符和数字，应声明为常量\"的例子\n<例子1>\npublic class MagicNumberExample {\n    public void calculate() {\n        int result = 42 * 2;\n    }\n}\n这段代码中直接使用了魔法值 42，所以被判定为代码规范问题。\n</例子1>\n<例子2>\npublic class MagicNumberExample {\n    public void calculate() {\n        String result = \"This is a result\";\n    }\n}\n这段代码中直接使用了魔法值 \"This is a result\"，所以被判定为代码规范问题。\n</例子2>",
+    "no_example": "### 不能被判定为\"避免出现魔法字符和数字，应声明为常量\"的例子\n<例子1>\npublic class MagicNumberExample {\n    private static final int MULTIPLIER = 42;\n    public void calculate() {\n        int result = MULTIPLIER * 2;\n    }\n}\n这段代码中将魔法值定义为了常量，所以不能被判定为代码规范问题。\n</例子1>"
+  },
+  {
+    "id": 31,
+    "text": "long 或 Long 赋值时，数值后使用大写 L，不能是小写 l，浮点数类型的数值后缀统一为大写的 D 或 F",
+    "detail": "缺陷类型：代码规范；修复方案：long 或 Long 赋值时使用大写 L，浮点数类型的数值后缀使用大写的 D 或 F。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"代码规范\"的例子\n<例子1>\npublic class NumberExample {\n    private long value = 1000l;\n    private double pi = 3.14d;\n}\n这段代码中使用了小写的 l 和 d，所以被判定为代码规范问题。\n</例子1>",
+    "no_example": "### 不能被判定为\"代码规范\"的例子\n<例子1>\npublic class NumberExample {\n    private long value = 1000L;\n    private double pi = 3.14D;\n}\n这段代码中使用了大写的 L 和 D，所以不能被判定为代码规范问题。\n</例子1>"
+  },
+  {
+    "id": 32,
+    "text": "如果大括号内为空，简洁地写成{}即可，大括号中间无需换行和空格；如果是非空代码块，则：1）左大括号前不换行。2）左大括号后换行。3）右大括号前换行。4）右大括号后还有 else 等代码则不换行；表示终止的右大括号后必须换行。",
+    "detail": "缺陷类型：代码格式；修复方案：遵循大括号的使用规范。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"代码格式\"的例子\n<例子1>\npublic class BracketExample{public void method(){\n        if (true) {\n        }}\n}\n这段代码中的大括号使用不符合规范，所以被判定为代码格式问题。\n</例子1>",
+    "no_example": "### 不能被判定为\"代码格式\"的例子\n<例子1>\npublic class BracketExample {\n    public void method() {\n        if (true) {\n            // do something\n        }\n    }\n}\n这段代码中的大括号使用符合规范，所以不能被判定为代码格式问题。\n</例子1>"
+  },
+  {
+    "id": 33,
+    "text": "左小括号和右边相邻字符之间不需要空格；右小括号和左边相邻字符之间也不需要空格；而左大括号前需要加空格。",
+    "detail": "缺陷类型：代码格式；修复方案：遵循括号和空格的使用规范。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"代码格式\"的例子\n<例子1>\npublic class SpaceExample {\n    public void method (){\n    }\n}\n这段代码中的括号和空格使用不符合规范，所以被判定为代码格式问题。\n</例子1>",
+    "no_example": "### 不能被判定为\"代码规范\"的例子\n<例子1>\npublic class SpaceExample {\n    public void method() {}\n}\n这段代码中的括号和空格使用符合规范，所以不能被判定为代码格式问题。\n</例子1>"
+  },
+  {
+    "id": 34,
+    "text": "if / for / while / switch / do 等保留字与左右括号之间都必须加空格。",
+    "detail": "缺陷类型：代码格式；修复方案：保留字与左右括号之间加空格。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"代码规范\"的例子\n<例子1>\npublic class KeywordExample {\n    public void method() {\n        if(true) {\n        }\n    }\n}\n这段代码中的 if 关键字与括号之间没有空格，所以被判定为代码格式问题。\n</例子1>",
+    "no_example": "### 不能被判定为\"代码规范\"的例子\n<例子1>\npublic class KeywordExample {\n    public void method() {\n        if (true) {\n        }\n    }\n}\n这段代码中的 if 关键字与括号之间有空格，所以不能被判定为代码格式问题。\n</例子1>"
+  },
+  {
+    "id": 35,
+    "text": "所有整型包装类对象之间值的比较，全部使用 equals 方法比较",
+    "detail": "缺陷类型：代码规范；修复方案：整型包装类对象之间的值比较使用 equals 方法。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"代码规范\"的例子\n<例子1>\npublic class IntegerComparison {\n    public void compare() {\n        Integer a = 100;\n        Integer b = 100;\n        if (a == b) {\n        }\n    }\n}\n这段代码中使用了 == 比较整型包装类对象，所以被判定为代码规范问题。\n</例子1>",
+    "no_example": "### 不能被判定为\"代码规范\"的例子\n<例子1>\npublic class IntegerComparison {\n    public void compare() {\n        Integer a = 100;\n        Integer b = 100;\n        if (a.equals(b)) {\n        }\n    }\n}\n这段代码中使用了 equals 方法比较整型包装类对象，所以不能被判定为代码规范问题。\n</例子1>"
+  },
+  {
+    "id": 36,
+    "text": "BigDecimal 的等值比较应使用 compareTo() 方法，而不是 equals() 方法。",
+    "detail": "缺陷类型：BigDecimal 的等值比较应使用 compareTo() 方法，而不是 equals() 方法；修复方案：使用 compareTo() 方法进行比较。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"BigDecimal 的等值比较应使用 compareTo() 方法，而不是 equals() 方法\"的例子\n<例子1>\nBigDecimal a = new BigDecimal(\"1.0\");\nBigDecimal b = new BigDecimal(\"1.00\");\nif (a.equals(b)) {\n    // 这段代码会返回 false，因为 equals() 方法会比较精度\n}\n</例子1>",
+    "no_example": "### 不能被判定为\"BigDecimal 的等值比较应使用 compareTo() 方法，而不是 equals() 方法\"的例子\n<例子1>\nBigDecimal a = new BigDecimal(\"1.0\");\nBigDecimal b = new BigDecimal(\"1.00\");\nif (a.compareTo(b) == 0) {\n    // 这段代码会返回 true，因为 compareTo() 方法只比较数值\n}\n</例子1>"
+  },
+  {
+    "id": 37,
+    "text": "禁止在 POJO 类中，同时存在对应属性 xxx 的 isXxx() 和 getXxx() 方法。",
+    "detail": "缺陷类型：POJO 类中存在重复的 getter 方法；修复方案：确保只存在一个 getter 方法。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"禁止在 POJO 类中，同时存在对应属性 xxx 的 isXxx() 和 getXxx() 方法\"的例子\n<例子1>\npublic class User {\n    private boolean active;\n    public boolean isActive() {\n        return active;\n    }\n    public boolean getActive() {\n        return active;\n    }\n}\n</例子1>",
+    "no_example": "### 不能被判定为\"禁止在 POJO 类中，同时存在对应属性 xxx 的 isXxx() 和 getXxx() 方法\"的例子\n<例子1>\npublic class User {\n    private int age;\n    public int getAge() {\n        return age;\n    }\n}\n</例子1>"
+  },
+  {
+    "id": 38,
+    "text": "日期格式化时，传入 pattern 中表示年份统一使用小写的 y。",
+    "detail": "缺陷类型：日期格式化错误；修复方案：使用小写的 y 表示年份。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"日期格式化时，传入 pattern 中表示年份统一使用小写的 y\"的例子\n<例子1>\nSimpleDateFormat sdf = new SimpleDateFormat(\"YYYY-MM-dd\");\n</例子1>",
+    "no_example": "### 不能被判定为\"日期格式化时，传入 pattern 中表示年份统一使用小写的 y\"的例子\n<例子1>\nSimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd\");\n</例子1>"
+  },
+  {
+    "id": 39,
+    "text": "禁止在程序任何地方中使用：1）java.sql.Date 2）java.sql.Time 3）java.sql.Timestamp。",
+    "detail": "缺陷类型：使用了 java.sql 包中的日期类；修复方案：使用 java.time 包中的日期类。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"禁止在程序任何地方中使用：1）java.sql.Date 2）java.sql.Time 3）java.sql.Timestamp\"的例子\n<例子1>\njava.sql.Date sqlDate = new java.sql.Date(System.currentTimeMillis());\n</例子1>",
+    "no_example": "### 不能被判定为\"禁止在程序任何地方中使用：1）java.sql.Date 2）java.sql.Time 3）java.sql.Timestamp\"的例子\n<例子1>\njava.time.LocalDate localDate = java.time.LocalDate.now();\n</例子1>"
+  },
+  {
+    "id": 40,
+    "text": "判断所有集合内部的元素是否为空，使用 isEmpty() 方法，而不是 size() == 0 的方式。",
+    "detail": "缺陷类型：集合判空方式错误；修复方案：使用 isEmpty() 方法。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"判断所有集合内部的元素是否为空，使用 isEmpty() 方法，而不是 size() == 0 的方式\"的例子\n<例子1>\nList<String> list = new ArrayList<>();\nif (list.size() == 0) {\n    // 判空逻辑\n}\n</例子1>",
+    "no_example": "### 不能被判定为\"判断所有集合内部的元素是否为空，使用 isEmpty() 方法，而不是 size() == 0 的方式\"的例子\n<例子1>\nList<String> list = new ArrayList<>();\nif (list.isEmpty()) {\n    // 判空逻辑\n}\n</例子1>"
+  },
+  {
+    "id": 41,
+    "text": "只要重写 equals，就必须重写 hashCode。",
+    "detail": "缺陷类型：未重写 hashCode 方法；修复方案：同时重写 equals 和 hashCode 方法。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"只要重写 equals，就必须重写 hashCode\"的例子\n<例子1>\npublic class User {\n    private String name;\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) return true;\n        if (o == null || getClass() != o.getClass()) return false;\n        User user = (User) o;\n        return Objects.equals(name, user.name);\n    }\n}\n</例子1>",
+    "no_example": "### 不能被判定为\"只要重写 equals，就必须重写 hashCode\"的例子\n<例子1>\npublic class User {\n    private String name;\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) return true;\n        if (o == null || getClass() != o.getClass()) return false;\n        User user = (User) o;\n        return Objects.equals(name, user.name);\n    }\n    @Override\n    public int hashCode() {\n        return Objects.hash(name);\n    }\n}\n</例子1>"
+  },
+  {
+    "id": 42,
+    "text": "使用 Map 的方法 keySet() / values() / entrySet() 返回集合对象时，不可以对其进行添加元素操作，否则会抛出 UnsupportedOperationException 异常。",
+    "detail": "缺陷类型：对 Map 的 keySet() / values() / entrySet() 返回的集合进行添加操作；修复方案：避免对这些集合进行添加操作。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"使用 Map 的方法 keySet() / values() / entrySet() 返回集合对象时，不可以对其进行添加元素操作，否则会抛出 UnsupportedOperationException 异常\"的例子\n<例子1>\nMap<String, String> map = new HashMap<>();\nmap.put(\"key1\", \"value1\");\nSet<String> keys = map.keySet();\nkeys.add(\"key2\");\n</例子1>",
+    "no_example": "### 不能被判定为\"使用 Map 的方法 keySet() / values() / entrySet() 返回集合对象时，不可以对其进行添加元素操作，否则会抛出 UnsupportedOperationException 异常\"的例子\n<例子1>\nMap<String, String> map = new HashMap<>();\nmap.put(\"key1\", \"value1\");\nSet<String> keys = map.keySet();\n// 不进行添加操作\n</例子1>"
+  },
+  {
+    "id": 43,
+    "text": "不要在 foreach 循环里进行元素的 remove / add 操作。remove 元素请使用 iterator 方式，如果并发操作，需要对 iterator",
+    "detail": "缺陷类型：在 foreach 循环中进行元素的 remove / add 操作；修复方案：使用 iterator 进行元素的 remove 操作。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"不要在 foreach 循环里进行元素的 remove / add 操作。remove 元素请使用 iterator 方式，如果并发操作，需要对 iterator\"的例子\n<例子1>\nList<String> list = new ArrayList<>(Arrays.asList(\"a\", \"b\", \"c\"));\nfor (String s : list) {\n    if (s.equals(\"a\")) {\n        list.remove(s);\n    }\n}\n</例子1>",
+    "no_example": "### 不能被判定为\"不要在 foreach 循环里进行元素的 remove / add 操作。remove 元素请使用 iterator 方式，如果并发操作，需要对 iterator\"的例子\n<例子1>\nList<String> list = new ArrayList<>(Arrays.asList(\"a\", \"b\", \"c\"));\nIterator<String> iterator = list.iterator();\nwhile (iterator.hasNext()) {\n    String s = iterator.next();\n    if (s.equals(\"a\")) {\n        iterator.remove();\n    }\n}\n</例子1>"
+  },
+  {
+    "id": 44,
+    "text": "类、类属性、类方法的注释必须使用 Javadoc 规范，使用 /** 内容 */ 格式，不得使用 // xxx方式。",
+    "detail": "缺陷类型：注释不符合 Javadoc 规范；修复方案：使用 Javadoc 规范的注释格式。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"类、类属性、类方法的注释必须使用 Javadoc 规范，使用 /** 内容 */ 格式，不得使用 // xxx方式\"的例子\n<例子1>\npublic class Example {\n    // 这是一个类注释\n    private String name;\n    // 这是一个属性注释\n    public String getName() {\n        return name;\n    }\n    // 这是一个方法注释\n}\n</例子1>",
+    "no_example": "### 不能被判定为\"类、类属性、类方法的注释必须使用 Javadoc 规范，使用 /** 内容 */ 格式，不得使用 // xxx方式\"的例子\n<例子1>\n/**\n * 这是一个类注释\n */\npublic class Example {\n    /**\n     * 这是一个属性注释\n     */\n    private String name;\n    /**\n     * 这是一个方法注释\n     */\n    public String getName() {\n        return name;\n    }\n}\n</例子1>"
+  },
+  {
+    "id": 45,
+    "text": "所有的抽象方法（包括接口中的方法）必须要用 Javadoc 注释",
+    "detail": "缺陷类型：所有的抽象方法（包括接口中的方法）必须要用 Javadoc 注释；修复方案：为所有的抽象方法（包括接口中的方法）添加 Javadoc 注释，除了返回值、参数异常说明外，还必须指出该方法做什么事情，实现什么功能。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"所有的抽象方法（包括接口中的方法）必须要用 Javadoc 注释\"的例子\n<例子1>\npublic interface MyInterface {\n    void doSomething();\n}\n这段代码中的接口方法 doSomething() 没有 Javadoc 注释，所以被判定为缺少 Javadoc 注释。\n</例子1>",
+    "no_example": "### 不能被判定为\"所有的抽象方法（包括接口中的方法）必须要用 Javadoc 注释\"的例子\n<例子1>\n/**\n * 执行某个操作\n * @param param 参数说明\n * @return 返回值说明\n * @throws Exception 异常说明\n */\npublic interface MyInterface {\n    void doSomething(String param) throws Exception;\n}\n这段代码中的接口方法 doSomething() 有完整的 Javadoc 注释，所以不能被判定为缺少 Javadoc 注释。\n</例子1>"
+  },
+  {
+    "id": 46,
+    "text": "方法内部单行注释和多行注释的使用规范",
+    "detail": "缺陷类型：注释使用不规范；修复方案：方法内部单行注释，在被注释语句上方另起一行，使用 // 注释。方法内部多行注释使用 /* */注释，注意与代码对齐。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"注释使用不规范\"的例子\n<例子1>\npublic void exampleMethod() {\n    int a = 1; // 初始化变量a\n    int b = 2; /* 初始化变量b */\n}\n这段代码中的单行注释和多行注释没有按照规范使用，所以被判定为注释使用不规范。\n</例子1>",
+    "no_example": "### 不能被判定为\"注释使用不规范\"的例子\n<例子1>\npublic void exampleMethod() {\n    // 初始化变量a\n    int a = 1;\n    /*\n     * 初始化变量b\n     */\n    int b = 2;\n}\n这段代码中的单行注释和多行注释按照规范使用，所以不能被判定为注释使用不规范。\n</例子1>"
+  },
+  {
+    "id": 47,
+    "text": "所有的枚举类型字段必须要有注释",
+    "detail": "缺陷类型：枚举类型字段缺少注释；修复方案：为所有的枚举类型字段添加注释，说明每个数据项的用途。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"枚举类型字段缺少注释\"的例子\n<例子1>\npublic enum Status {\n    ACTIVE,\n    INACTIVE\n}\n这段代码中的枚举类型字段没有注释，所以被判定为枚举类型字段缺少注释。\n</例子1>",
+    "no_example": "### 不能被判定为\"枚举类型字段缺少注释\"的例子\n<例子1>\npublic enum Status {\n    /**\n     * 活跃状态\n     */\n    ACTIVE,\n    /**\n     * 非活跃状态\n     */\n    INACTIVE\n}\n这段代码中的枚举类型字段有注释，所以不能被判定为枚举类型字段缺少注释。\n</例子1>"
+  },
+  {
+    "id": 48,
+    "text": "finally 块必须对资源对象、流对象进行关闭",
+    "detail": "缺陷类型：资源对象、流对象未在 finally 块中关闭；修复方案：在 finally 块中对资源对象、流对象进行关闭，有异常也要做 try-catch。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"资源对象、流对象未在 finally 块中关闭\"的例子\n<例子1>\npublic void readFile() {\n    FileInputStream fis = null;\n    try {\n        fis = new FileInputStream(\"file.txt\");\n        // 读取文件内容\n    } catch (IOException e) {\n        e.printStackTrace();\n    }\n}\n这段代码中的 FileInputStream 对象没有在 finally 块中关闭，所以被判定为资源对象、流对象未在 finally 块中关闭。\n</例子1>",
+    "no_example": "### 不能被判定为\"资源对象、流对象未在 finally 块中关闭\"的例子\n<例子1>\npublic void readFile() {\n    FileInputStream fis = null;\n    try {\n        fis = new FileInputStream(\"file.txt\");\n        // 读取文件内容\n    } catch (IOException e) {\n        e.printStackTrace();\n    } finally {\n        if (fis != null) {\n            try {\n                fis.close();\n            } catch (IOException e) {\n                e.printStackTrace();\n            }\n        }\n    }\n}\n这段代码中的 FileInputStream 对象在 finally 块中关闭，所以不能被判定为资源对象、流对象未在 finally 块中关闭。\n</例子1>"
+  },
+  {
+    "id": 49,
+    "text": "常量命名应该全部大写，单词间用下划线隔开",
+    "detail": "缺陷类型：常量命名不规范；修复方案：常量命名应该全部大写，单词间用下划线隔开，力求语义表达完整清楚，不要嫌名字长。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"常量命名应该全部大写，单词间用下划线隔开\"的例子\n<例子1>\npublic static final int maxCount = 100;\n</例子1>",
+    "no_example": "### 不能被判定为\"常量命名应该全部大写，单词间用下划线隔开\"的例子\n<例子1>\npublic static final int MAX_COUNT = 100;\n</例子1>"
+  },
+  {
+    "id": 50,
+    "text": "任何二目、三目运算符的左右两边都需要加一个空格",
+    "detail": "缺陷类型：运算符两边缺少空格；修复方案：任何二目、三目运算符的左右两边都需要加一个空格。",
+    "language": "Java",
+    "yes_example": "### 被判定为\"任何二目、三目运算符的左右两边都需要加一个空格\"的例子\n<例子1>\nint a=b+c;\n</例子1>",
+    "no_example": "### 不能被判定为\"任何二目、三目运算符的左右两边都需要加一个空格\"的例子\n<例子1>\nint a = b + c;\n</例子1>"
+  },
+  {
+    "id": 51,
+    "text": "避免使用from <module> import *",
+    "detail": "缺陷类型：避免使用from <module> import *，导入所有内容会造成命名冲突；修复方案：每个使用到的子依赖需分别导入。",
+    "language": "Python",
+    "yes_example": "### 被判定为\"避免使用from <module> import *\"的例子\n<例子1>from math import * \n</例子1>",
+    "no_example": "### 不能被判定为\"避免使用from <module> import *\"的例子\n<例子1>from math import sqrt, pi \n</例子1>"
+  },
+  {
+    "id": 52,
+    "text": "避免使用__import__()函数动态导入模块",
+    "detail": "缺陷类型：避免使用__import__()函数动态导入模块；修复方案：使用标准的import语句。",
+    "language": "Python",
+    "yes_example": "### 被判定为\"使用__import__()函数动态导入模块\"的例子\n<例子1>module = __import__('math') \n</例子1>",
+    "no_example": "### 不能被判定为\"使用__import__()函数动态导入模块\"的例子\n<例子1>import math \n</例子1>"
+  },
+  {
+    "id": 53,
+    "text": "导入语句未按标准库导入、相关第三方导入、本地应用/库特定导入的顺序分组",
+    "detail": "缺陷类型：导入语句未按标准库导入、相关第三方导入、本地应用/库特定导入的顺序分组；修复方案：按顺序分组导入语句。",
+    "language": "Python",
+    "yes_example": "### 被判定为'导入语句未按标准库导入、相关第三方导入、本地应用/库特定导入的顺序分组'的例子\n<例子1>\nimport numpy as np\nimport os\nimport sys\nfrom my_local_module import my_function\n在这个样例中，先导入了第三方库，然后导入了标准库。\n</例子1>\n<例子2>\nfrom my_project import my_local_function\nimport datetime\nimport requests\n在这个样例中，先导入了本地模块，然后导入了标准库。\n</例子2>\n<例子3>\nimport os\nfrom my_project.local_module import some_function\nimport pandas as pd\nimport sys\nfrom another_local_module import another_function\nimport math\n在这个样例中，导入语句完全混乱，没有遵循任何顺序。\n</例子3>\n<例子4>\nimport os\nimport requests\nimport sys\nimport numpy as np\nfrom local_package import local_module\n在这个样例中，导入标准库和第三方库交替进行。\n</例子4>",
+    "no_example": "### 不能被判定为'导入语句未按标准库导入、相关第三方导入、本地应用/库特定导入的顺序分组'的例子\n<例子1>import os \n\n import requests \n\n import mymodule \n</例子1>"
+  },
+  {
+    "id": 54,
+    "text": "避免未使用的函数形参",
+    "detail": "缺陷类型：避免未使用的函数形参；修复方案：移除未使用的函数形参。",
+    "language": "Python",
+    "yes_example": "### 被判定为'避免未使用的函数形参'的例子\n<例子1>def func(a, b): \n    return a</例子1>\n<例子2>def start_game(unused_param): \npuzzle = Puzzle() \npuzzle.solve()</例子2>\n<例子3>def make_move(self, board):\npass \n</例子3>\n<例子4>def move(self, direction):\npass \n</例子4>",
+    "no_example": "### 不能被判定为'避免未使用的函数形参'的例子\n<例子1>def func(a): \n    return a</例子1>"
+  },
+  {
+    "id": 55,
+    "text": "使用is not None来检查一个变量是否不是None",
+    "detail": "缺陷类型：未使用is not None来检查一个变量是否不是None；修复方案：使用is not None来检查。",
+    "language": "Python",
+    "yes_example": "### 被判定为'未使用is not None来检查一个变量是否不是None'的例子\n<例子1>if variable != None:\n    pass</例子1>",
+    "no_example": "### 不能被判定为'未使用is not None来检查一个变量是否不是None'的例子\n<例子1>if variable is not None:\n    pass</例子1>"
+  },
+  {
+    "id": 56,
+    "text": "避免使用==或!=来比较对象实例的等价性",
+    "detail": "缺陷类型：使用==或!=来比较对象实例的等价性；修复方案：应使用equals比较。",
+    "language": "Python",
+    "yes_example": "### 被判定为'使用==或!=来比较对象实例的等价性'的例子\n<例子1>obj1 = MyClass() \n obj2 = MyClass() if obj1 == obj2: \n    pass\n</例子1>",
+    "no_example": "### 不能被判定为'使用==或!=来比较对象实例的等价性'的例子\n<例子1>obj1 = MyClass() \n obj2 = MyClass() if obj1.equals(obj2): \n    pass\n</例子1>\n<例子2>obj1 = 21 \n obj2 = 22 \n if obj1.equals(obj2):\n    pass</例子2>"
+  },
+  {
+    "id": 57,
+    "text": "避免使用单字母变量名，使用描述性变量名",
+    "detail": "缺陷类型：避免使用单字母变量名，使用描述性变量名；修复方案：使用描述性变量名。",
+    "language": "Python",
+    "yes_example": "### 被判定为'避免使用单字母变量名，使用描述性变量名'的例子\n<例子1>x = 10 \n</例子1>\n<例子2>y = 10 \n</例子2>",
+    "no_example": "### 不能被判定为'避免使用单字母变量名，使用描述性变量名'的例子\n<例子1>count = 10 \n</例子1>"
+  },
+  {
+    "id": 58,
+    "text": "常量命名使用全大写字母，并用下划线分隔",
+    "detail": "缺陷类型：常量命名未使用全大写字母或未用下划线分隔；修复方案：常量命名使用全大写字母，并用下划线分隔。",
+    "language": "Python",
+    "yes_example": "### 被判定为'常量命名未使用全大写字母，并用下划线分隔'的例子\n<例子1>pi = 3.14159</例子1>",
+    "no_example": "### 不能被判定为'常量命名未使用全大写字母，并用下划线分隔'的例子\n<例子1>PI = 3.14159</例子1>\n<例子2>max_size = 1 \n max_size += 1</例子2>"
+  },
+  {
+    "id": 59,
+    "text": "类名应使用驼峰式命名（CamelCase）",
+    "detail": "缺陷类型：类名未使用驼峰式命名；修复方案：类名使用驼峰式命名。",
+    "language": "Python",
+    "yes_example": "### 被判定为'类名未使用驼峰式命名（CamelCase）'的例子\n<例子1>class my_class: \n    pass</例子1>\n<例子2>class my_class:  \n def solve(self):\n pass</例子2>",
+    "no_example": "### 不能被判定为'类名未使用驼峰式命名（CamelCase）'的例子\n<例子1>class MyClass: \n    pass</例子1>"
+  },
+  {
+    "id": 60,
+    "text": "尽量使用with语句来管理资源",
+    "detail": "缺陷类型：未使用with语句来管理资源；修复方案：使用with语句来管理资源。",
+    "language": "Python",
+    "yes_example": "### 被判定为'未使用with语句来管理资源'的例子\n<例子1>file = open('file.txt', 'r') \n content = file.read() \n file.close()</例子1>",
+    "no_example": "### 不能被判定为'未使用with语句来管理资源'的例子\n<例子1>with open('file.txt', 'r') as file: \n    content = file.read()</例子1>"
+  },
+  {
+    "id": 61,
+    "text": "避免使用except 或 通用的Exception来捕获所有异常，应该指定异常类型",
+    "detail": "缺陷类型：捕获所有异常；修复方案：指定具体的异常类型。",
+    "language": "Python",
+    "yes_example": "### 被判定为'使用except:来捕获所有异常'的例子\n<例子1>try: \n    # some code \n except: \n    handle_error()</例子1>\n### 被判定为'抛出通用的Exception异常'的例子\n<例子2>\n try:\n    process_data(data) \n except: \n    raise Exception('An error occurred') \n </例子2>",
+    "no_example": "### 不能被判定为'使用except:来捕获所有异常'的例子\n<例子1>try: \n    # some code \n except ValueError: \n    handle_value_error()</例子1>"
+  },
+  {
+    "id": 62,
+    "text": "尽量避免手动拼接字符串",
+    "detail": "缺陷类型：手动拼接字符串；修复方案：使用格式化字符串或join方法。",
+    "language": "Python",
+    "yes_example": "### 被判定为'手动拼接字符串'的例子\n<例子1>\n name = 'John' \n greeting = 'Hello, ' + name + '!' \n</例子1> \n <例子2>greeting = '2048' + 'game' \n</例子2> \n <例子3>pygame.display.set_caption('贪吃蛇' + '游戏')</例子3>",
+    "no_example": "### 不能被判定为'手动拼接字符串'的例子\n<例子1>\n name = 'John' \n greeting = f'Hello, {name}!' \n</例子1>"
+  },
+  {
+    "id": 63,
+    "text": "避免出现魔法字符和数字，应声明为常量",
+    "detail": "缺陷类型：使用魔法字符和数字；修复方案：将其声明为常量。",
+    "language": "Python",
+    "yes_example": "### 被判定为'出现魔法字符和数字'的例子\n<例子1>\n if status == 1: \n    print('Active')' \n</例子1>\n<例子2>\n self.board = [[0] * 4 for _ in range(4)] \n self.score = 0</例子2>\n<例子3>\ndef __init__(self, width=10, height=10, mines=15):\n</例子3>\n<例子4>\nx, y = event.x // 20, event.y // 20\n</例子4>\n<例子5>\nraise ValueError(\"余额不足\")\n</例子5>\n<例子6>\ntransfer(bank, \"123\", \"456\", 200)\n</例子6>\n<例子7>\nbank.add_account(Account(\"123\", 1000))\n</例子7>",
+    "no_example": "### 不能被判定为'出现魔法字符和数字'的例子\n<例子1>\n ACTIVE_STATUS = 1 \n if status == ACTIVE_STATUS:\n    print(ACTIVE_STATUS)' \n</例子1>"
+  },
+  {
+    "id": 64,
+    "text": "boolean变量判断无需显式比较",
+    "detail": "缺陷类型：显式比较boolean变量；修复方案：直接使用boolean变量进行判断。",
+    "language": "Python",
+    "yes_example": "### 被判定为'显式比较boolean变量'的例子\n<例子1>flag = True \n if flag == True: \n    print('Flag is true')</例子1>\n<例子2>if self.game.is_game_over() == True: \n return</例子2><例子3>if self.canvas.drawings ==True:</例子3>",
+    "no_example": "### 不能被判定为'显式比较boolean变量'的例子\n<例子1>flag = True \n if flag: \n    print('Flag is true') \n</例子1>"
+  },
+  {
+    "id": 65,
+    "text": "避免使用type()检查对象类型",
+    "detail": "缺陷类型：避免使用type()检查对象类型；修复方案：使用isinstance()函数。",
+    "language": "Python",
+    "yes_example": "### 被判定为'避免使用type()检查对象类型'的例子\n<例子1>\n if type(obj) == list: \n    print('obj is a list')</例子1>",
+    "no_example": "### 不能被判定为'避免使用type()检查对象类型'的例子\n<例子1>\n if isinstance(obj, list): \n    print('obj is a list') \n</例子1>"
+  },
+  {
+    "id": 66,
+    "text": "避免使用os.system()来调用外部命令",
+    "detail": "缺陷类型：使用os.system()调用外部命令；修复方案：使用subprocess模块。",
+    "language": "Python",
+    "yes_example": "### 被判定为'使用os.system()来调用外部命令'的例子\n<例子1>os.system('ls -l')</例子1>\n<例子2>os.system('ls -l')</例子2>",
+    "no_example": "### 不能被判定为'使用os.system()来调用外部命令'的例子\n<例子1>import subprocess \n subprocess.run(['ls', '-l'])</例子1>"
+  },
+  {
+    "id": 67,
+    "text": "只使用@property装饰器创建只读属性，而非修改属性",
+    "detail": "缺陷类型：使用@property装饰器创建可修改属性；修复方案：只使用@property装饰器创建只读属性。",
+    "language": "Python",
+    "yes_example": "### 被判定为'使用@property装饰器来创建可修改属性'的例子\n<例子1>@property \n    def value(self, new_value): \n        self._value = new_value</例子1>\n<例子2>@property \n    def game_over(self): \n        return self._is_game_over() \n def _is_game_over(self): \n pass</例子2>",
+    "no_example": "### 不能被判定为'使用@property装饰器来创建可修改属性'的例子\n<例子1>@property \n    def value(self): \n        return self._value</例子1>\n<例子2>@property \n    def __str__(self): \n        return 'Maze Game State'</例子2>"
+  },
+  {
+    "id": 68,
+    "text": "在使用索引或切片时，不要在方括号或冒号内加空格",
+    "detail": "缺陷类型：在索引或切片的方括号或冒号内加空格；修复方案：去掉方括号或冒号内的空格。",
+    "language": "Python",
+    "yes_example": "### 被判定为'在使用索引或切片时，在方括号或冒号内加空格'的例子\n<例子1>list = [1, 2, 3, 4] \n sublist = list[ 1 : 3 ]</例子1>\n<例子2>start_point = self.canvas.drawings[  -1] </例子2>\n<例子3>if head[ 0] < 0 or head[ 0] >= GRID_WIDTH or head[ 1] < 0 or head[ 1] >= GRID_HEIGHT:</例子3>\n<例子4>for segment in self.snake[ 1:]:</例子4>",
+    "no_example": "### 不能被判定为'在使用索引或切片时，在方括号或冒号内加空格'的例子\n<例子1>list = [1, 2, 3, 4] \n sublist = list[1:3]</例子1>"
+  },
+  {
+    "id": 69,
+    "text": "在逗号、分号或冒号前不要加空格，但在它们之后要加空格",
+    "detail": "缺陷类型：在逗号、分号或冒号前加空格或在它们之后不加空格；修复方案：在逗号、分号或冒号前不要加空格，但在它们之后要加空格。",
+    "language": "Python",
+    "yes_example": "### 被判定为'在逗号、分号或冒号前加空格，或没在它们之后加空格'的例子\n<例子1>if x == 4 : \n print(x , y)</例子1>\n<例子2>if event.keysym == 'Up' or event.keysym == 'Down' or event.keysym == 'Left' or event.keysym == 'Right' :</例子2>\n<例子3>x ,y = 1 ,2</例子3>\n<例子4>def on_key_press(self , event)  :</例子4>\n<例子5>elif event.keysym == 'Down' ; </例子5>\n<例子6>def update_status(self ,message: str) : \n pass </例子6>",
+    "no_example": "### 不能被判定为'在逗号、分号或冒号前加空格，或没在它们之后加空格'的例子\n<例子1>if x == 4: \n print(x, y)</例子1>"
+  },
+  {
+    "id": 70,
+    "text": "对于二元操作符，两边都应有空格",
+    "detail": "缺陷类型：二元操作符两边没有空格；修复方案：在二元操作符两边加空格",
+    "language": "Python",
+    "yes_example": "### 被判定为'二元操作符两边没有空格'的例子\n<例子1>a=b+1</例子1>",
+    "no_example": "### 不能被判定为'二元操作符两边没有空格'的例子\n<例子1>a = b + 1</例子1>\n<例子2>label = tk.Label(self.root, text=str(cell), bg='white')</例子2>\n<例子3>label.grid(row=i, column=j)</例子3>"
+  },
+  {
+    "id": 71,
+    "text": "避免使用Python关键字作为变量名或函数名",
+    "detail": "缺陷类型：使用Python关键字作为变量名或函数名；修复方案：使用非关键字的名称。",
+    "language": "Python",
+    "yes_example": "### 被判定为'使用Python关键字作为变量名或函数名'的例子\n<例子1>def class(): \n    pass</例子1>\n<例子2>for = 5</例子2>\n<例子3>def if(self): </例子3>",
+    "no_example": "### 不能被判定为'使用Python关键字作为变量名或函数名'的例子\n<例子1>def my_function(): \n    pass</例子1>\n<例子2>number = 5</例子2>"
+  },
+  {
+    "id": 72,
+    "text": "避免使用特殊字符作为变量名/方法名/类名，例如$或@",
+    "detail": "缺陷类型：使用特殊字符作为变量名/方法名/类名；修复方案：使用合法的变量名。",
+    "language": "Python",
+    "yes_example": "### 被判定为'使用特殊字符作为变量名/方法名/类名，例如$或@'的例子\n<例子1>my$var = 10</例子1>\n<例子2>@var = 20</例子2>\n<例子3>def add_score@(self, points): \n self.score += points</例子3>\n<例子4>class @MyClass: \n pass</例子4>\n<例子5>def mine@(self):</例子5>",
+    "no_example": "### 不能被判定为'使用特殊字符作为变量名/方法名/类名，例如$或@'的例子\n<例子1>my_var = 10</例子1>\n<例子2>var_20 = 20</例子2>"
+  },
+  {
+    "id": 73,
+    "text": "避免使用raise来重新抛出当前的异常，这会丢失原始的栈跟踪",
+    "detail": "缺陷类型：使用raise重新抛出当前异常；修复方案：使用raise ... from ...语法。",
+    "language": "Python",
+    "yes_example": "### 被判定为'避免使用raise来重新抛出当前的异常，这会丢失原始的栈跟踪'的例子\n<例子1>\n try: \n    1 / 0 \n except ZeroDivisionError: \n    raise SomeException('新的异常信息')\n</例子1>\n<例子2>\ntry:\n    db.get_data()\nexcept ValueError as e:\n    raise ValueError(\"Something went wrong!\")\n</例子2>\n<例子3>\ntry:\n\traise Exception(\"形状添加失败\")\nexcept Exception as e:\n\tpass\n</例子3>",
+    "no_example": "### 不能被判定为'避免使用raise来重新抛出当前的异常，这会丢失原始的栈跟踪'的例子\n<例子1>\n try: \n    1 / 0 \n except ZeroDivisionError as e: \n    raise RuntimeError('Error occurred') from e \n</例子1>\n<例子2>\n try: \n    1 / 0 \n except ZeroDivisionError as e: \n\tlogger.error(e)\n    raise \n</例子2>"
+  },
+  {
+    "id": 74,
+    "text": "避免在except块中使用pass，这会捕获并忽略异常",
+    "detail": "缺陷类型：在except块中使用pass；修复方案：处理异常或记录日志。",
+    "language": "Python",
+    "yes_example": "### 被判定为'在except块中使用pass'的例子\n<例子1>\n try: \n    1 / 0 \n except ZeroDivisionError: \n    pass \n </例子1>\n<例子2>\n try: \n    1 / 0 \n except ZeroDivisionError: \n    pass \n</例子2>",
+    "no_example": "### 不能被判定为'在except块中使用pass'的例子\n<例子1>\n try: \n    1 / 0 \n except ZeroDivisionError as e: \n    logging.error('Error occurred: %s', e) \n</例子1>"
+  },
+  {
+    "id": 75,
+    "text": "避免使用assert语句来执行重要的运行时检查",
+    "detail": "缺陷类型：使用assert语句执行重要的运行时检查；修复方案：使用显式的条件检查和异常处理。",
+    "language": "Python",
+    "yes_example": "### 被判定为'使用assert语句来执行重要的运行时检查'的例子\n<例子1>\n def divide(a, b): \n    assert b != 0 \n    return a / b \n</例子1>",
+    "no_example": "### 不能被判定为'使用assert语句来执行重要的运行时检查'的例子\n<例子1>\n def divide(a, b): \n    if b == 0: \n        raise ValueError('b cannot be zero') \n    return a / b \n</例子1>"
+  },
+  {
+    "id": 76,
+    "text": "避免使用eval()和exec()，这些函数可能会带来安全风险",
+    "detail": "缺陷类型：使用eval()和exec()函数；修复方案：使用安全的替代方案。",
+    "language": "Python",
+    "yes_example": "### 被判定为'使用eval()和exec()'的例子\n<例子1>\n eval('print(1)') \n</例子1>\n<例子2> \n exec('a = 1') \n</例子2>",
+    "no_example": "### 不能被判定为'使用eval()和exec()'的例子\n<例子1>\n compiled_code = compile('print(1)', '<string>', 'exec') \n exec(compiled_code) \n</例子1>"
+  },
+  {
+    "id": 77,
+    "text": "避免使用sys.exit()，应使用异常来控制程序的退出",
+    "detail": "缺陷类型：避免使用sys.exit()，应使用异常来控制程序的退出；修复方案：使用异常来控制程序的退出。",
+    "language": "Python",
+    "yes_example": "### 被判定为'避免使用sys.exit()，应使用异常来控制程序的退出'的例子\n<例子1>\n import sys\nsys.exit(1)\n</例子1>\n<例子2>\n import sys \n sys.exit()\n</例子2>\n<例子3>\nif event.type == pygame.QUIT:\n\tpygame.quit()\n\texit()\n</例子3>\n<例子4>\n import sys \n sys.exit('退出程序'))\n</例子4>",
+    "no_example": "### 不能被判定为'避免使用sys.exit()，应使用异常来控制程序的退出'的例子\n<例子1>\n raise SystemExit(1)\n</例子1>"
+  },
+  {
+    "id": 78,
+    "text": "避免使用time.sleep()进行线程同步，应使用同步原语，如锁或事件",
+    "detail": "缺陷类型：使用time.sleep()进行线程同步；修复方案：使用同步原语。",
+    "language": "Python",
+    "yes_example": "### 被判定为'使用time.sleep()进行线程同步'的例子\n<例子1>\n import time \n\n def worker(): \n    time.sleep(1) \n</例子1>\n<例子2>\n import time \n\n time.sleep(1) \n</例子2>",
+    "no_example": "### 不能被判定为'使用time.sleep()进行线程同步'的例子\n<例子1>\n import threading \n\n event = threading.Event() \n\n def worker(): \n    event.wait()\n</例子1>"
+  },
+  {
+    "id": 79,
+    "text": "每行代码避免超过79个字符",
+    "detail": "缺陷类型：每行代码避免超过79个字符；修复方案：将长行代码格式化为多行。",
+    "language": "Python",
+    "yes_example": "### 被判定为'每行代码避免超过79个字符'的例子\n<例子1>\n print('This is a very long line of code that exceeds the 79 characters limit........') \n</例子1>",
+    "no_example": "### 不能被判定为'每行代码避免超过79个字符'的例子\n<例子1>\n print('This is a very long line of code that exceeds the 79 characters limit' + \n      ' but it is split into two lines')\n</例子1>"
+  },
+  {
+    "id": 80,
+    "text": "模块级别的函数和类定义之间用两个空行分隔，类内部的方法定义之间用一个空行分隔",
+    "detail": "缺陷类型：模块级别的函数和类定义之间没有用两个空行分隔，类内部的方法定义之间没有用一个空行分隔；修复方案：按照规范添加空行。",
+    "language": "Python",
+    "yes_example": "### 被判定为'模块级别的函数和类定义之间没用两个空行分隔，类内部的方法定义之间没用一个空行分隔'的例子\n<例子1>\n def func1(): \n    pass \n def func2(): \n    pass \n</例子1>\n<例子2>\n class MyClass: \n    def method1(self): \n        pass \n    def method2(self): \n        pass \n</例子2>",
+    "no_example": "### 不能被判定为'模块级别的函数和类定义之间没用两个空行分隔，类内部的方法定义之间没用一个空行分隔'的例子\n<例子1>\n def func1(): \n    pass \n\n\n def func2(): \n    pass \n</例子1>\n<例子2>\n class MyClass: \n    def method1(self): \n        pass \n\n    def method2(self): \n        pass \n</例子2>"
+  },
+  {
+    "id": 81,
+    "text": "使用小写字母和下划线分隔的方式命名变量和函数名",
+    "detail": "缺陷类型：变量和函数命名不符合小写字母和下划线分隔的方式；修复方案：使用小写字母和下划线分隔的方式命名。",
+    "language": "Python",
+    "yes_example": "### 被判定为'未使用小写字母和下划线分隔的方式命名变量和函数'的例子\n<例子1>\n def myFunction(): \n    pass \n</例子1>\n<例子2>\n myVariable = 10 \n</例子2>\n<例子3>\n def Calculatesquareroot(self, x): \n    return 1 \n</例子3>",
+    "no_example": "### 不能被判定为'未使用小写字母和下划线分隔的方式命名变量和函数'的例子\n<例子1>\n def my_function(): \n    pass \n</例子1>\n<例子2>\n my_variable = 10 \n</例子2>"
+  },
+  {
+    "id": 82,
+    "text": "不允许使用print()函数来记录日志，使用logging模块等来记录日志",
+    "detail": "缺陷类型：使用print()函数记录日志；修复方案：使用logging模块记录日志。",
+    "language": "Python",
+    "yes_example": "### 被判定为'使用print()函数来记录日志'的例子\n<例子1>\n print('Error occurred') \n</例子1>\n<例子2>\n print('打印的日志字符串内容') \n</例子2>\n<例子3>\n task = 'xxx' \n print(task) \n</例子3>\n<例子4>\n print(1)\n</例子4>",
+    "no_example": "### 不能被判定为'使用print()函数来记录日志'的例子\n<例子1>\n import logging \n logging.error('Error occurred') \n</例子1>"
+  }
+]

metagpt/ext/cr/utils/cleaner.py

@@ -0,0 +1,68 @@
+"""Cleaner."""
+
+from unidiff import Hunk, PatchedFile, PatchSet
+
+from metagpt.logs import logger
+
+
+def rm_patch_useless_part(patch: PatchSet, used_suffix: list[str] = ["java", "py"]) -> PatchSet:
+    new_patch = PatchSet("")
+    useless_files = []
+    for pfile in patch:
+        suffix = str(pfile.target_file).split(".")[-1]
+        if suffix not in used_suffix or pfile.is_removed_file:
+            useless_files.append(pfile.path)
+            continue
+        new_patch.append(pfile)
+    logger.info(f"total file num: {len(patch)}, used file num: {len(new_patch)}, useless_files: {useless_files}")
+    return new_patch
+
+
+def add_line_num_on_patch(patch: PatchSet, start_line_num: int = 1) -> PatchSet:
+    new_patch = PatchSet("")
+    lineno = start_line_num
+    for pfile in patch:
+        new_pfile = PatchedFile(
+            source=pfile.source_file,
+            target=pfile.target_file,
+            source_timestamp=pfile.source_timestamp,
+            target_timestamp=pfile.target_timestamp,
+        )
+        for hunk in pfile:
+            arr = [str(line) for line in hunk]
+            new_hunk = Hunk(
+                src_start=hunk.source_start,
+                src_len=hunk.source_length,
+                tgt_start=hunk.target_start,
+                tgt_len=hunk.target_length,
+                section_header=hunk.section_header,
+            )
+
+            for line in arr:
+                # if len(line) > 0 and line[0] in ["+", "-"]:
+                #     line = f"{lineno} {line}"
+                #     lineno += 1
+                line = f"{lineno} {line}"
+                lineno += 1
+                new_hunk.append(line)
+            new_pfile.append(new_hunk)
+        new_patch.append(new_pfile)
+    return new_patch
+
+
+def get_code_block_from_patch(patch: PatchSet, code_start_line: str, code_end_line: str) -> str:
+    line_arr = str(patch).split("\n")
+    code_arr = []
+    add_line_tag = False
+    for line in line_arr:
+        if line.startswith(f"{code_start_line} "):
+            add_line_tag = True
+
+        if add_line_tag:
+            new_line = " ".join(line.split(" ")[1:])  # rm line-no tag
+            code_arr.append(new_line)
+
+        if line.startswith(f"{code_end_line} "):
+            add_line_tag = False
+
+    return "\n".join(code_arr)

metagpt/ext/cr/utils/schema.py

@@ -0,0 +1,20 @@
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class Point(BaseModel):
+    id: int = Field(default=0, description="ID of the point.")
+    text: str = Field(default="", description="Content of the point.")
+    language: Literal["Python", "Java"] = Field(
+        default="Python", description="The programming language that the point corresponds to."
+    )
+    file_path: str = Field(default="", description="The file that the points come from.")
+    start_line: int = Field(default=0, description="The starting line number that the point refers to.")
+    end_line: int = Field(default=0, description="The ending line number that the point refers to.")
+    detail: str = Field(default="", description="File content from start_line to end_line.")
+    yes_example: str = Field(default="", description="yes of point examples")
+    no_example: str = Field(default="", description="no of point examples")
+
+    def rag_key(self) -> str:
+        return self.text

metagpt/ext/stanford_town/actions/st_action.py

@@ -8,7 +8,6 @@ from pathlib import Path
 from typing import Any, Optional, Union
 
 from metagpt.actions.action import Action
-from metagpt.config2 import config
 from metagpt.ext.stanford_town.utils.const import PROMPTS_DIR
 from metagpt.logs import logger
 
@@ -62,13 +61,13 @@ class STAction(Action):
     async def _run_gpt35_max_tokens(self, prompt: str, max_tokens: int = 50, retry: int = 3):
         for idx in range(retry):
             try:
-                tmp_max_tokens_rsp = getattr(config.llm, "max_token", 1500)
-                setattr(config.llm, "max_token", max_tokens)
+                tmp_max_tokens_rsp = getattr(self.config.llm, "max_token", 1500)
+                setattr(self.config.llm, "max_token", max_tokens)
                 self.llm.use_system_prompt = False  # to make it behave like a non-chat completions
 
                 llm_resp = await self._aask(prompt)
 
-                setattr(config.llm, "max_token", tmp_max_tokens_rsp)
+                setattr(self.config.llm, "max_token", tmp_max_tokens_rsp)
                 logger.info(f"Action: {self.cls_name} llm _run_gpt35_max_tokens raw resp: {llm_resp}")
                 if self._func_validate(llm_resp, prompt):
                     return self._func_cleanup(llm_resp, prompt)