diff --git a/examples/cr.py b/examples/cr.py new file mode 100644 index 000000000..7171fc383 --- /dev/null +++ b/examples/cr.py @@ -0,0 +1,15 @@ +import fire + +from metagpt.roles.di.engineer2 import Engineer2 +from metagpt.tools.libs.cr import CodeReview + + +async def main(msg): + role = Engineer2(tools=["Plan", "Editor:write,read", "RoleZero", "ReviewAndRewriteCode", "CodeReview"]) + cr = CodeReview() + role.tool_execution_map.update({"CodeReview.review": cr.review, "CodeReview.fix": cr.fix}) + await role.run(msg) + + +if __name__ == "__main__": + fire.Fire(main) diff --git a/examples/mgx_write_project_framework.py b/examples/mgx_write_project_framework.py new file mode 100644 index 000000000..b43d97b85 --- /dev/null +++ b/examples/mgx_write_project_framework.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : write_project_framework.py +@Desc : The implementation of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" +import asyncio +import json +import uuid +from json import JSONDecodeError +from pathlib import Path +from typing import Dict, List + +import typer +from pydantic import BaseModel + +from metagpt.config2 import Config +from metagpt.const import DEFAULT_WORKSPACE_ROOT +from metagpt.context import Context +from metagpt.environment import Environment +from metagpt.environment.mgx.mgx_env import MGXEnv +from metagpt.logs import logger +from metagpt.roles import Architect +from metagpt.roles.di.team_leader import TeamLeader +from metagpt.schema import AIMessage, UserMessage +from metagpt.strategy.experience_retriever import TRDToolExpRetriever +from metagpt.utils.common import aread + +app = typer.Typer(add_completion=False) + + +class EnvBuilder(BaseModel): + context: Context + user_requirements: List[str] + actors: Dict[str, str] + technical_constraint: str + output_dir: Path + + def build(self) -> Environment: + env = MGXEnv(context=self.context) + team_leader = TeamLeader() + architect = Architect(experience_retriever=TRDToolExpRetriever()) + + # Prepare context + use_case_actors = "".join([f"- {v}: {k}\n" for k, v in self.actors.items()]) + msg = """ +The content of "Actor, System, External System" provides an explanation of actors and systems that appear in UML Use Case diagram. +## Actor, System, External System +{use_case_actors} + """ + architect.rc.memory.add(AIMessage(content=msg.format(use_case_actors=use_case_actors))) + + # Prepare technical requirements + msg = """ +"Additional Technical Requirements" specifies the additional technical requirements that the generated software framework code must meet. +## Additional Technical Requirements +{technical_requirements} +""" + architect.rc.memory.add(AIMessage(content=msg.format(technical_requirements=self.technical_constraint))) + + env.add_roles([team_leader, architect]) + return env + + +async def develop( + context: Context, + user_requirement_filename: str, + actors_filename: str, + constraint_filename: str, + output_dir: str, +): + output_dir = Path(output_dir) if output_dir else DEFAULT_WORKSPACE_ROOT / uuid.uuid4().hex + + v = await aread(filename=user_requirement_filename) + try: + user_requirements = json.loads(v) + except JSONDecodeError: + user_requirements = [v] + v = await aread(filename=actors_filename) + actors = json.loads(v) + technical_constraint = await aread(filename=constraint_filename) + env_builder = EnvBuilder( + context=context, + user_requirements=user_requirements, + actors=actors, + technical_constraint=technical_constraint, + output_dir=output_dir, + ) + env = env_builder.build() + msg = """ +Given the user requirement of "User Requirements", write out the software framework. +## User Requirements +{user_requirements} + """ + env.publish_message( + UserMessage(content=msg.format(user_requirements="\n".join(user_requirements)), send_to="Bob"), + user_defined_recipient="Bob", + ) + + while not env.is_idle: + await env.run() + + +@app.command() +def startup( + user_requirement_filename: str = typer.Argument(..., help="The filename of the user requirements."), + actors_filename: str = typer.Argument(..., help="The filename of UML use case actors description."), + llm_config: str = typer.Option(default="", help="Low-cost LLM config"), + constraint_filename: str = typer.Option(default="", help="What technical dependency constraints are."), + output_dir: str = typer.Option(default="", help="Output directory."), +): + if llm_config and Path(llm_config).exists(): + config = Config.from_yaml_file(Path(llm_config)) + else: + logger.info("GPT 4 turbo is recommended") + config = Config.default() + ctx = Context(config=config) + + asyncio.run(develop(ctx, user_requirement_filename, actors_filename, constraint_filename, output_dir)) + + +if __name__ == "__main__": + app() diff --git a/examples/write_project_framework.py b/examples/write_project_framework.py new file mode 100644 index 000000000..8d23695a7 --- /dev/null +++ b/examples/write_project_framework.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : write_project_framework.py +@Desc : The implementation of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" +import asyncio +import json +import uuid +from pathlib import Path +from typing import Dict, List + +import typer + +from metagpt.actions.requirement_analysis.framework import ( + EvaluateFramework, + WriteFramework, + save_framework, +) +from metagpt.actions.requirement_analysis.trd import ( + CompressExternalInterfaces, + DetectInteraction, + EvaluateTRD, + WriteTRD, +) +from metagpt.config2 import Config +from metagpt.const import DEFAULT_WORKSPACE_ROOT +from metagpt.context import Context +from metagpt.logs import logger +from metagpt.utils.common import aread + +app = typer.Typer(add_completion=False) + + +async def _write_trd( + context: Context, actors: Dict[str, str], user_requirements: List[str], available_external_interfaces: str +) -> (str, str): + detect_interaction = DetectInteraction(context=context) + write_trd = WriteTRD(context=context) + evaluate_trd = EvaluateTRD(context=context) + use_case_actors = "".join([f"- {v}: {k}\n" for k, v in actors.items()]) + legacy_user_requirements = [] + legacy_user_requirements_interaction_events = [] + legacy_user_requirements_trd = "" + for ix, r in enumerate(user_requirements): + is_pass = False + evaluation_conclusion = "" + interaction_events = "" + trd = "" + while not is_pass and (context.cost_manager.total_cost < context.cost_manager.max_budget): + interaction_events = await detect_interaction.run( + user_requirements=r, + use_case_actors=use_case_actors, + legacy_interaction_events=interaction_events, + evaluation_conclusion=evaluation_conclusion, + ) + if ix == 0: + trd = await write_trd.run( + user_requirements=r, + use_case_actors=use_case_actors, + available_external_interfaces=available_external_interfaces, + evaluation_conclusion=evaluation_conclusion, + interaction_events=interaction_events, + previous_version_trd=trd, + ) + else: + trd = await write_trd.run( + user_requirements=r, + use_case_actors=use_case_actors, + available_external_interfaces=available_external_interfaces, + evaluation_conclusion=evaluation_conclusion, + interaction_events=interaction_events, + previous_version_trd=trd, + legacy_user_requirements="\n".join(legacy_user_requirements), + legacy_user_requirements_trd=legacy_user_requirements_trd, + legacy_user_requirements_interaction_events="\n".join(legacy_user_requirements_interaction_events), + ) + evaluation = await evaluate_trd.run( + user_requirements=r, + use_case_actors=use_case_actors, + trd=trd, + interaction_events=interaction_events, + legacy_user_requirements_interaction_events="\n".join(legacy_user_requirements_interaction_events), + ) + is_pass = evaluation.is_pass + evaluation_conclusion = evaluation.conclusion + legacy_user_requirements.append(r) + legacy_user_requirements_interaction_events.append(interaction_events) + legacy_user_requirements_trd = trd + + return use_case_actors, legacy_user_requirements_trd + + +async def _write_framework(context: Context, use_case_actors: str, trd: str, acknowledge: str, constraint: str) -> str: + write_framework = WriteFramework(context=context) + evaluate_framework = EvaluateFramework(context=context) + is_pass = False + framework = "" + evaluation_conclusion = "" + while not is_pass and (context.cost_manager.total_cost < context.cost_manager.max_budget): + try: + framework = await write_framework.run( + use_case_actors=use_case_actors, + trd=trd, + acknowledge=acknowledge, + legacy_output=framework, + evaluation_conclusion=evaluation_conclusion, + additional_technical_requirements=constraint, + ) + except Exception as e: + logger.info(f"{e}") + break + evaluation = await evaluate_framework.run( + use_case_actors=use_case_actors, + trd=trd, + acknowledge=acknowledge, + legacy_output=framework, + additional_technical_requirements=constraint, + ) + is_pass = evaluation.is_pass + evaluation_conclusion = evaluation.conclusion + return framework + + +async def develop( + context: Context, + user_requirement_filename: str, + actors_filename: str, + acknowledge_filename: str, + constraint_filename: str, + output_dir: str, +): + output_dir = Path(output_dir) if output_dir else DEFAULT_WORKSPACE_ROOT / uuid.uuid4().hex + + v = await aread(filename=user_requirement_filename) + user_requirements = json.loads(v) + v = await aread(filename=actors_filename) + actors = json.loads(v) + acknowledge = await aread(filename=acknowledge_filename) + technical_constraint = await aread(filename=constraint_filename) + + # Compress acknowledge + compress_acknowledge = CompressExternalInterfaces(context=context) + available_external_interfaces = await compress_acknowledge.run(acknowledge=acknowledge) + + # Write TRD + use_case_actors, trd = await _write_trd( + context=context, + actors=actors, + user_requirements=user_requirements, + available_external_interfaces=available_external_interfaces, + ) + + # Write framework + framework = await _write_framework( + context=context, + use_case_actors=use_case_actors, + trd=trd, + acknowledge=acknowledge, + constraint=technical_constraint, + ) + + # Save + file_list = await save_framework(dir_data=framework, trd=trd, output_dir=output_dir) + logger.info(f"Output:\n{file_list}") + + +@app.command() +def startup( + user_requirement_filename: str = typer.Argument(..., help="The filename of the user requirements."), + actors_filename: str = typer.Argument(..., help="The filename of UML use case actors description."), + acknowledge_filename: str = typer.Argument(..., help="External interfaces declarations."), + llm_config: str = typer.Option(default="", help="Low-cost LLM config"), + constraint_filename: str = typer.Option(default="", help="What technical dependency constraints are."), + output_dir: str = typer.Option(default="", help="Output directory."), + investment: float = typer.Option(default=15.0, help="Dollar amount to invest in the AI company."), +): + if llm_config and Path(llm_config).exists(): + config = Config.from_yaml_file(Path(llm_config)) + else: + logger.info("GPT 4 turbo is recommended") + config = Config.default() + ctx = Context(config=config) + ctx.cost_manager.max_budget = investment + + asyncio.run( + develop(ctx, user_requirement_filename, actors_filename, acknowledge_filename, constraint_filename, output_dir) + ) + + +if __name__ == "__main__": + app() diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index cc88171ff..7fac6710b 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -254,25 +254,29 @@ class WriteDesign(Action): extra_info=to_markdown_code_block(extra_info), prd=to_markdown_code_block(prd_content), ) - if not legacy_design_filename: - node = await self._new_system_design(context=context) - design = Document(content=node.instruct_content.model_dump_json()) - else: - old_design_content = await aread(filename=legacy_design_filename) - design = await self._merge( - prd_doc=Document(content=context), system_design_doc=Document(content=old_design_content) - ) + async with DocsReporter(enable_llm_stream=True) as reporter: + await reporter.async_report({"type": "design"}, "meta") + if not legacy_design_filename: + node = await self._new_system_design(context=context) + design = Document(content=node.instruct_content.model_dump_json()) + else: + old_design_content = await aread(filename=legacy_design_filename) + design = await self._merge( + prd_doc=Document(content=context), system_design_doc=Document(content=old_design_content) + ) - if not output_pathname: - output_pathname = Path(output_pathname) / "docs" / "sytem_design.json" - output_pathname.mkdir(parents=True, exist_ok=True) - elif not Path(output_pathname).is_absolute(): - output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname - output_pathname = Path(output_pathname) - await awrite(filename=output_pathname, data=design.content) - output_filename = output_pathname.parent / f"{output_pathname.stem}-class-diagram" - await self._save_data_api_design(design_doc=design, output_filename=output_filename) - output_filename = output_pathname.parent / f"{output_pathname.stem}-sequence-diagram" - await self._save_seq_flow(design_doc=design, output_filename=output_filename) - await save_json_to_markdown(content=design.content, output_filename=output_pathname.with_suffix(".md")) + if not output_pathname: + output_pathname = Path(output_pathname) / "docs" / "sytem_design.json" + output_pathname.mkdir(parents=True, exist_ok=True) + elif not Path(output_pathname).is_absolute(): + output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname + output_pathname = Path(output_pathname) + await awrite(filename=output_pathname, data=design.content) + output_filename = output_pathname.parent / f"{output_pathname.stem}-class-diagram" + await self._save_data_api_design(design_doc=design, output_filename=output_filename) + output_filename = output_pathname.parent / f"{output_pathname.stem}-sequence-diagram" + await self._save_seq_flow(design_doc=design, output_filename=output_filename) + md_output_filename = output_pathname.with_suffix(".md") + await save_json_to_markdown(content=design.content, output_filename=md_output_filename) + await reporter.async_report(md_output_filename, "path") return f'System Design filename: "{str(output_pathname)}"' diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index a39840bf1..e810a5dbe 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -180,16 +180,20 @@ class WriteTasks(Action): if design_filename: content = await aread(filename=design_filename) context += to_markdown_code_block(content) - node = await self._run_new_tasks(context) - file_content = node.instruct_content.model_dump_json() - if not output_pathname: - output_pathname = Path(output_pathname) / "docs" / "project_schedule.json" - output_pathname.mkdir(parents=True, exist_ok=True) - elif not Path(output_pathname).is_absolute(): - output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname - output_pathname = Path(output_pathname) - await awrite(filename=output_pathname, data=file_content) - await save_json_to_markdown(content=file_content, output_filename=output_pathname.with_suffix(".md")) + async with DocsReporter(enable_llm_stream=True) as reporter: + await reporter.async_report({"type": "task"}, "meta") + node = await self._run_new_tasks(context) + file_content = node.instruct_content.model_dump_json() + if not output_pathname: + output_pathname = Path(output_pathname) / "docs" / "project_schedule.json" + output_pathname.mkdir(parents=True, exist_ok=True) + elif not Path(output_pathname).is_absolute(): + output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname + output_pathname = Path(output_pathname) + await awrite(filename=output_pathname, data=file_content) + md_output_filename = output_pathname.with_suffix(".md") + await save_json_to_markdown(content=file_content, output_filename=md_output_filename) + await reporter.async_report(md_output_filename, "path") return f'Project Schedule filename: "{str(output_pathname)}"' diff --git a/metagpt/actions/requirement_analysis/__init__.py b/metagpt/actions/requirement_analysis/__init__.py new file mode 100644 index 000000000..d196bafee --- /dev/null +++ b/metagpt/actions/requirement_analysis/__init__.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : __init__.py +@Desc : The implementation of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" +from metagpt.actions.requirement_analysis.evaluate_action import EvaluationData, EvaluateAction + +__all__ = [EvaluationData, EvaluateAction] diff --git a/metagpt/actions/requirement_analysis/evaluate_action.py b/metagpt/actions/requirement_analysis/evaluate_action.py new file mode 100644 index 000000000..376c73f2c --- /dev/null +++ b/metagpt/actions/requirement_analysis/evaluate_action.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : evaluate_action.py +@Desc : The implementation of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" +from typing import Optional + +from pydantic import BaseModel +from tenacity import retry, stop_after_attempt, wait_random_exponential + +from metagpt.actions import Action +from metagpt.logs import logger +from metagpt.utils.common import CodeParser, general_after_log, to_markdown_code_block + + +class EvaluationData(BaseModel): + """Model to represent evaluation data. + + Attributes: + is_pass (bool): Indicates if the evaluation passed or failed. + conclusion (Optional[str]): Conclusion or remarks about the evaluation. + """ + + is_pass: bool + conclusion: Optional[str] = None + + +class EvaluateAction(Action): + """The base class for an evaluation action. + + This class provides methods to evaluate prompts using a specified language model. + """ + + @retry( + wait=wait_random_exponential(min=1, max=20), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) + async def _evaluate(self, prompt: str) -> (bool, str): + """Evaluates a given prompt. + + Args: + prompt (str): The prompt to be evaluated. + + Returns: + tuple: A tuple containing: + - bool: Indicates if the evaluation passed. + - str: The JSON string containing the evaluation data. + """ + rsp = await self.llm.aask(prompt) + json_data = CodeParser.parse_code(text=rsp, lang="json") + data = EvaluationData.model_validate_json(json_data) + return data.is_pass, to_markdown_code_block(val=json_data, type_="json") + + async def _vote(self, prompt: str) -> EvaluationData: + """Evaluates a prompt multiple times and returns the consensus. + + Args: + prompt (str): The prompt to be evaluated. + + Returns: + EvaluationData: An object containing the evaluation result and a summary of evaluations. + """ + evaluations = {} + for i in range(3): + vote, evaluation = await self._evaluate(prompt) + val = evaluations.get(vote, []) + val.append(evaluation) + if len(val) > 1: + return EvaluationData(is_pass=vote, conclusion="\n".join(val)) + evaluations[vote] = val diff --git a/metagpt/actions/requirement_analysis/framework/__init__.py b/metagpt/actions/requirement_analysis/framework/__init__.py new file mode 100644 index 000000000..3f7d31b99 --- /dev/null +++ b/metagpt/actions/requirement_analysis/framework/__init__.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : __init__.py +@Desc : The implementation of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" +import json +import uuid +from datetime import datetime +from pathlib import Path +from typing import Optional, Union, List + +from pydantic import BaseModel + +from metagpt.actions.requirement_analysis.framework.evaluate_framework import EvaluateFramework +from metagpt.actions.requirement_analysis.framework.write_framework import WriteFramework +from metagpt.const import DEFAULT_WORKSPACE_ROOT +from metagpt.utils.common import awrite + + +async def save_framework( + dir_data: str, trd: Optional[str] = None, output_dir: Optional[Union[str, Path]] = None +) -> List[str]: + """ + Saves framework data to files based on input JSON data and optionally saves a TRD (technical requirements document). + + Args: + dir_data (str): JSON data in string format enclosed in triple backticks ("```json" "...data..." "```"). + trd (str, optional): Technical requirements document content to be saved. Defaults to None. + output_dir (Union[str, Path], optional): Output directory path where files will be saved. If not provided, + a default directory is created based on the current timestamp and a random UUID suffix. + + Returns: + List[str]: List of file paths where data was saved. + + Raises: + Any exceptions raised during file writing operations. + + Notes: + - JSON data should be provided in the format "```json ...data... ```". + - The function ensures that paths and filenames are correctly formatted and creates necessary directories. + + Example: + ```python + dir_data = "```json\n[{\"path\": \"/folder\", \"filename\": \"file1.txt\", \"content\": \"Some content\"}]\n```" + trd = "Technical requirements document content." + output_dir = '/path/to/output/dir' + saved_files = await save_framework(dir_data, trd, output_dir) + print(saved_files) + ``` + """ + output_dir = ( + Path(output_dir) + if output_dir + else DEFAULT_WORKSPACE_ROOT / (datetime.now().strftime("%Y%m%d%H%M%ST") + uuid.uuid4().hex[0:8]) + ) + output_dir.mkdir(parents=True, exist_ok=True) + + json_data = dir_data.removeprefix("```json").removesuffix("```") + items = json.loads(json_data) + + class Data(BaseModel): + path: str + filename: str + content: str + + if trd: + pathname = output_dir / "TRD.md" + await awrite(filename=pathname, data=trd) + + files = [] + for i in items: + v = Data.model_validate(i) + if v.path and v.path[0] == "/": + v.path = "." + v.path + pathname = output_dir / v.path + pathname.mkdir(parents=True, exist_ok=True) + pathname = pathname / v.filename + await awrite(filename=pathname, data=v.content) + files.append(str(pathname)) + return files + + +__all__ = [WriteFramework, EvaluateFramework] diff --git a/metagpt/actions/requirement_analysis/framework/evaluate_framework.py b/metagpt/actions/requirement_analysis/framework/evaluate_framework.py new file mode 100644 index 000000000..2f9239658 --- /dev/null +++ b/metagpt/actions/requirement_analysis/framework/evaluate_framework.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : evaluate_framework.py +@Desc : The implementation of Chapter 2.1.8 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" + +from metagpt.actions.requirement_analysis import EvaluateAction, EvaluationData +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import to_markdown_code_block + + +@register_tool(include_functions=["run"]) +class EvaluateFramework(EvaluateAction): + """WriteFramework deal with the following situations: + 1. Given a TRD and the software framework based on the TRD, evaluate the quality of the software framework. + """ + + async def run( + self, + *, + use_case_actors: str, + trd: str, + acknowledge: str, + legacy_output: str, + additional_technical_requirements: str, + ) -> EvaluationData: + """ + Run the evaluation of the software framework based on the provided TRD and related parameters. + + Args: + use_case_actors (str): A description of the actors involved in the use case. + trd (str): The Technical Requirements Document (TRD) that outlines the requirements for the software framework. + acknowledge (str): External acknowledgments or acknowledgments information related to the framework. + legacy_output (str): The previous versions of software framework returned by `WriteFramework`. + additional_technical_requirements (str): Additional technical requirements that need to be considered during evaluation. + + Returns: + EvaluationData: An object containing the results of the evaluation. + + Example: + >>> evaluate_framework = EvaluateFramework() + >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;" + >>> trd = "## TRD\\n..." + >>> acknowledge = "## Interfaces\\n..." + >>> framework = '{"path":"balabala", "filename":"...", ...' + >>> constraint = "Using Java language, ..." + >>> evaluation = await evaluate_framework.run( + >>> use_case_actors=use_case_actors, + >>> trd=trd, + >>> acknowledge=acknowledge, + >>> legacy_output=framework, + >>> additional_technical_requirements=constraint, + >>> ) + >>> is_pass = evaluation.is_pass + >>> print(is_pass) + True + >>> evaluation_conclusion = evaluation.conclusion + >>> print(evaluation_conclusion) + Balabala... + """ + prompt = PROMPT.format( + use_case_actors=use_case_actors, + trd=to_markdown_code_block(val=trd), + acknowledge=to_markdown_code_block(val=acknowledge), + legacy_output=to_markdown_code_block(val=legacy_output), + additional_technical_requirements=to_markdown_code_block(val=additional_technical_requirements), + ) + return await self._vote(prompt) + + +PROMPT = """ +## Actor, System, External System +{use_case_actors} + +## Legacy TRD +{trd} + +## Acknowledge +{acknowledge} + +## Legacy Outputs +{legacy_output} + +## Additional Technical Requirements +{additional_technical_requirements} + +--- +You are a tool that evaluates the quality of framework code based on the TRD content; +You need to refer to the content of the "Legacy TRD" section to check for any errors or omissions in the framework code found in "Legacy Outputs"; +The content of "Actor, System, External System" provides an explanation of actors and systems that appear in UML Use Case diagram; +Information about the external system missing from the "Legacy TRD" can be found in the "Acknowledge" section; +Which interfaces defined in "Acknowledge" are used in the "Legacy TRD"? +Do not implement the interface in "Acknowledge" section until it is used in "Legacy TRD", you can check whether they are the same interface by looking at its ID or url; +Parts not mentioned in the "Legacy TRD" will be handled by other TRDs, therefore, processes not present in the "Legacy TRD" are considered ready; +"Additional Technical Requirements" specifies the additional technical requirements that the generated software framework code must meet; +Do the parameters of the interface of the external system used in the code comply with it's specifications in 'Acknowledge'? +Is there a lack of necessary configuration files? +Return a markdown JSON object with: +- an "issues" key containing a string list of natural text about the issues that need to addressed, found in the "Legacy Outputs" if any exits, each issue found must provide a detailed description and include reasons; +- a "conclusion" key containing the evaluation conclusion; +- a "misalignment" key containing the judgement detail of the natural text string list about the misalignment with "Legacy TRD"; +- a "is_pass" key containing a true boolean value if there is not any issue in the "Legacy Outputs"; +""" diff --git a/metagpt/actions/requirement_analysis/framework/write_framework.py b/metagpt/actions/requirement_analysis/framework/write_framework.py new file mode 100644 index 000000000..2aa03f447 --- /dev/null +++ b/metagpt/actions/requirement_analysis/framework/write_framework.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : write_framework.py +@Desc : The implementation of Chapter 2.1.8 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" +import json + +from tenacity import retry, stop_after_attempt, wait_random_exponential + +from metagpt.actions import Action +from metagpt.logs import logger +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import general_after_log, to_markdown_code_block + + +@register_tool(include_functions=["run"]) +class WriteFramework(Action): + """WriteFramework deal with the following situations: + 1. Given a TRD, write out the software framework. + """ + + async def run( + self, + *, + use_case_actors: str, + trd: str, + acknowledge: str, + legacy_output: str, + evaluation_conclusion: str, + additional_technical_requirements: str, + ) -> str: + """ + Run the action to generate a software framework based on the provided TRD and related information. + + Args: + use_case_actors (str): Description of the use case actors involved. + trd (str): Technical Requirements Document detailing the requirements. + acknowledge (str): External acknowledgements or acknowledgements required. + legacy_output (str): Previous version of the software framework returned by `WriteFramework.run`. + evaluation_conclusion (str): Conclusion from the evaluation of the requirements. + additional_technical_requirements (str): Any additional technical requirements. + + Returns: + str: The generated software framework as a string. + + Example: + >>> write_framework = WriteFramework() + >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;" + >>> trd = "## TRD\\n..." + >>> acknowledge = "## Interfaces\\n..." + >>> legacy_output = '{"path":"balabala", "filename":"...", ...' + >>> evaluation_conclusion = "Balabala..." + >>> constraint = "Using Java language, ..." + >>> framework = await write_framework.run( + >>> use_case_actors=use_case_actors, + >>> trd=trd, + >>> acknowledge=acknowledge, + >>> legacy_output=framework, + >>> evaluation_conclusion=evaluation_conclusion, + >>> additional_technical_requirements=constraint, + >>> ) + >>> print(framework) + {"path":"balabala", "filename":"...", ... + + """ + acknowledge = await self._extract_external_interfaces(trd=trd, knowledge=acknowledge) + prompt = PROMPT.format( + use_case_actors=use_case_actors, + trd=to_markdown_code_block(val=trd), + acknowledge=to_markdown_code_block(val=acknowledge), + legacy_output=to_markdown_code_block(val=legacy_output), + evaluation_conclusion=evaluation_conclusion, + additional_technical_requirements=to_markdown_code_block(val=additional_technical_requirements), + ) + return await self._write(prompt) + + @retry( + wait=wait_random_exponential(min=1, max=20), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) + async def _write(self, prompt: str) -> str: + rsp = await self.llm.aask(prompt) + # Do not use `CodeParser` here. + tags = ["```json", "```"] + bix = rsp.find(tags[0]) + eix = rsp.rfind(tags[1]) + if bix >= 0: + rsp = rsp[bix : eix + len(tags[1])] + json_data = rsp.removeprefix("```json").removesuffix("```") + json.loads(json_data) # validate + return json_data + + @retry( + wait=wait_random_exponential(min=1, max=20), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) + async def _extract_external_interfaces(self, trd: str, knowledge: str) -> str: + prompt = f"## TRD\n{to_markdown_code_block(val=trd)}\n\n## Knowledge\n{to_markdown_code_block(val=knowledge)}\n" + rsp = await self.llm.aask( + prompt, + system_msgs=[ + "You are a tool that removes impurities from articles; you can remove irrelevant content from articles.", + 'Identify which interfaces are used in "TRD"? Remove the relevant content of the interfaces NOT used in "TRD" from "Knowledge" and return the simplified content of "Knowledge".', + ], + ) + return rsp + + +PROMPT = """ +## Actor, System, External System +{use_case_actors} + +## TRD +{trd} + +## Acknowledge +{acknowledge} + +## Legacy Outputs +{legacy_output} + +## Evaluation Conclusion +{evaluation_conclusion} + +## Additional Technical Requirements +{additional_technical_requirements} + +--- +You are a tool that generates software framework code based on TRD. +The content of "Actor, System, External System" provides an explanation of actors and systems that appear in UML Use Case diagram; +The descriptions of the interfaces of the external system used in the "TRD" can be found in the "Acknowledge" section; Do not implement the interface of the external system in "Acknowledge" section until it is used in "TRD"; +"Legacy Outputs" contains the software framework code generated by you last time, which you can improve by addressing the issues raised in "Evaluation Conclusion"; +"Additional Technical Requirements" specifies the additional technical requirements that the generated software framework code must meet; +Develop the software framework based on the "TRD", the output files should include: +- The `README.md` file should include: + - The folder structure diagram of the entire project; + - Correspondence between classes, interfaces, and functions with the content in the "TRD" section； + - Prerequisites if necessary; + - Installation if necessary; + - Configuration if necessary; + - Usage if necessary; +- The `CLASS.md` file should include the class diagram in PlantUML format based on the "TRD"; +- The `SEQUENCE.md` file should include the sequence diagram in PlantUML format based on the "TRD"; +- The source code files that implement the "TRD" and "Additional Technical Requirements"; do not add comments to source code files; +- The configuration files that required by the source code files, "TRD" and "Additional Technical Requirements"; + +Return a markdown JSON object list, each object containing: +- a "path" key with a value specifying its path; +- a "filename" key with a value specifying its file name; +- a "content" key with a value containing its file content; +""" diff --git a/metagpt/actions/requirement_analysis/requirement/__init__.py b/metagpt/actions/requirement_analysis/requirement/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/metagpt/actions/requirement_analysis/requirement/pic2txt.py b/metagpt/actions/requirement_analysis/requirement/pic2txt.py new file mode 100644 index 000000000..b8f236dac --- /dev/null +++ b/metagpt/actions/requirement_analysis/requirement/pic2txt.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/27 +@Author : mashenquan +@File : pic2txt.py +""" +import json +from pathlib import Path +from typing import List + +from tenacity import retry, stop_after_attempt, wait_random_exponential + +from metagpt.actions import Action +from metagpt.logs import logger +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import encode_image, general_after_log, to_markdown_code_block + + +@register_tool(include_functions=["run"]) +class Pic2Txt(Action): + """Pic2Txt deal with the following situations: + Given some pictures depicting user requirements alongside contextual description, write out the intact textual user requirements. + """ + + async def run( + self, + *, + image_paths: List[str], + textual_user_requirement: str = "", + legacy_output: str = "", + evaluation_conclusion: str = "", + additional_technical_requirements: str = "", + ) -> str: + """ + Given some pictures depicting user requirements alongside contextual description, write out the intact textual user requirements + + Args: + image_paths (List[str]): A list of file paths to the input image(s) depicting user requirements. + textual_user_requirement (str, optional): Textual user requirement that alongside the given images, if any. + legacy_output (str, optional): The intact textual user requirements generated by you last time, if any. + evaluation_conclusion (str, optional): Conclusion or evaluation based on the processed requirements. + additional_technical_requirements (str, optional): Any supplementary technical details relevant to the process. + + Returns: + str: Textual representation of user requirements extracted from the provided image(s). + + Raises: + ValueError: If image_paths list is empty. + OSError: If there is an issue accessing or reading the image files. + + Example: + >>> images = ["requirements/pic/1.png", "requirements/pic/2.png", "requirements/pic/3.png"] + >>> textual_user_requirements = "User requirement paragraph 1 ..., ![](1.png). paragraph 2...![](2.png)..." + >>> action = Pic2Txt() + >>> intact_textual_user_requirements = await action.run(image_paths=images, textual_user_requirement=textual_user_requirements) + >>> print(intact_textual_user_requirements) + "User requirement paragraph 1 ..., ![...](1.png) This picture describes... paragraph 2...![...](2.png)..." + + """ + descriptions = {} + for i in image_paths: + filename = Path(i) + base64_image = encode_image(filename) + rsp = await self._pic2txt( + "Generate a paragraph of text based on the content of the image, the language of the text is consistent with the language in the image.", + base64_image=base64_image, + ) + descriptions[filename.name] = rsp + + prompt = PROMPT.format( + textual_user_requirement=textual_user_requirement, + acknowledge=to_markdown_code_block(val=json.dumps(descriptions), type_="json"), + legacy_output=to_markdown_code_block(val=legacy_output), + evaluation_conclusion=evaluation_conclusion, + additional_technical_requirements=to_markdown_code_block(val=additional_technical_requirements), + ) + return await self._write(prompt) + + @retry( + wait=wait_random_exponential(min=1, max=20), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) + async def _write(self, prompt: str) -> str: + rsp = await self.llm.aask(prompt) + return rsp + + @retry( + wait=wait_random_exponential(min=1, max=20), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) + async def _pic2txt(self, prompt: str, base64_image: str) -> str: + rsp = await self.llm.aask(prompt, images=base64_image) + return rsp + + +PROMPT = """ +## Textual User Requirements +{textual_user_requirement} + +## Acknowledge +{acknowledge} + +## Legacy Outputs +{legacy_output} + +## Evaluation Conclusion +{evaluation_conclusion} + +## Additional Technical Requirements +{additional_technical_requirements} + +--- +You are a tool that generates an intact textual user requirements given a few of textual fragments of user requirements and some fragments of UI pictures. +The content of "Textual User Requirements" provides a few of textual fragments of user requirements; +The content of "Acknowledge" provides the descriptions of pictures used in "Textual User Requirements"; +"Legacy Outputs" contains the intact textual user requirements generated by you last time, which you can improve by addressing the issues raised in "Evaluation Conclusion"; +"Additional Technical Requirements" specifies the additional technical requirements that the generated textual user requirements must meet; +You need to merge the text content of the corresponding image in the "Acknowledge" into the "Textual User Requirements" to generate a complete, natural and coherent description of the user requirements; +Return the intact textual user requirements according to the given fragments of the user requirement of "Textual User Requirements" and the UI pictures; +""" diff --git a/metagpt/actions/requirement_analysis/trd/__init__.py b/metagpt/actions/requirement_analysis/trd/__init__.py new file mode 100644 index 000000000..4603532c4 --- /dev/null +++ b/metagpt/actions/requirement_analysis/trd/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : __init__.py +@Desc : The implementation of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" + + +from metagpt.actions.requirement_analysis.trd.detect_interaction import DetectInteraction +from metagpt.actions.requirement_analysis.trd.evaluate_trd import EvaluateTRD +from metagpt.actions.requirement_analysis.trd.write_trd import WriteTRD +from metagpt.actions.requirement_analysis.trd.compress_external_interfaces import CompressExternalInterfaces + +__all__ = [CompressExternalInterfaces, DetectInteraction, WriteTRD, EvaluateTRD] diff --git a/metagpt/actions/requirement_analysis/trd/compress_external_interfaces.py b/metagpt/actions/requirement_analysis/trd/compress_external_interfaces.py new file mode 100644 index 000000000..abaf6fc30 --- /dev/null +++ b/metagpt/actions/requirement_analysis/trd/compress_external_interfaces.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : compress_external_interfaces.py +@Desc : The implementation of Chapter 2.1.5 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" +from tenacity import retry, stop_after_attempt, wait_random_exponential + +from metagpt.actions import Action +from metagpt.logs import logger +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import general_after_log + + +@register_tool(include_functions=["run"]) +class CompressExternalInterfaces(Action): + """CompressExternalInterfaces deal with the following situations: + 1. Given a natural text of acknowledgement, it extracts and compresses the information about external system interfaces. + """ + + @retry( + wait=wait_random_exponential(min=1, max=20), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) + async def run( + self, + *, + acknowledge: str, + ) -> str: + """ + Extracts and compresses information about external system interfaces from a given acknowledgement text. + + Args: + acknowledge (str): A natural text of acknowledgement containing details about external system interfaces. + + Returns: + str: A compressed version of the information about external system interfaces. + + Example: + >>> compress_acknowledge = CompressExternalInterfaces() + >>> acknowledge = "## Interfaces\\n..." + >>> available_external_interfaces = await compress_acknowledge.run(acknowledge=acknowledge) + >>> print(available_external_interfaces) + ```json\n[\n{\n"id": 1,\n"inputs": {... + """ + return await self.llm.aask( + msg=acknowledge, + system_msgs=[ + "Extracts and compresses the information about external system interfaces.", + "Return a markdown JSON list of objects, each object containing:\n" + '- an "id" key containing the interface id;\n' + '- an "inputs" key containing a dict of input parameters that consist of name and description pairs;\n' + '- an "outputs" key containing a dict of returns that consist of name and description pairs;\n', + ], + ) diff --git a/metagpt/actions/requirement_analysis/trd/detect_interaction.py b/metagpt/actions/requirement_analysis/trd/detect_interaction.py new file mode 100644 index 000000000..b77193194 --- /dev/null +++ b/metagpt/actions/requirement_analysis/trd/detect_interaction.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : detect_interaction.py +@Desc : The implementation of Chapter 2.1.6 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" +from tenacity import retry, stop_after_attempt, wait_random_exponential + +from metagpt.actions import Action +from metagpt.logs import logger +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import general_after_log, to_markdown_code_block + + +@register_tool(include_functions=["run"]) +class DetectInteraction(Action): + """DetectInteraction deal with the following situations: + 1. Given a natural text of user requirements, it identifies the interaction events and the participants of those interactions from the original text. + """ + + @retry( + wait=wait_random_exponential(min=1, max=20), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) + async def run( + self, + *, + user_requirements: str, + use_case_actors: str, + legacy_interaction_events: str, + evaluation_conclusion: str, + ) -> str: + """ + Identifies interaction events and participants from the user requirements. + + Args: + user_requirements (str): A natural language text detailing the user's requirements. + use_case_actors (str): A description of the actors involved in the use case. + legacy_interaction_events (str): The previous version of the interaction events identified by you. + evaluation_conclusion (str): The external evaluation conclusions regarding the interactions events identified by you. + + Returns: + str: A string summarizing the identified interaction events and their participants. + + Example: + >>> detect_interaction = DetectInteraction() + >>> user_requirements = "User requirements 1. ..." + >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;" + >>> previous_version_interaction_events = "['interaction ...', ...]" + >>> evaluation_conclusion = "Issues: ..." + >>> interaction_events = await detect_interaction.run( + >>> user_requirements=user_requirements, + >>> use_case_actors=use_case_actors, + >>> legacy_interaction_events=previous_version_interaction_events, + >>> evaluation_conclusion=evaluation_conclusion, + >>> ) + >>> print(interaction_events) + "['interaction ...', ...]" + """ + msg = PROMPT.format( + use_case_actors=use_case_actors, + original_user_requirements=to_markdown_code_block(val=user_requirements), + previous_version_of_interaction_events=legacy_interaction_events, + the_evaluation_conclusion_of_previous_version_of_trd=evaluation_conclusion, + ) + return await self.llm.aask(msg=msg) + + +PROMPT = """ +## Actor, System, External System +{use_case_actors} + +## User Requirements +{original_user_requirements} + +## Legacy Interaction Events +{previous_version_of_interaction_events} + +## Evaluation Conclusion +{the_evaluation_conclusion_of_previous_version_of_trd} + +--- +You are a tool for capturing interaction events. +"Actor, System, External System" provides the possible participants of the interaction event; +"Legacy Interaction Events" is the contents of the interaction events that you output earlier; +Some descriptions in the "Evaluation Conclusion" relate to the content of "User Requirements", and these descriptions in the "Evaluation Conclusion" address some issues regarding the content of "Legacy Interaction Events"; +You need to capture the interaction events occurring in the description within the content of "User Requirements" word-for-word, including: +1. Who is interacting with whom. An interaction event has a maximum of 2 participants. If there are multiple participants, it indicates that multiple events are combined into one event and should be further split; +2. When an interaction event occurs, who is the initiator? What data did the initiator enter? +3. What data does the interaction event ultimately return according to the "User Requirements"? + +You can check the data flow described in the "User Requirements" to see if there are any missing interaction events; +Return a markdown JSON object list, each object of the list containing: +- a "name" key containing the name of the interaction event; +- a "participants" key containing a string list of the names of the two participants; +- a "initiator" key containing the name of the participant who initiate the interaction; +- a "input" key containing a natural text description about the input data; +""" diff --git a/metagpt/actions/requirement_analysis/trd/evaluate_trd.py b/metagpt/actions/requirement_analysis/trd/evaluate_trd.py new file mode 100644 index 000000000..5c256ed07 --- /dev/null +++ b/metagpt/actions/requirement_analysis/trd/evaluate_trd.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : evaluate_trd.py +@Desc : The implementation of Chapter 2.1.6~2.1.7 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" + +from metagpt.actions.requirement_analysis import EvaluateAction, EvaluationData +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import to_markdown_code_block + + +@register_tool(include_functions=["run"]) +class EvaluateTRD(EvaluateAction): + """EvaluateTRD deal with the following situations: + 1. Given a TRD, evaluates the quality and returns a conclusion. + """ + + async def run( + self, + *, + user_requirements: str, + use_case_actors: str, + trd: str, + interaction_events: str, + legacy_user_requirements_interaction_events: str = "", + ) -> EvaluationData: + """ + Evaluates the given TRD based on user requirements, use case actors, interaction events, and optionally external legacy interaction events. + + Args: + user_requirements (str): The requirements provided by the user. + use_case_actors (str): The actors involved in the use case. + trd (str): The TRD (Technical Requirements Document) to be evaluated. + interaction_events (str): The interaction events related to the user requirements and the TRD. + legacy_user_requirements_interaction_events (str, optional): External legacy interaction events tied to the user requirements. Defaults to an empty string. + + Returns: + EvaluationData: The conclusion of the TRD evaluation. + + Example: + >>> evaluate_trd = EvaluateTRD() + >>> user_requirements = "User requirements 1. ..." + >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;" + >>> trd = "## TRD\\n..." + >>> interaction_events = "['interaction ...', ...]" + >>> evaluation_conclusion = "Issues: ..." + >>> legacy_user_requirements_interaction_events = ["user requirements 1. ...", ...] + >>> evaluation = await evaluate_trd.run( + >>> user_requirements=user_requirements, + >>> use_case_actors=use_case_actors, + >>> trd=trd, + >>> interaction_events=interaction_events, + >>> legacy_user_requirements_interaction_events=str(legacy_user_requirements_interaction_events), + >>> ) + >>> is_pass = evaluation.is_pass + >>> print(is_pass) + True + >>> evaluation_conclusion = evaluation.conclusion + >>> print(evaluation_conclusion) + ## Conclustion\n balabalabala... + + """ + prompt = PROMPT.format( + use_case_actors=use_case_actors, + user_requirements=to_markdown_code_block(val=user_requirements), + trd=to_markdown_code_block(val=trd), + legacy_user_requirements_interaction_events=legacy_user_requirements_interaction_events, + interaction_events=interaction_events, + ) + return await self._vote(prompt) + + +PROMPT = """ +## Actor, System, External System +{use_case_actors} + +## User Requirements +{user_requirements} + +## TRD Design +{trd} + +## External Interaction Events +{legacy_user_requirements_interaction_events} + +## Interaction Events +{legacy_user_requirements_interaction_events} +{interaction_events} + +--- +You are a tool to evaluate the TRD design. +"Actor, System, External System" provides the all possible participants in interaction events; +"User Requirements" provides the original requirements description, any parts not mentioned in this description will be handled by other modules, so do not fabricate requirements; +"External Interaction Events" is provided by an external module for your use, its content is also referred to "Interaction Events" section; The content in "External Interaction Events" can be determined to be problem-free; +"External Interaction Events" provides some identified interaction events and the interacting participants based on the part of the content of the "User Requirements"; +"Interaction Events" provides some identified interaction events and the interacting participants based on the content of the "User Requirements"; +"TRD Design" provides a comprehensive design of the implementation steps for the original requirements, incorporating the interaction events from "Interaction Events" and adding additional steps to connect the complete upstream and downstream data flows; +In order to integrate the full upstream and downstream data flow, the "TRD Design" allows for the inclusion of steps that do not appear in the original requirements description, but do not conflict with those explicitly described in the "User Requirements"; +Which interactions from "Interaction Events" correspond to which steps in "TRD Design"? Please provide reasons. +Which aspects of "TRD Design" and "Interaction Events" do not align with the descriptions in "User Requirements"? Please provide detailed descriptions and reasons. +If the descriptions in "User Requirements" are divided into multiple steps in "TRD Design" and "Interaction Events," it can be considered compliant with the descriptions in "User Requirements" as long as it does not conflict with them; +There is a possibility of missing details in the descriptions of "User Requirements". Any additional steps in "TRD Design" and "Interaction Events" are considered compliant with "User Requirements" as long as they do not conflict with the descriptions provided in "User Requirements"; +If there are interaction events with external systems in "TRD Design", you must explicitly specify the ID of the external interface to use for the interaction events, the input and output parameters of the used external interface must explictly match the input and output of the interaction event； +Does the sequence of steps in "Interaction Events" cause performance or cost issues? Please provide detailed descriptions and reasons; +If each step of "TRD Design" has input data, its input data is provided either by the output of the previous steps or by participants of "Actor, System, External System", and there should be no passive data; +Return a markdown JSON object with: +- an "issues" key containing a string list of natural text about the issues that need to be addressed, found in the "TRD Design" if any exist, each issue found must provide a detailed description and include reasons; +- a "conclusion" key containing the evaluation conclusion; +- a "correspondence_between" key containing the judgement detail of the natural text string list about the correspondence between "Interaction Events" and "TRD Design" steps; +- a "misalignment" key containing the judgement detail of the natural text string list about the misalignment with "User Requirements"; +- a "is_pass" key containing a true boolean value if there is not any issue in the "TRD Design"; +""" diff --git a/metagpt/actions/requirement_analysis/trd/write_trd.py b/metagpt/actions/requirement_analysis/trd/write_trd.py new file mode 100644 index 000000000..bad93ea76 --- /dev/null +++ b/metagpt/actions/requirement_analysis/trd/write_trd.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/6/13 +@Author : mashenquan +@File : write_trd.py +@Desc : The implementation of Chapter 2.1.6~2.1.7 of RFC243. https://deepwisdom.feishu.cn/wiki/QobGwPkImijoyukBUKHcrYetnBb +""" +from tenacity import retry, stop_after_attempt, wait_random_exponential + +from metagpt.actions import Action +from metagpt.logs import logger +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import general_after_log, to_markdown_code_block + + +@register_tool(include_functions=["run"]) +class WriteTRD(Action): + """WriteTRD deal with the following situations: + 1. Given some new user requirements, write out a new TRD(Technical Requirements Document). + 2. Given some incremental user requirements, update the legacy TRD. + """ + + async def run( + self, + *, + user_requirements: str = "", + use_case_actors: str, + available_external_interfaces: str, + evaluation_conclusion: str = "", + interaction_events: str, + previous_version_trd: str = "", + legacy_user_requirements: str = "", + legacy_user_requirements_trd: str = "", + legacy_user_requirements_interaction_events: str = "", + ) -> str: + """ + Handles the writing or updating of a Technical Requirements Document (TRD) based on user requirements. + + Args: + user_requirements (str): The new/incremental user requirements. + use_case_actors (str): Description of the actors involved in the use case. + available_external_interfaces (str): List of available external interfaces. + evaluation_conclusion (str, optional): The conclusion of the evaluation of the TRD written by you. Defaults to an empty string. + interaction_events (str): The interaction events related to the user requirements that you are handling. + previous_version_trd (str, optional): The previous version of the TRD written by you, for updating. + legacy_user_requirements (str, optional): Existing user requirements handled by an external object for your use. Defaults to an empty string. + legacy_user_requirements_trd (str, optional): The TRD associated with the existing user requirements handled by an external object for your use. Defaults to an empty string. + legacy_user_requirements_interaction_events (str, optional): Interaction events related to the existing user requirements handled by an external object for your use. Defaults to an empty string. + + Returns: + str: The newly created or updated TRD written by you. + + Example: + >>> # Given a new user requirements, write out a new TRD. + >>> user_requirements = "Write a 'snake game' TRD." + >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;" + >>> available_external_interfaces = "The available external interfaces returned by `CompressExternalInterfaces.run` are ..." + >>> previous_version_trd = "TRD ..." # The last version of the TRD written out if there is. + >>> evaluation_conclusion = "Conclusion ..." # The conclusion returned by `EvaluateTRD.run` if there is. + >>> interaction_events = "Interaction ..." # The interaction events returned by `DetectInteraction.run`. + >>> write_trd = WriteTRD() + >>> new_version_trd = await write_trd.run( + >>> user_requirements=user_requirements, + >>> use_case_actors=use_case_actors, + >>> available_external_interfaces=available_external_interfaces, + >>> evaluation_conclusion=evaluation_conclusion, + >>> interaction_events=interaction_events, + >>> previous_version_trd=previous_version_trd, + >>> ) + >>> print(new_version_trd) + ## Technical Requirements Document\n ... + + >>> # Given an incremental requirements, update the legacy TRD. + >>> legacy_user_requirements = ["User requirements 1. ...", "User requirements 2. ...", ...] + >>> legacy_user_requirements_trd = "## Technical Requirements Document\\n ..." # The TRD before integrating more user requirements. + >>> legacy_user_requirements_interaction_events = ["The interaction events list of user requirements 1 ...", "The interaction events list of user requiremnts 2 ...", ...] + >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;" + >>> available_external_interfaces = "The available external interfaces returned by `CompressExternalInterfaces.run` are ..." + >>> increment_requirements = "The incremental user requirements are ..." + >>> evaluation_conclusion = "Conclusion ..." # The conclusion returned by `EvaluateTRD.run` if there is. + >>> previous_version_trd = "TRD ..." # The last version of the TRD written out if there is. + >>> write_trd = WriteTRD() + >>> new_version_trd = await write_trd.run( + >>> user_requirements=increment_requirements, + >>> use_case_actors=use_case_actors, + >>> available_external_interfaces=available_external_interfaces, + >>> evaluation_conclusion=evaluation_conclusion, + >>> interaction_events=interaction_events, + >>> previous_version_trd=previous_version_trd, + >>> legacy_user_requirements=str(legacy_user_requirements), + >>> legacy_user_requirements_trd=legacy_user_requirements_trd, + >>> legacy_user_requirements_interaction_events=str(legacy_user_requirements_interaction_events), + >>> ) + >>> print(new_version_trd) + ## Technical Requirements Document\n ... + """ + if legacy_user_requirements: + return await self._write_incremental_trd( + use_case_actors=use_case_actors, + legacy_user_requirements=legacy_user_requirements, + available_external_interfaces=available_external_interfaces, + legacy_user_requirements_trd=legacy_user_requirements_trd, + legacy_user_requirements_interaction_events=legacy_user_requirements_interaction_events, + incremental_user_requirements=user_requirements, + previous_version_trd=previous_version_trd, + evaluation_conclusion=evaluation_conclusion, + incremental_user_requirements_interaction_events=interaction_events, + ) + return await self._write_new_trd( + use_case_actors=use_case_actors, + original_user_requirement=user_requirements, + available_external_interfaces=available_external_interfaces, + legacy_trd=previous_version_trd, + evaluation_conclusion=evaluation_conclusion, + interaction_events=interaction_events, + ) + + @retry( + wait=wait_random_exponential(min=1, max=20), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) + async def _write_new_trd( + self, + *, + use_case_actors: str, + original_user_requirement: str, + available_external_interfaces: str, + legacy_trd: str, + evaluation_conclusion: str, + interaction_events: str, + ) -> str: + prompt = NEW_PROMPT.format( + use_case_actors=use_case_actors, + original_user_requirement=to_markdown_code_block(val=original_user_requirement), + available_external_interfaces=available_external_interfaces, + legacy_trd=to_markdown_code_block(val=legacy_trd), + evaluation_conclusion=evaluation_conclusion, + interaction_events=interaction_events, + ) + return await self.llm.aask(prompt) + + @retry( + wait=wait_random_exponential(min=1, max=20), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) + async def _write_incremental_trd( + self, + *, + use_case_actors: str, + legacy_user_requirements: str, + available_external_interfaces: str, + legacy_user_requirements_trd: str, + legacy_user_requirements_interaction_events: str, + incremental_user_requirements: str, + previous_version_trd: str, + evaluation_conclusion: str, + incremental_user_requirements_interaction_events: str, + ): + prompt = INCREMENTAL_PROMPT.format( + use_case_actors=use_case_actors, + legacy_user_requirements=to_markdown_code_block(val=legacy_user_requirements), + available_external_interfaces=available_external_interfaces, + legacy_user_requirements_trd=to_markdown_code_block(val=legacy_user_requirements_trd), + legacy_user_requirements_interaction_events=legacy_user_requirements_interaction_events, + incremental_user_requirements=to_markdown_code_block(val=incremental_user_requirements), + previous_version_trd=to_markdown_code_block(val=previous_version_trd), + evaluation_conclusion=evaluation_conclusion, + incremental_user_requirements_interaction_events=incremental_user_requirements_interaction_events, + ) + return await self.llm.aask(prompt) + + +NEW_PROMPT = """ +## Actor, System, External System +{use_case_actors} + +## User Requirements +{original_user_requirement} + +## Available External Interfaces +{available_external_interfaces} + +## Legacy TRD +{legacy_trd} + +## Evaluation Conclusion +{evaluation_conclusion} + +## Interaction Events +{interaction_events} + +--- +You are a TRD generator. +The content of "Actor, System, External System" provides an explanation of actors and systems that appear in UML Use Case diagram; +The content of "Available External Interfaces" provides the candidate steps, along with the inputs and outputs of each step; +"User Requirements" provides the original requirements description, any parts not mentioned in this description will be handled by other modules, so do not fabricate requirements; +"Legacy TRD" provides the old version of the TRD based on the "User Requirements" and can serve as a reference for the new TRD; +"Evaluation Conclusion" provides a summary of the evaluation of the old TRD in the "Legacy TRD" and can serve as a reference for the new TRD; +"Interaction Events" provides some identified interaction events and the interacting participants based on the content of the "User Requirements"; +1. What inputs and outputs are described in the "User Requirements"? +2. How many steps are needed to achieve the inputs and outputs described in the "User Requirements"? Which actors from the "Actor, System, External System" section are involved in each step? What are the inputs and outputs of each step? Where is this output used, for example, as input for which interface or where it is required in the requirements, etc.? +3. Output a complete Technical Requirements Document (TRD)： + 3.1. In the description, use the actor and system names defined in the "Actor, System, External System" section to describe the interactors; + 3.2. The content should include the original text of the requirements from "User Requirements"; + 3.3. In the TRD, each step can involve a maximum of two participants. If there are more than two participants, the step needs to be further split; + 3.4. In the TRD, each step must include detailed descriptions, inputs, outputs, participants, initiator, and the rationale for the step's existence. The rationale should reference the original text to justify it, such as specifying which interface requires the output of this step as parameters or where in the requirements this step is mandated, etc.; + 3.5. In the TRD, if you need to call interfaces of external systems, you must explicitly specify the interface IDs of the external systems you want to call; +""" + +INCREMENTAL_PROMPT = """ +## Actor, System, External System +{use_case_actors} + +## Legacy User Requirements +{legacy_user_requirements} + +## Available External Interfaces +{available_external_interfaces} + +## The TRD of Legacy User Requirements +{legacy_user_requirements_trd} + + +## The Interaction Events of Legacy User Requirements +{legacy_user_requirements_interaction_events} + +## Incremental Requirements +{incremental_user_requirements} + +## Legacy TRD +{previous_version_trd} + +## Evaluation Conclusion +{evaluation_conclusion} + +## Interaction Events +{incremental_user_requirements_interaction_events} + +--- +You are a TRD generator. +The content of "Actor, System, External System" provides an explanation of actors and systems that appear in UML Use Case diagram; +The content of "Available External Interfaces" provides the candidate steps, along with the inputs and outputs of each step; +"Legacy User Requirements" provides the original requirements description handled by other modules for your use; +"The TRD of Legacy User Requirements" is the TRD generated by other modules based on the "Legacy User Requirements" for your use; +"The Interaction Events of Legacy User Requirements" is the interaction events list generated by other modules based on the "Legacy User Requirements" for your use; +"Incremental Requirements" provides the original requirements description that you need to address, any parts not mentioned in this description will be handled by other modules, so do not fabricate requirements; +The requirements in "Legacy User Requirements" combined with the "Incremental Requirements" form a complete set of requirements, therefore, you need to add the TRD portion of the "Incremental Requirements" to "The TRD of Legacy User Requirements", the added content must not conflict with the original content of "The TRD of Legacy User Requirements"; +"Legacy TRD" provides the old version of the TRD you previously wrote based on the "Incremental Requirements" and can serve as a reference for the new TRD; +"Evaluation Conclusion" provides a summary of the evaluation of the old TRD you generated in the "Legacy TRD", and the identified issues can serve as a reference for the new TRD you create; +"Interaction Events" provides some identified interaction events and the interacting participants based on the content of the "Incremental Requirements"; +1. What inputs and outputs are described in the "Incremental Requirements"？ +2. How many steps are needed to achieve the inputs and outputs described in the "Incremental Requirements"? Which actors from the "Actor, System, External System" section are involved in each step? What are the inputs and outputs of each step? Where is this output used, for example, as input for which interface or where it is required in the requirements, etc.? +3. Output a complete Technical Requirements Document (TRD)： + 3.1. In the description, use the actor and system names defined in the "Actor, System, External System" section to describe the interactors; + 3.2. The content should include the original text of the requirements from "User Requirements"; + 3.3. In the TRD, each step can involve a maximum of two participants. If there are more than two participants, the step needs to be further split; + 3.4. In the TRD, each step must include detailed descriptions, inputs, outputs, participants, initiator, and the rationale for the step's existence. The rationale should reference the original text to justify it, such as specifying which interface requires the output of this step as parameters or where in the requirements this step is mandated, etc. + """ diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 7199ec415..4d29d8c6f 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -300,23 +300,27 @@ class WritePRD(Action): user_requirement=to_markdown_code_block(val=user_requirement), extra_info=to_markdown_code_block(val=extra_info), ) - req = Document(content=content) - if not legacy_prd_filename: - node = await self._new_prd(requirement=req.content) - new_prd = Document(content=node.instruct_content.model_dump_json()) - else: - content = await aread(filename=legacy_prd_filename) - old_prd = Document(content=content) - new_prd = await self._merge(req=req, related_doc=old_prd) + async with DocsReporter(enable_llm_stream=True) as reporter: + await reporter.async_report({"type": "prd"}, "meta") + req = Document(content=content) + if not legacy_prd_filename: + node = await self._new_prd(requirement=req.content) + new_prd = Document(content=node.instruct_content.model_dump_json()) + else: + content = await aread(filename=legacy_prd_filename) + old_prd = Document(content=content) + new_prd = await self._merge(req=req, related_doc=old_prd) - if not output_pathname: - output_pathname = DEFAULT_WORKSPACE_ROOT / "docs" / "prd.json" - output_pathname.mkdir(parents=True, exist_ok=True) - elif not Path(output_pathname).is_absolute(): - output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname - output_pathname = Path(output_pathname) - await awrite(filename=output_pathname, data=new_prd.content) - competitive_analysis_filename = output_pathname.parent / f"{output_pathname.stem}-competitive-analysis" - await self._save_competitive_analysis(prd_doc=new_prd, output_filename=Path(competitive_analysis_filename)) - await save_json_to_markdown(content=new_prd.content, output_filename=output_pathname.with_suffix(".md")) + if not output_pathname: + output_pathname = DEFAULT_WORKSPACE_ROOT / "docs" / "prd.json" + output_pathname.mkdir(parents=True, exist_ok=True) + elif not Path(output_pathname).is_absolute(): + output_pathname = DEFAULT_WORKSPACE_ROOT / output_pathname + output_pathname = Path(output_pathname) + await awrite(filename=output_pathname, data=new_prd.content) + competitive_analysis_filename = output_pathname.parent / f"{output_pathname.stem}-competitive-analysis" + await self._save_competitive_analysis(prd_doc=new_prd, output_filename=Path(competitive_analysis_filename)) + md_output_filename = output_pathname.with_suffix(".md") + await save_json_to_markdown(content=new_prd.content, output_filename=md_output_filename) + await reporter.async_report(md_output_filename, "path") return f'PRD filename: "{str(output_pathname)}"' diff --git a/metagpt/const.py b/metagpt/const.py index 6e823d56c..17ce9210a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -20,12 +20,6 @@ import metagpt def get_metagpt_package_root(): """Get the root directory of the installed package.""" package_root = Path(metagpt.__file__).parent.parent - for i in (".git", ".project_root", ".gitignore"): - if (package_root / i).exists(): - break - else: - package_root = Path.cwd() - logger.info(f"Package root set to {str(package_root)}") return package_root @@ -40,6 +34,12 @@ def get_metagpt_root(): else: # Fallback to package root if no environment variable is set project_root = get_metagpt_package_root() + for i in (".git", ".project_root", ".gitignore"): + if (project_root / i).exists(): + break + else: + project_root = Path.cwd() + return project_root @@ -149,3 +149,6 @@ METAGPT_REPORTER_DEFAULT_URL = os.environ.get("METAGPT_REPORTER_URL", "") # Metadata defines AGENT = "agent" + +# SWE agent +SWE_SETUP_PATH = get_metagpt_package_root() / "metagpt/tools/swe_agent_commands/setup_default.sh" diff --git a/metagpt/ext/cr/__init__.py b/metagpt/ext/cr/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/metagpt/ext/cr/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/metagpt/ext/cr/actions/code_review.py b/metagpt/ext/cr/actions/code_review.py new file mode 100644 index 000000000..5586567fa --- /dev/null +++ b/metagpt/ext/cr/actions/code_review.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : + +import json +import re + +from unidiff import PatchedFile, PatchSet + +from metagpt.actions.action import Action +from metagpt.ext.cr.utils.cleaner import ( + add_line_num_on_patch, + get_code_block_from_patch, + rm_patch_useless_part, +) +from metagpt.ext.cr.utils.schema import Point +from metagpt.logs import logger +from metagpt.utils.common import parse_json_code_block + +CODE_REVIEW_PROMPT_TEMPLATE = """ +NOTICE +With the given pull-request(PR) Patch, and referenced Points(Code Standards), you should compare each point with the code one-by-one. + +The Patch code has added line number at the first character each line for reading, but the review should focus on new added code inside the `Patch` (lines starting with line number and '+'). +Each point is start with a line number and follows with the point description. + +## Patch +``` +{patch} +``` + +## Points +{points} + +## Output Format +```json +[ + {{ + "commented_file": "The file path which you give a comment from the patch", + "comment": "The chinese comment of code which do not meet point description and give modify suggestions", + "code_start_line": "the code start line number like `10` in the Patch of current comment,", + "code_end_line": "the code end line number like `15` in the Patch of current comment", + "point_id": "The point id which the `comment` references to" + }} +] +``` + +CodeReview guidelines: +- Generate code `comment` that do not meet the point description. +- Each `comment` should be restricted inside the `commented_file` +- Try to provide diverse and insightful comments across different `commented_file`. +- Don't suggest to add docstring unless it's necessary indeed. +- If the same code error occurs multiple times, it cannot be omitted, and all places need to be identified.But Don't duplicate at the same place with the same comment! +- Every line of code in the patch needs to be carefully checked, and laziness cannot be omitted. It is necessary to find out all the places. + +Just print the PR Patch comments in json format like **Output Format**. +""" + +CODE_REVIEW_COMFIRM_SYSTEM_PROMPT = """ +You are a professional engineer with Java stack, and good at code review comment result judgement. +""" + +CODE_REVIEW_COMFIRM_TEMPLATE = """ +## Code +``` +{code} +``` +## Code Review Comments +{comment} + +## Description of Defects +{desc} + +## Reference Example for Judgment +{example} + +## Your Task: +1. First, check if the code meets the requirements and does not violate any defects. If it meets the requirements and does not violate any defects, print `False` and do not proceed with further judgment. +2. If the check in step 1 does not print `False`, proceed to further judgment. Based on the "Reference Example for Judgment" provided, determine if the "Code" and "Code Review Comments" match. If they match, print `True`; otherwise, print `False`. + +Note: Your output should only be `True` or `False` without any explanations. +""" + + +class CodeReview(Action): + name: str = "CodeReview" + + def format_comments(self, comments: list[dict], points: list[Point], patch: PatchSet): + new_comments = [] + logger.debug(f"original comments: {comments}") + for cmt in comments: + for p in points: + if int(cmt.get("point_id", -1)) == p.id: + code_start_line = cmt.get("code_start_line") + code_end_line = cmt.get("code_end_line") + code = get_code_block_from_patch(patch, code_start_line, code_end_line) + + new_comments.append( + { + "commented_file": cmt.get("commented_file"), + "code": code, + "code_start_line": code_start_line, + "code_end_line": code_end_line, + "comment": cmt.get("comment"), + "point_id": p.id, + "point": p.text, + "point_detail": p.detail, + } + ) + break + + logger.debug(f"new_comments: {new_comments}") + return new_comments + + async def confirm_comments(self, patch: PatchSet, comments: list[dict], points: list[Point]) -> list[dict]: + points_dict = {point.id: point for point in points} + new_comments = [] + for cmt in comments: + point = points_dict[cmt.get("point_id")] + + code_start_line = cmt.get("code_start_line") + code_end_line = cmt.get("code_end_line") + # 如果代码位置为空的话，那么就将这条记录丢弃掉 + if not code_start_line or not code_end_line: + logger.info("False") + continue + + # 代码增加上下文，提升confirm的准确率 + code = get_code_block_from_patch(patch, str(max(1, int(code_start_line) - 3)), str(int(code_end_line) + 3)) + pattern = r"^[ \t\n\r(){}[\];,]*$" + if re.match(pattern, code): + code = get_code_block_from_patch( + patch, str(max(1, int(code_start_line) - 5)), str(int(code_end_line) + 5) + ) + prompt = CODE_REVIEW_COMFIRM_TEMPLATE.format( + code=code, + comment=cmt.get("comment"), + desc=point.text, + example=point.yes_example + "\n" + point.no_example, + ) + resp = await self.llm.aask(prompt, system_msgs=[CODE_REVIEW_COMFIRM_SYSTEM_PROMPT]) + if "True" in resp or "true" in resp: + new_comments.append(cmt) + logger.info(f"original comments num: {len(comments)}, confirmed comments num: {len(new_comments)}") + return new_comments + + async def cr_by_full_points(self, patch: PatchSet, points: list[Point]): + comments = [] + points_str = "\n".join([f"{p.id} {p.text}" for p in points]) + for patched_file in patch: + if patched_file.path.endswith(".py"): + points_str = "\n".join([f"{p.id} {p.text}" for p in points if p.language == "Python"]) + elif patched_file.path.endswith(".java"): + points_str = "\n".join([f"{p.id} {p.text}" for p in points if p.language == "Java"]) + else: + continue + if len(str(patched_file).splitlines()) >= 50: + cr_by_segment_points_comments = await self.cr_by_segment_points( + patched_file=patched_file, points=points + ) + comments += cr_by_segment_points_comments + continue + prompt = CODE_REVIEW_PROMPT_TEMPLATE.format(patch=str(patched_file), points=points_str) + resp = await self.llm.aask(prompt) + json_str = parse_json_code_block(resp)[0] + comments += json.loads(json_str) + + return comments + + async def cr_by_segment_points(self, patched_file: PatchedFile, points: list[Point]): + comments = [] + group_points = [points[i : i + 3] for i in range(0, len(points), 3)] + for group_point in group_points: + points_str = "\n".join([f"{p.id} {p.text}" for p in group_point]) + prompt = CODE_REVIEW_PROMPT_TEMPLATE.format(patch=str(patched_file), points=points_str) + resp = await self.llm.aask(prompt) + json_str = parse_json_code_block(resp)[0] + comments_batch = json.loads(json_str) + comments += comments_batch + + return comments + + async def run(self, patch: PatchSet, points: list[Point]): + patch: PatchSet = rm_patch_useless_part(patch) + patch: PatchSet = add_line_num_on_patch(patch) + + result = [] + comments = await self.cr_by_full_points(patch=patch, points=points) + if len(comments) != 0: + comments = self.format_comments(comments, points, patch) + comments = await self.confirm_comments(patch=patch, comments=comments, points=points) + for comment in comments: + if comment["code"]: + if not (comment["code"].startswith("-") or comment["code"].isspace()): + result.append(comment) + return result diff --git a/metagpt/ext/cr/actions/modify_code.py b/metagpt/ext/cr/actions/modify_code.py new file mode 100644 index 000000000..33a368463 --- /dev/null +++ b/metagpt/ext/cr/actions/modify_code.py @@ -0,0 +1,112 @@ +import datetime +import itertools +import re +from pathlib import Path +from typing import Optional + +from unidiff import PatchSet + +from metagpt.actions.action import Action +from metagpt.const import DEFAULT_WORKSPACE_ROOT +from metagpt.ext.cr.utils.cleaner import ( + add_line_num_on_patch, + get_code_block_from_patch, + rm_patch_useless_part, +) +from metagpt.utils.common import CodeParser +from metagpt.utils.report import EditorReporter + +SYSTEM_MSGS_PROMPT = """ +You're an adaptive software developer who excels at refining code based on user inputs. You're proficient in creating Git patches to represent code modifications. +""" + +MODIFY_CODE_PROMPT = """ +NOTICE +With the given pull-request(PR) Patch, and referenced Comments(Code Standards), you should modify the code according the Comments. + +The Patch code has added line no at the first character each line for reading, but the modification should focus on new added code inside the `Patch` (lines starting with line no and '+'). + +## Patch +``` +{patch} +``` + +## Comments +{comments} + +## Output Format + + + +Code Modification guidelines: +- Look at `point_detail`, modify the code by `point_detail`, use `code_start_line` and `code_end_line` to locate the problematic code, fix the problematic code by `point_detail` in Comments.Strictly,must handle the fix plan given by `point_detail` in every comment. +- Create a patch that satifies the git patch standard and your fixes need to be marked with '+' and '-',but notice:don't change the hunk header! +- Do not print line no in the new patch code. + +Just print the Patch in the format like **Output Format**. +""" + + +class ModifyCode(Action): + name: str = "Modify Code" + pr: str + + async def run(self, patch: PatchSet, comments: list[dict], output_dir: Optional[str] = None) -> str: + patch: PatchSet = rm_patch_useless_part(patch) + patch: PatchSet = add_line_num_on_patch(patch) + + # + for comment in comments: + code_start_line = comment.get("code_start_line") + code_end_line = comment.get("code_end_line") + # 如果代码位置为空的话，那么就将这条记录丢弃掉 + if code_start_line and code_end_line: + code = get_code_block_from_patch( + patch, str(max(1, int(code_start_line) - 3)), str(int(code_end_line) + 3) + ) + pattern = r"^[ \t\n\r(){}[\];,]*$" + if re.match(pattern, code): + code = get_code_block_from_patch( + patch, str(max(1, int(code_start_line) - 5)), str(int(code_end_line) + 5) + ) + # 代码增加上下文，提升代码修复的准确率 + comment["code"] = code + # 去掉CR时LLM给的comment的影响，应该使用既定的修复方案 + comment.pop("comment") + + # 按照 commented_file 字段进行分组 + comments.sort(key=lambda x: x["commented_file"]) + grouped_comments = { + key: list(group) for key, group in itertools.groupby(comments, key=lambda x: x["commented_file"]) + } + resp = None + for patched_file in patch: + patch_target_file_name = str(patched_file.target_file).split("/", maxsplit=1)[-1] + if patch_target_file_name not in grouped_comments: + continue + comments_prompt = "" + index = 1 + for grouped_comment in grouped_comments[patch_target_file_name]: + comments_prompt += f""" + + {grouped_comment} + \n + """ + prompt = MODIFY_CODE_PROMPT.format(patch=patched_file, comments=comments_prompt) + output_dir = ( + Path(output_dir) + if output_dir + else DEFAULT_WORKSPACE_ROOT / "modify_code" / str(datetime.date.today()) / self.pr + ) + patch_file = output_dir / f"{patch_target_file_name}.patch" + patch_file.parent.mkdir(exist_ok=True, parents=True) + async with EditorReporter(enable_llm_stream=True) as reporter: + await reporter.async_report( + {"type": "Patch", "src_path": str(patch_file), "filename": patch_file.name}, "meta" + ) + resp = await self.llm.aask(msg=prompt, system_msgs=[SYSTEM_MSGS_PROMPT]) + resp = CodeParser.parse_code(resp, "diff") + with open(patch_file, "w", encoding="utf-8") as file: + file.writelines(resp) + await reporter.async_report(patch_file) + return resp diff --git a/metagpt/ext/cr/points.json b/metagpt/ext/cr/points.json new file mode 100644 index 000000000..b0497cb7b --- /dev/null +++ b/metagpt/ext/cr/points.json @@ -0,0 +1,664 @@ +[ + { + "id": 1, + "text": "避免未使用的临时变量", + "language": "java", + "detail": "缺陷类型：避免未使用的临时变量；对应Fixer：UnusedLocalVariableFixer；修复方案：删除未使用的临时变量", + "yes_example": "### 被判定为\"避免未使用的临时变量\"的例子\n<例子1>\npublic String initCreationForm(Map model) {\n\t\tOwner owner = new Owner();\n\t\tmodel.put(\"owner\", owner);\n\t\tint unusedVar = 10;\n\t\treturn VIEWS_OWNER_CREATE_OR_UPDATE_FORM;\n\t}\n上述代码中unusedVar变量未被使用，所以这个被判定为\"避免未使用的临时变量\"\n\n<例子2>\nint unusedVariable = 10;\nSystem.out.println(\"Hello, World!\");\n这段代码的变量\"unusedVariable\"未被使用或者引用，所以这个不能判定为\"避免未使用的临时变量\"\n", + "no_example": "### 不能被判定为\"避免未使用的临时变量\"的例子\n<例子1>\npublic void setTransientVariablesLocal(Map transientVariables) {\nthrow new UnsupportedOperationException(\"No execution active, no variables can be set\");\n}\n这段代码的\"transientVariables\"是函数参数而不是临时变量，虽然transientVariables没有被使用或者引用，但是这个也不能判定为\"避免未使用的临时变量\"\n\n\n<例子2>\npublic class TriggerCmd extends NeedsActiveExecutionCmd {\n protected Map transientVariables;\n public TriggerCmd(Map transientVariables) {\n this.transientVariables = transientVariables;\n }\n}\n上述代码中transientVariables不属于临时变量，它是类属性，且它在构造函数中被使用，所以这个不能被判定为\"避免未使用的临时变量\"\n" + }, + { + "id": 2, + "text": "不要使用 System.out.println 去打印", + "language": "java", + "detail": "缺陷类型：不要使用 System.out.println 去打印；对应Fixer：SystemPrintlnFixer；修复方案：注释System.out.println代码", + "yes_example": "### 被判定为\"不要使用 System.out.println 去打印\"的例子\n<例子1>\nSystem.out.println(\"Initializing new owner form.\");\n上述代码使用了\"System.out.println\"进行打印，所以这个被判定为\"不要使用 System.out.println 去打印\"\n", + "no_example": "### 不能被判定为\"不要使用 System.out.println 去打印\"的例子\n<例子1>\nthrow new IllegalStateException(\"There is no authenticated user, we need a user authenticated to find tasks\");\n上述代码是抛出异常的代码，没有使用\"System.out.print\"，所以这个不能被判定为\"不要使用 System.out.println 去打印\"\n" + }, + { + "id": 3, + "text": "避免函数中未使用的形参", + "language": "java", + "detail": "缺陷类型：避免函数中未使用的形参；修复方案：忽略", + "yes_example": "### 被判定为\"避免函数中未使用的形参\"的例子\n<例子1>\npublic void setTransientVariablesLocal(Map transientVariables) {\n throw new UnsupportedOperationException(\"No execution active, no variables can be set\");\n}这段代码中的形参\"transientVariables\"未在函数体内出现，所以这个被判定为\"避免函数中未使用的形参\"\n\n\n<例子2>\nprotected void modifyFetchPersistencePackageRequest(PersistencePackageRequest ppr, Map pathVars) {}\n这段代码中的形参\"ppr\"和\"pathVars\"未在函数体内出现，所以这个被判定为\"避免函数中未使用的形参\"\n", + "no_example": "### 不能被判定为\"避免函数中未使用的形参\"的例子\n<例子1>\npublic String processFindForm(@RequestParam(value = \"pageNo\", defaultValue = \"1\") int pageNo) {\n\tlastName = owner.getLastName();\n\treturn addPaginationModel(pageNo, paginationModel, lastName, ownersResults);\n}上述代码中的形参\"pageNo\"在当前函数'processFindForm'内被'return addPaginationModel(pageNo, paginationModel, lastName, ownersResults);'这一句被使用，虽然pageNo没有被用于逻辑计算，但作为了函数调用其他函数的参数使用了，所以这个不能被判定为\"避免函数中未使用的形参\"\n" + }, + { + "id": 4, + "text": "if语句块不能为空", + "language": "java", + "detail": "缺陷类型：if 语句块不能为空；对应Fixer：EmptyIfStmtFixer；修复方案：删除if语句块 或 适当的逻辑处理 或 注释说明为何为空", + "yes_example": "### 被判定为\"if语句块不能为空\"的例子\n<例子1>\npublic void emptyIfStatement() {\n\tif (getSpecialties().isEmpty()) {\n\t}\n}这段代码中的if语句块内容是空的，所以这个被判定为\"if语句块不能为空\"\n\n\n<例子2>\npublic void judgePersion() {\n\tif (persion != null) {\n\t\t// judge persion if not null\n\t}\n}\n这段代码中的if语句块虽然有内容，但是\"// judge persion if not null\"只是代码注释，if语句块内并没有实际的逻辑代码，所以这个被判定为\"if语句块不能为空\"\n", + "no_example": "### 不能被判定为\"if语句块不能为空\"的例子\n<例子1>\npublic void judgePersion() {\n\tif (persion != null) {\n\t\treturn 0;\n\t}\n}这段代码中的if语句块里有内容，且里面有非注释代码的逻辑代码\"return 0;\"，所以这个不能被判定为\"if语句块不能为空\"\n" + }, + { + "id": 5, + "text": "循环体不能为空", + "language": "java", + "detail": "缺陷类型：循环体不能为空；对应Fixer：EmptyStatementNotInLoopFixer；修复方案：删除对应while、for、foreach 循环体 或 添加适当的逻辑处理或者注释说明为何为空", + "yes_example": "### 被判定为\"循环体不能为空\"的例子\n<例子1>\npublic void emptyLoopBody() {\n\tfor (Specialty specialty : getSpecialties()) {\n\t}\n}这段代码中的for循环体的内容是空的，所以这个被判定为\"循环体不能为空\"\n\n\n<例子2>\npublic void emptyLoopBody() {\n\twhile (True) {\n\t\t// this is a code example\n\t}\n}这段代码中的while循环体的内容虽然不是空的，但内容只是代码注释，无逻辑内容，所以这个被判定为\"循环体不能为空\"\n\n\n<例子3>\npublic void emptyLoopBody() {\n\twhile (True) {\n\t\t\n\t}\n}这段代码中的while循环体内容是空的，所以这个被判定为\"循环体不能为空\"\n", + "no_example": "### 不能被判定为\"循环体不能为空\"的例子\n<例子1>\npublic void emptyLoopBody() {\n\tfor (Specialty specialty : getSpecialties()) {\n\t\ta = 1;\n\t\tif (a == 1) {\n\t\t\tretrun a;\n\t\t}\n\t}\n}上述代码的for循环体的内容不为空，且内容不全是代码注释，所以这个不能被判定为\"循环体不能为空\"\n" + }, + { + "id": 6, + "text": "避免使用 printStackTrace()，应该使用日志的方式去记录", + "language": "java", + "detail": "缺陷类型：避免使用 printStackTrace()，应该使 用日志的方式去记录；修复方案：用日志的方式去记录", + "yes_example": "### 被判定为\"避免使用 printStackTrace()，应该使用日志的方式去记录\"的例子\n<例子1>\npublic void usePrintStackTrace() {\n\ttry {\n\t\tthrow new Exception(\"Fake exception\");\n\t} catch (Exception e) {\n\t\te.printStackTrace();\n\t}\n}这段代码中的catch语句中使用了printStackTrace()，所以这个被判定为\"避免使用 printStackTrace()，应该使用日志的方式去记录\"\n", + "no_example": "### 不能被判定为\"避免使用 printStackTrace()，应该使用日志的方式去记录\"的例子\n<例子1>\npublic void usePrintStackTrace() {\n\ttry {\n\t\tthrow new Exception(\"Fake exception\");\n\t} catch (Exception e) {\n\t\tlogging.info(\"info\");\n\t}\n}这段代码的catch语句中使用的是日志记录的方式，所以这个不能被判定为\"避免使用 printStackTrace()，应该使用日志的方式去记录\"\n" + }, + { + "id": 7, + "text": "catch 语句块不能为空", + "language": "java", + "detail": "缺陷类型：catch 语句块不能为空；对应Fixer：EmptyCatchBlockFixer；修复方案：在catch里面添加注释", + "yes_example": "### 被判定为\"catch语句块不能为空\"的例子\n<例子1>\ntry {\n int[] array = new int[5];\n int number = array[10];\n} catch (ArrayIndexOutOfBoundsException e) {\n \n}\n这段代码中的catch语句中没有内容，所以这个被判定为\"catch语句块不能为空\"\n\n\n<例子2>\ntry {\n String str = null;\n str.length();\n} catch (NullPointerException e) {\n \n}这段代码中的catch语句中没有内容，所以这个被判定为\"catch语句块不能为空\"\n\n\n<例子3>\npublic class EmptyCatchExample {\n public static void main(String[] args) {\n try {\n // 尝试除以零引发异常\n int result = 10 / 0;\n } catch (ArithmeticException e) {\n \n }\n }\n}这段代码中的catch语句中没有内容，所以这个被判定为\"catch语句块不能为空\"\n\n<例子4>\ntry {\n FileReader file = new FileReader(\"nonexistentfile.txt\");\n} catch (FileNotFoundException e) {\n \n}这段代码中的catch语句中没有内容，所以这个被判定为\"catch语句块不能为空\"\n\n<例子5>\ntry {\n Object obj = \"string\";\n Integer num = (Integer) obj;\n} catch (ClassCastException e) {\n\t\n}这段代码中的catch语句中没有内容，所以这个被判定为\"catch语句块不能为空\"\n", + "no_example": "### 不能被判定为\"catch语句块不能为空\"的例子\n<例子1>\npersionNum = 1\ntry {\n\treturn True;\n} catch (Exception e) {\n\t// 如果人数为1则返回false\n\tif (persionNum == 1){\n\t\treturn False;\n\t}\n}这段代码的catch语句中不为空，所以不能把这个被判定为\"catch语句块不能为空\"\n\n\n<例子2>\ntry {\n\tthrow new Exception(\"Fake exception\");\n} catch (Exception e) {\n\te.printStackTrace();\n}这段代码的catch语句中虽然只有\"e.printStackTrace();\"但确实不为空，所以不能把这个被判定为\"catch语句块不能为空\"\n" + }, + { + "id": 8, + "text": "避免不必要的永真/永假判断", + "language": "java", + "detail": "缺陷类型：避免不必要的永真/永假判断；对应Fixer：UnconditionalIfStatement Fixer；修复方案：删除永真/永假判断逻辑", + "yes_example": "### 被判定为\"避免不必要的永真/永假判断\"的例子\n<例子1>\npublic void someMethod() {\n\twhile (true) {\n\t}\n}这段代码中的\"while (true)\"是一个使用true做判断条件，但是没有循环结束标记，所以这个被判定为\"避免不必要的永真/永假判断\"\n\n\n<例子2>\nif (true) {\n\tSystem.out.println(\"This is always true\");\n}这段代码中的\"if (true)\"是一个使用true条件做条件，但是没有循环结束标记，所以这个被判定为\"避免不必要的永真/永假判断\"\n\n\n<例子3>\na = 1;\nwhile(a > 0){\n\ta = a + 1\n}这段代码初始化a=1，是大于0的，while循环体的逻辑是每次加1，那么判断条件a > 0会永远是真的，不会退出循环，所以这个被判定为\"避免不必要的永真/永假判断\"\n<例子3>", + "no_example": "### 不能被判定为\"避免不必要的永真/永假判断\"的例子\n<例子1>\na = 0;\nwhile (a < 5) {\n\ta = a + 1;\n}这段代码中的a<5是一个判断，当执行了5次while语句中的逻辑a=a+1之后，a会满足a < 5，就会退出循环，所以这个能被判定为\"避免不必要的永真/永假判断\"\n" + }, + { + "id": 9, + "text": "switch 中 default 必须放在最后", + "language": "java", + "detail": "缺陷类型：switch 中 default 必须放在最后；对应Fixer：DefaultLabelNotLastInSwitchStmtFixer；修复方案：switch 中 default 放在最后", + "yes_example": "### 被判定为\"switch 中 default 必须放在最后\"的例子\n<例子1>\nswitch (number) {\n\tdefault:\n\t\tSystem.out.println(\"This is the default block, which is incorrectly placed here.\");\n\t\tbreak;\n\tcase 1:\n\t\tSystem.out.println(\"Number one\");\n\t\tbreak;\n\tcase 2:\n\t\tSystem.out.println(\"Number two\");\n\t\tbreak;\n}这段代码是一个switch语句，但是里面的default没有放在最后，所以这个被判定为\"switch 中 default 必须放在最后\"\n", + "no_example": "### 不能被判定为\"switch 中 default 必须放在最后\"的例子\n<例子1>\nswitch (number) {\ncase 3:\n\tSystem.out.println(\"Number one\");\n\tbreak;\ncase 4:\n\tSystem.out.println(\"Number two\");\n\tbreak;\ndefault:\n\tSystem.out.println(\"This is the default block, which is incorrectly placed here.\");\n\tbreak;\n}这段代码是一个switch语句且里面的default放在了最后，所以这个不能被判定为\"switch 中 default 必须放在最后\"\n" + }, + { + "id": 10, + "text": "未使用equals()函数对 String 作比较", + "language": "java", + "detail": "缺陷类型：未使用equals()函数对 String 作比较；对应Fixer：UnSynStaticDateFormatter Fixer；修复方案：使用equals()函数对 String 作比较", + "yes_example": "### 被判定为\"未使用equals()函数对 String 作比较\"的例子\n<例子1>\nif (existingPet != null && existingPet.getName() == petName) {\n\tresult.rejectValue(\"name\", \"duplicate\", \"already exists\");\n}这段代码中所涉及的existingPet.getName()和petName均是字符串，但是在if语句里做比较的时候使用了==而没有使用equals()对string做比较，所以这个被判定为\"未使用equals()函数对 String 作比较\"\n\n\n<例子2>\nString isOk = \"ok\";\nif (\"ok\" == isOk) {\n\tresult.rejectValue(\"name\", \"duplicate\", \"already exists\");\n}这段代码中的isOk是个字符串，但在if判断中与\"ok\"比较的时候使用的是==，未使用equals()对string做比较，所以这个被判定为\"未使用equals()函数对 String 作比较\"\n", + "no_example": "### 不能被判定为\"未使用equals()函数对 String 作比较\"的例子\n<例子1>\nif (PROPERTY_VALUE_YES.equalsIgnoreCase(readWriteReqNode))\n formProperty.setRequired(true);\n这段代码中的PROPERTY_VALUE_YES和readWriteReqNode均是字符串，在if语句里比较PROPERTY_VALUE_YES和readWriteReqNode的使用的是equalsIgnoreCase（字符串比较忽略大小写），所以equalsIgnoreCase也是符合使用equals()函数对 String 作比较的，所以这个不能被判定为\"未使用equals()函数对 String 作比较\"\n\n\n<例子2>\nString isOk = \"ok\";\nif (\"ok\".equals(isOk)) {\n\tresult.rejectValue(\"name\", \"duplicate\", \"already exists\");\n}这段代码中的isOk是个字符串，在if判断中与\"ok\"比较的时候使用的是equals()对string做比较，所以这个不能被判定为\"未使用equals()函数对 String 作比较\"\n" + }, + { + "id": 11, + "text": "禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象", + "language": "java", + "detail": "缺陷类型：禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象 输出异常；对应Fixer：ConcatExceptionFixer；修复方案：使用占位符传递异常对象", + "yes_example": "### 被判定为\"禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象\"的例子\n<例子1>\ntry {\n listenersNode = objectMapper.readTree(listenersNode.asText());\n} catch (Exception e) {\n LOGGER.info(\"Listeners node can not be read\", e);\n}这段代码中日志输出内容内容是直接使用字符串\"Listeners node can not be read\"拼接，日志输出异常时，应使用占位符输出异常信息，而不是直接使用字符串拼接，所以这个被判定为\"禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象\"\n", + "no_example": "### 不能被判定为\"禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象\"的例子\n<例子1>\nPersion persion = persionService.getPersion(1);\nif (persion == null){\n\tLOGGER.error(PERSION_NOT_EXIT);\n}这段代码中的PERSION_NOT_EXIT是一个用户自定义的异常常量，代表persion不存在，没有直接使用字符串\"persion not exit\"拼接，所以这个不能被判定为\"禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象\"\n<例子1>\n\n<例子2>\ntry {\n a = a + 1;\n} catch (Exception e) {\n Persion persion = persionService.getPersion(1);\n LOGGER.info(persion);\n}这段代码中输出日志没有直接使用字符串拼接，而是使用的Persion对象输出，所以这个不能被判定为\"禁止在日志中直接使用字符串输出异常，请使用占位符传递异常对象\"\n" + }, + { + "id": 12, + "text": "finally 语句块不能为空", + "language": "java", + "detail": "缺陷类型：finally 语句块不能为空；对应Fixer：EmptyFinallyBlockFixer；修复方案：删除空 finally 语句块", + "yes_example": "### 被判定为\"finally 语句块不能为空\"的例子\n<例子1>\ntry {\n\tPersion persion = persionService.getPersion(1);\n\treturn persion;\n} finally {\n\t\n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n\n\n<例子2>\ntry {\n\tSystem.out.println(\"Inside try block\");\n} finally {\n\t// 空的finally块，没有任何语句，这是一个缺陷\n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n\n\n<例子3>\ntry {\n int result = 10 / 0;\n} catch (ArithmeticException e) {\n e.printStackTrace();\n} finally {\n \n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n\n\n<例子4>\ntry {\n String str = null;\n System.out.println(str.length());\n} catch (NullPointerException e) {\n e.printStackTrace();\n} finally {\n \n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n\n\n<例子5>\ntry {\n int[] array = new int[5];\n int number = array[10];\n} catch (ArrayIndexOutOfBoundsException e) {\n e.printStackTrace();\n} finally {\n // 只有注释的 finally 语句块\n // 这是一个空的 finally 块\n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n\n\n<例子6>\ntry {\n FileReader file = new FileReader(\"nonexistentfile.txt\");\n} catch (FileNotFoundException e) {\n e.printStackTrace();\n} finally {\n // 只有空行的 finally 语句块\n \n}这段代码中的finally语句块内没有内容，所以这个被判定为\"finally 语句块不能为空\"\n", + "no_example": "### 不能被判定为\"finally 语句块不能为空\"的例子\n<例子1>\npublic void getPersion() {\n\ttry {\n\t\tPersion persion = persionService.getPersion(1);\n\t\tif (persion != null){ \n\t\t\treturn persion;\n\t\t}\n\t} finally {\n\t\treturn null;\n\t}\n}这段代码中的finally语句块中有非注释意外的内容\"return null;\"，所以这个不能被判定为\"finally 语句块不能为空\"\n" + }, + { + "id": 13, + "text": "try 语句块不能为空", + "language": "java", + "detail": "缺陷类型：try 语句块不能为空；对应Fixer：EmptyTryBlockFixer；修复方案：删除整个 try 语句", + "yes_example": "### 被判定为\"try 语句块不能为空\"的例子\n<例子1>\npublic void getPersion() {\n\ttry {\n\n\t}\n\treturn null;\n}这段代码中的try语句块内没有内容，所以这个被判定为\"try 语句块不能为空\"\n\n\n<例子2>\npublic void demoFinallyBlock() {\n\ttry {\n\n\t} finally {\n\t\treturn null;\n\t}\n}这段代码中的try语句块内没有内容，所以这个被判定为\"try 语句块不能为空\"\n\n\n<例子3>\ntry {\n \n} catch (Exception e) {\n e.printStackTrace();\n}这段代码中的try语句块内没有内容，所以这个被判定为\"try 语句块不能为空\"\n\n\n<例子4>\ntry {\n // 只有注释的 try 语句块\n\t\n} catch (Exception e) {\n e.printStackTrace();\n}这段代码中的try语句块内只有注释和空行，也可以认定为这种情况是try语句块内没有内容，所以这个被判定为\"try 语句块不能为空\"\n", + "no_example": "### 不能被判定为\"try 语句块不能为空\"的例子\n<例子1>\ntry {\n\ta = a + 1;\n} catch (Exception e) {\n\te.printStackTrace();\n}\n这段代码中的try语句块中有非注释意外的内容\"return null;\"，所以这个不能被判定为\"try 语句块不能为空\"\n" + }, + { + "id": 14, + "text": "避免对象进行不必要的 NULL或者null 检查", + "language": "java", + "detail": "缺陷类型：避免对象进行不必要的 NULL或者null 检查；对应Fixer：LogicalOpNpeFixer；修复方案：删除对对象不必要的 NULL 检查的逻辑", + "yes_example": "### 被判定为\"避免对象进行不必要的 NULL或者null 检查\"的例子\n<例子1>\na = \"dog\";\nif (a != null){\n\treturn a;\n}这段代码中的对象a已经是确定的值\"dog\"，所以if条件句的判断\"a != null\"是不必要的，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n\n\n<例子2>\nif (authenticatedUserId != null && !authenticatedUserId.isEmpty() && userGroupManager!=null){\n\treturn authenticatedUserId;\n}这段代码中的\"authenticatedUserId != null\"和\"!authenticatedUserId.isEmpty()\"都是对\"authenticatedUserId\"的空判断，重复了，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n\n\n<例子3>\nList list = new ArrayList<>();\nif (list != null) {\n list.add(1);\n}这段代码中的list已经被初始化，不需要进行 null 检查，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n\n\n<例子4>\nif (this.type != null && this.type.getName() != null) {\n\tSystem.out.println(\"Type name is not null\");\n}这段代码中的对象type已经检查过非null，再次检查getName()是否为null是不必要的，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n\n\n\n<例子5>\nif (\"dog\".equals(null)){\n\treturn a;\n}这段代码中的\"dog\"是个确定的字符串，不需要进行null 检查，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n\n\n<例子6>\nInteger num = 10;\nif (num != null) {\n System.out.println(num);\n}这段代码中的num 已经被初始化，不需要进行 null 检查，所以这个被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n", + "no_example": "### 不能被判定为\"避免对象进行不必要的 NULL或者null 检查\"的例子\n<例子1>\nCat cat = catService.get(1);\nif (cat != null){\n\tretrun cat;\n}这段代码中的对象\"cat\"是通过service获取到的，不确定是否为空，所以if条件句的判断的\"cat != null\"是必要的，所以这个不能被判定为\"避免对象进行不必要的 NULL或者null 检查\"\n" + }, + { + "id": 15, + "text": "避免 finally 块中出现 return", + "language": "java", + "detail": "缺陷类型：避免 finally 块中出现 return；修复方案：无需修复", + "yes_example": "### 被判定为\"避免 finally 块中出现 return\"的例子\n<例子1>\npublic void getPersion() {\n\ttry {\n\t\tPersion persion = persionService.getPersion(1);\n\t\tif (persion != null){ \n\t\t\treturn persion;\n\t\t}\n\t} finally {\n\t\treturn null;\n\t}\n}这段代码中的finally语句块内容包含\"return\"，所以这个被判定为\"避免 finally 块中出现 return\"\n", + "no_example": "### 不能被判定为\"避免 finally 块中出现 return\"的例子\n<例子1>\npublic void getPersion() {\n\ttry {\n\t\tPersion persion = persionService.getPersion(1);\n\t\tif (persion != null){ \n\t\t\treturn persion;\n\t\t}\n\t} finally {\n\t\tLOGGER.info(PERSION_NOT_EXIT);\n\t}\n}这段代码中的finally语句块中内容不包含\"return\"，所以这个不能被判定为\"避免 finally 块中出现 return\"\n" + }, + { + "id": 16, + "text": "避免空的 static 初始化", + "language": "java", + "detail": "缺陷类型：避免空的 static 初始化；对应Fixer：EmptyInitializerFixer；修复方案：删除整个空初始化块", + "yes_example": "### 被判定为\"避免空的 static 初始化\"的例子\n<例子1>\npublic class PetValidator implements Validator {\n\tstatic {\n\n\t}\n}这段代码中的static语句块没有内容，是空的，所以这个被判定为\"避免空的 static 初始化\"\n\n\n<例子2>\npublic class Persion {\n\tstatic {\n\t\t// 初始化的静态块\n\t}\n}这段代码中的static语句块是有内容的，不是空的，但是static初始化语句块中只有注释代码，没有实际的逻辑，所以这个被判定为\"避免空的 static 初始化\"\n", + "no_example": "### 不能被判定为\"避免空的 static 初始化\"的例子\n<例子1>\npublic class Cat {\n\tstatic {\n\t\t// 初始化的静态块\n\t\tcat = null;\n\t}\n}这段代码中的static语句块是有内容的，不是空的，且static初始化语句块中有非注释代码，有实际的逻辑，所以这个不能被判定为\"避免空的 static 初始化\"\n" + }, + { + "id": 17, + "text": "避免日历类用法不当风险", + "language": "java", + "detail": "缺陷类型：避免日历类用法不当风险；修复方案：使用Java 8 及以上版本中的 java.time 包的LocalDate", + "yes_example": "### 被判定为\"避免日历类用法不当风险\"的例子\n<例子1>\nprivate static final Calendar calendar = new GregorianCalendar(2020, Calendar.JANUARY, 1);\n这段代码中的Calendar和GregorianCalendar是线程不安全的，所以这个被判定为\"避免日历类用法不当风险\"\n", + "no_example": "### 不能被判定为\"避免日历类用法不当风险\"的例子\n<例子1>\nprivate static final LocalDate calendar = LocalDate.of(2020, 1, 1);\n这段代码中的LocalDate使用的是Java 8 及以上版本中的 java.time 包，LocalDate 是不可变的并且是线程安全的，不会有线程安全和性能方面的问题，所以这个不能被判定为\"避免日历类用法不当风险\"\n" + }, + { + "id": 18, + "text": "使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()", + "language": "java", + "detail": "缺陷类型：使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()；对应Fixer：ClassCastExpWithToArrayF ixer；修复方案：使用集合的toArray(T[]array)，且传入的是类型完全一样的数组", + "yes_example": "### 被判定为\"使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()\"的例子\n<例子1>\nList stringList = new ArrayList<>();\nstringList.add(\"Apple\");\nstringList.add(\"Banana\");\nObject[] objectArray = stringList.toArray(new Object[5]);\n这段代码使用集合转数组的方法的时候使用了toArray(new Object[5])，但是传入的数组类型不一致，所以这个被判定为\"使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()\"\n", + "no_example": "### 不能被判定为\"使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()\"的例子\n<例子1>\nList stringList = new ArrayList<>();\nstringList.add(\"Apple\");\nstringList.add(\"Banana\");\nString[] stringArray = stringList.toArray(new String[stringList.size()]);\n这段代码使用集合转数组的方法的时候使用了toArray(new String[stringList.size()])，传入的是类型完全一样的数组，所以这个不能被判定为\"使用集合转数组的方法，必须使用集合的toArray(T[]array)，传入的是类型完全一样的数组，大小就是list.size()\"\n" + }, + { + "id": 19, + "text": "禁止在 equals()中使用 NULL或者null 做比较", + "language": "java", + "detail": "缺陷类型：禁止在 equals()中使用 NULL或者null 做比较；对应Fixer：EqualsNullFixer；修复方案：使用Object的判空函数 做比较", + "yes_example": "### 被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"的例子\n<例子1>\nif (\"test\".equals(null)) {\n\tSystem.out.println(\"test\");\n}这段代码中if条件中的代码\"test\".equals(null)使用equals()函数与null进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n\n\n<例子2>\nif (!rangeValues[1].equals(\"null\")) {\n\tmaxValue = new BigDecimal(rangeValues[1]);\n}这段代码中if条件中的代码!rangeValues[1].equals(\"null\")使用equals()函数与Nnull进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n\n\n<例子3>\nString str1 = \"example\";\nif (str1.equals(\"null\")) {\n System.out.println(\"str1 is null\");\n}这段代码中if条件中的代码str1.equals(null)使用equals()函数与null进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n\n\n<例子4>\nString str3 = \"example\";\nif (str3 != null && str3.equals(\"null\")) {\n System.out.println(\"str3 is null\");\n}这段代码中if条件中的代码str3.equals(\"null\")使用equals()函数与\"null\"进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n\n\n<例子5>\nInteger num1 = 10;\nif (num1.equals(null)) {\n System.out.println(\"num1 is null\");\n}这段代码中if条件中的代码num1.equals(null)使用equals()函数与\"null\"进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n\n\n<例子6>\nObject obj = new Object();\nif (obj.equals(null)) {\n System.out.println(\"obj is null\");\n}这段代码中if条件中的代码obj.equals(null)使用equals()函数与\"null\"进行了比较，所以这个被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n", + "no_example": "### 不能被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"的例子\n<例子1>\na = \"test\";\nif (a.equals(\"test\")) {\n\tSystem.out.println(\"test\");\n}这段代码中if条件中的代码a.equals(\"test\")使用equals()函数与\"test\"进行了比较，所以这个不能被判定为\"禁止在 equals()中使用 NULL或者null 做比较\"\n" + }, + { + "id": 20, + "text": "switch 语句块不能为空", + "language": "java", + "detail": "缺陷类型：switch 语句块不能为空；对应Fixer：EmptySwitchStatementsFix；修复方案：删除整个空 switch 语句块", + "yes_example": "### 被判定为\"switch 语句块不能为空\"的例子\n<例子1>\nswitch (number) {\n\t\n}这段代码是一个switch语句块，但是里面没有内容，所以这个被判定为\"switch 语句块不能为空\"\n\n\n<例子2>\nswitch (number) {\n\t// 这是一个switch语句块\n}这段代码是一个switch语句块，里面虽然有内容，但是内容仅仅是注释内容，没有实际的逻辑，所以这个被判定为\"switch 语句块不能为空\"\n", + "no_example": "### 不能被判定为\"switch 语句块不能为空\"的例子\n<例子1>\nswitch (number) {\n\tcase 1:\n\t\tSystem.out.println(\"Number one\");\n\t\tbreak;\n\tdefault:\n\t\tSystem.out.println(\"This is the default block, which is incorrectly placed here.\");\n\t\tbreak;\n}这段代码是一个switch语句块，里面有内容，而且内容里有非注释的代码，有实际的逻辑，所以这个不能被判定为\"switch 语句块不能为空\"\n" + }, + { + "id": 21, + "text": "在进行类型强制转换时，右括号与强制转换值之间不需要任何空格隔开", + "detail": "缺陷类型：类型强制转换时空格使用不当；修复方案：在进行类型强制转换时，右括号与强制转换值之间不需要任何空格隔开。", + "language": "Java", + "yes_example": "### 被判定为\"在进行类型强制转换时，右括号与强制转换值之间不需要任何空格隔开\"的例子\n<例子1>\nint a = (int) 3.5;\n", + "no_example": "### 不能被判定为\"在进行类型强制转换时，右括号与强制转换值之间不需要任何空格隔开\"的例子\n<例子1>\nint a = (int)3.5;\n" + }, + { + "id": 22, + "text": "方法参数在定义和传入时，多个参数逗号后面必须加空格", + "detail": "缺陷类型：方法参数逗号后缺少空格；修复方案：方法参数在定义和传入时，多个参数逗号后面必须加空格。", + "language": "Java", + "yes_example": "### 被判定为\"方法参数在定义和传入时，多个参数逗号后面必须加空格\"的例子\n<例子1>\npublic void exampleMethod(int a,int b,int c) {}\n", + "no_example": "### 不能被判定为\"方法参数在定义和传入时，多个参数逗号后面必须加空格\"的例子\n<例子1>\npublic void exampleMethod(int a, int b, int c) {}\n" + }, + { + "id": 23, + "text": "禁止使用构造方法 BigDecimal(double) 的方式把 double 值转化为 BigDecimal 对象", + "detail": "缺陷类型：使用不推荐的 BigDecimal 构造方法；修复方案：禁止使用构造方法 BigDecimal(double) 的方式把 double 值转化为 BigDecimal 对象，推荐使用 BigDecimal 的 valueOf 方法。", + "language": "Java", + "yes_example": "### 被判定为\"禁止使用构造方法 BigDecimal(double) 的方式把 double 值转化为 BigDecimal 对象\"的例子\n<例子1>\nBigDecimal bd = new BigDecimal(0.1);\n", + "no_example": "### 不能被判定为\"禁止使用构造方法 BigDecimal(double) 的方式把 double 值转化为 BigDecimal 对象\"的例子\n<例子1>\nBigDecimal bd = BigDecimal.valueOf(0.1);\n" + }, + { + "id": 24, + "text": "不能有多余的分号", + "detail": "缺陷类型：多余的分号；修复方案：删除多余的分号", + "yes_example": "### 被判定为\"不能有多余的分号\"的例子\n<例子1>\npublic void trigger(String executionId, Map processVariables) {\n commandExecutor.execute(new TriggerCmd(executionId, processVariables));\n}\n;\na = 1;\nb = 2;\nsum = a + b;\n这段代码中包含一个多余的分号\";\"，所以这个被判定为\"不能有多余的分号\"\n", + "no_example": "### 不能被判定为\"不能有多余的分号\"的例子\n<例子1>\nwhile (True) {\n\ta = a + 1;\n\tbreak;\n}这段代码每个分号都是必须要的，所以这个能被判定为\"不能有多余的分号\"\n" + }, + { + "id": 25, + "text": "非线程安全的 SimpleDateFormat 使用，必须在函数或代码块级别使用synchronized", + "detail": "缺陷类型：非线程安全的 SimpleDateFormat 使用；修复方案：在函数或代码块级别加上synchronized修饰 或 使用其他线程安全的方式", + "yes_example": "### 被判定为\"非线程安全的 SimpleDateFormat 使用，必须在函数或代码块级别使用synchronized\"的例子\n<例子1>\npublic void formatDate(Date date) {\n\tSimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd\");\n\tSystem.out.println(\"Formatted date: \" + sdf.format(date));\n}这段代码中的函数formatDate在未使用synchronized同步修饰的情况下使用了SimpleDateFormat，这是线程不安全的，所以这个被判定为\"非线程安全的 SimpleDateFormat 使用，必须在函数或代码块级别使用synchronized\"\n", + "no_example": "### 不能被判定为\"非线程安全的 SimpleDateFormat 使用，必须在函数或代码块级别使用synchronized\"的例子\n<例子1>\npublic synchronized void formatDate(Date date) {\n\tSimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd\");\n\tSystem.out.println(\"Formatted date: \" + sdf.format(date));\n}这段代码是在synchronized同步块对函数'formatDate'进行保护，保证了线程安全，所以这个不能被判定为\"非线程安全的 SimpleDateFormat 使用，必须在函数或代码块级别使用synchronized\"\n" + }, + { + "id": 26, + "text": "类名使用驼峰式UpperCamelCase风格, 方法名、参数名、成员变量、局部变量都统一使用lowerCamelCase风格", + "detail": "缺陷类型：命名规范；修复方案：类名使用UpperCamelCase风格，方法名、参数名、成员变量、局部变量使用lowerCamelCase风格。", + "language": "Java", + "yes_example": "### 被判定为命名规范的例子\n<例子1>\npublic class myClass {\n private int MyVariable;\n public void MyMethod() {}\n}\n这段代码中的类名、成员变量和方法名没有遵循驼峰命名法，所以被判定为命名规范问题。\n", + "no_example": "### 不能被判定为命名规范的例子\n<例子1>\npublic class MyClass {\n private int myVariable;\n public void myMethod() {}\n}\n这段代码中的类名、成员变量和方法名都遵循了驼峰命名法，所以不能被判定为命名规范问题。\n" + }, + { + "id": 27, + "text": "抽象类命名使用 Abstract 或 Base 开头；异常类命名使用 Exception 结尾，测试类命名以它要测试的类的名称开始，以 Test 结尾", + "detail": "缺陷类型：命名规范；修复方案：抽象类命名使用 Abstract 或 Base 开头，异常类命名使用 Exception 结尾，测试类命名以它要测试的类的名称开始，以 Test 结尾。", + "language": "Java", + "yes_example": "### 被判定为命名规范的例子\n<例子1>\npublic class MyAbstractClass {}\npublic class MyExceptionClass {}\npublic class TestMyClass {}\n这段代码中的抽象类、异常类和测试类的命名不符合规范，所以被判定为命名规范问题。\n", + "no_example": "### 不能被判定为命名规范的例子\n<例子1>\npublic abstract class AbstractMyClass {}\npublic class MyCustomException extends Exception {}\npublic class MyClassTest {}\n这段代码中的抽象类、异常类和测试类的命名都符合规范，所以不能被判定为命名规范问题。\n" + }, + { + "id": 28, + "text": "POJO 类中的任何布尔类型的变量，都不要加 is 前缀", + "detail": "缺陷类型：命名规范；修复方案：POJO 类中的布尔类型变量不要加 is 前缀。", + "language": "Java", + "yes_example": "### 被判定为命名规范的例子\n<例子1>\npublic class User {\n private boolean isActive;\n}\n这段代码中的布尔类型变量加了 is 前缀，所以被判定为命名规范问题。\n", + "no_example": "### 不能被判定为命名规范的例子\n<例子1>\npublic class User {\n private boolean active;\n}\n这段代码中的布尔类型变量没有加 is 前缀，所以不能被判定为命名规范问题。\n" + }, + { + "id": 29, + "text": "杜绝完全不规范的英文缩写，避免望文不知义。", + "detail": "缺陷类型：命名规范；修复方案：避免使用不规范的英文缩写，确保代码可读性。", + "language": "Java", + "yes_example": "### 被判定为命名规范的例子\n<例子1>\npublic class CfgMgr {\n private int cnt;\n}\n这段代码中的类名和变量名使用了不规范的英文缩写，所以被判定为命名规范问题。\n", + "no_example": "### 不能被判定为命名规范的例子\n<例子1>\npublic class ConfigManager {\n private int count;\n}\n这段代码中的类名和变量名没有使用不规范的英文缩写，所以不能被判定为命名规范问题。\n" + }, + { + "id": 30, + "text": "不允许任何魔法值（即未经预先定义的常量）直接出现在代码中", + "detail": "缺陷类型：代码规范；修复方案：将魔法值定义为常量。", + "language": "Java", + "yes_example": "### 被判定为代码规范的例子\n<例子1>\npublic class MagicNumberExample {\n public void calculate() {\n int result = 42 * 2;\n }\n}\n这段代码中直接使用了魔法值 42，所以被判定为代码规范问题。\n", + "no_example": "### 不能被判定为代码规范的例子\n<例子1>\npublic class MagicNumberExample {\n private static final int MULTIPLIER = 42;\n public void calculate() {\n int result = MULTIPLIER * 2;\n }\n}\n这段代码中将魔法值定义为了常量，所以不能被判定为代码规范问题。\n" + }, + { + "id": 31, + "text": "long 或 Long 赋值时，数值后使用大写 L，不能是小写 l，浮点数类型的数值后缀统一为大写的 D 或 F", + "detail": "缺陷类型：代码规范；修复方案：long 或 Long 赋值时使用大写 L，浮点数类型的数值后缀使用大写的 D 或 F。", + "language": "Java", + "yes_example": "### 被判定为代码规范的例子\n<例子1>\npublic class NumberExample {\n private long value = 1000l;\n private double pi = 3.14d;\n}\n这段代码中使用了小写的 l 和 d，所以被判定为代码规范问题。\n", + "no_example": "### 不能被判定为代码规范的例子\n<例子1>\npublic class NumberExample {\n private long value = 1000L;\n private double pi = 3.14D;\n}\n这段代码中使用了大写的 L 和 D，所以不能被判定为代码规范问题。\n" + }, + { + "id": 32, + "text": "如果大括号内为空，简洁地写成{}即可，大括号中间无需换行和空格；如果是非空代码块，则：1）左大括号前不换行。2）左大括号后换行。3）右大括号前换行。4）右大括号后还有 else 等代码则不换行；表示终止的右大括号后必须换行。", + "detail": "缺陷类型：代码格式；修复方案：遵循大括号的使用规范。", + "language": "Java", + "yes_example": "### 被判定为代码格式的例子\n<例子1>\npublic class BracketExample{public void method(){\n if (true) {\n }}\n}\n这段代码中的大括号使用不符合规范，所以被判定为代码格式问题。\n", + "no_example": "### 不能被判定为代码格式的例子\n<例子1>\npublic class BracketExample {\n public void method() {\n if (true) {\n // do something\n }\n }\n}\n这段代码中的大括号使用符合规范，所以不能被判定为代码格式问题。\n" + }, + { + "id": 33, + "text": "左小括号和右边相邻字符之间不需要空格；右小括号和左边相邻字符之间也不需要空格；而左大括号前需要加空格。", + "detail": "缺陷类型：代码格式；修复方案：遵循括号和空格的使用规范。", + "language": "Java", + "yes_example": "### 被判定为代码格式的例子\n<例子1>\npublic class SpaceExample {\n public void method (){\n }\n}\n这段代码中的括号和空格使用不符合规范，所以被判定为代码格式问题。\n", + "no_example": "### 不能被判定为代码格式的例子\n<例子1>\npublic class SpaceExample {\n public void method() {}\n}\n这段代码中的括号和空格使用符合规范，所以不能被判定为代码格式问题。\n" + }, + { + "id": 34, + "text": "if / for / while / switch / do 等保留字与左右括号之间都必须加空格。", + "detail": "缺陷类型：代码格式；修复方案：保留字与左右括号之间加空格。", + "language": "Java", + "yes_example": "### 被判定为代码格式的例子\n<例子1>\npublic class KeywordExample {\n public void method() {\n if(true) {\n }\n }\n}\n这段代码中的 if 关键字与括号之间没有空格，所以被判定为代码格式问题。\n", + "no_example": "### 不能被判定为代码格式的例子\n<例子1>\npublic class KeywordExample {\n public void method() {\n if (true) {\n }\n }\n}\n这段代码中的 if 关键字与括号之间有空格，所以不能被判定为代码格式问题。\n" + }, + { + "id": 35, + "text": "所有整型包装类对象之间值的比较，全部使用 equals 方法比较", + "detail": "缺陷类型：代码规范；修复方案：整型包装类对象之间的值比较使用 equals 方法。", + "language": "Java", + "yes_example": "### 被判定为代码规范的例子\n<例子1>\npublic class IntegerComparison {\n public void compare() {\n Integer a = 100;\n Integer b = 100;\n if (a == b) {\n }\n }\n}\n这段代码中使用了 == 比较整型包装类对象，所以被判定为代码规范问题。\n", + "no_example": "### 不能被判定为代码规范的例子\n<例子1>\npublic class IntegerComparison {\n public void compare() {\n Integer a = 100;\n Integer b = 100;\n if (a.equals(b)) {\n }\n }\n}\n这段代码中使用了 equals 方法比较整型包装类对象，所以不能被判定为代码规范问题。\n" + }, + { + "id": 36, + "text": "BigDecimal 的等值比较应使用 compareTo() 方法，而不是 equals() 方法。", + "detail": "缺陷类型：BigDecimal 等值比较错误；修复方案：使用 compareTo() 方法进行比较。", + "language": "Java", + "yes_example": "### 被判定为\"BigDecimal 的等值比较应使用 compareTo() 方法，而不是 equals() 方法\"的例子\n<例子1>\nBigDecimal a = new BigDecimal(\"1.0\");\nBigDecimal b = new BigDecimal(\"1.00\");\nif (a.equals(b)) {\n // 这段代码会返回 false，因为 equals() 方法会比较精度\n}\n", + "no_example": "### 不能被判定为\"BigDecimal 的等值比较应使用 compareTo() 方法，而不是 equals() 方法\"的例子\n<例子1>\nBigDecimal a = new BigDecimal(\"1.0\");\nBigDecimal b = new BigDecimal(\"1.00\");\nif (a.compareTo(b) == 0) {\n // 这段代码会返回 true，因为 compareTo() 方法只比较数值\n}\n" + }, + { + "id": 37, + "text": "禁止在 POJO 类中，同时存在对应属性 xxx 的 isXxx() 和 getXxx() 方法。", + "detail": "缺陷类型：POJO 类中存在重复的 getter 方法；修复方案：确保只存在一个 getter 方法。", + "language": "Java", + "yes_example": "### 被判定为\"禁止在 POJO 类中，同时存在对应属性 xxx 的 isXxx() 和 getXxx() 方法\"的例子\n<例子1>\npublic class User {\n private boolean active;\n public boolean isActive() {\n return active;\n }\n public boolean getActive() {\n return active;\n }\n}\n", + "no_example": "### 不能被判定为\"禁止在 POJO 类中，同时存在对应属性 xxx 的 isXxx() 和 getXxx() 方法\"的例子\n<例子1>\npublic class User {\n private int age;\n public int getAge() {\n return age;\n }\n}\n" + }, + { + "id": 38, + "text": "日期格式化时，传入 pattern 中表示年份统一使用小写的 y。", + "detail": "缺陷类型：日期格式化错误；修复方案：使用小写的 y 表示年份。", + "language": "Java", + "yes_example": "### 被判定为\"日期格式化时，传入 pattern 中表示年份统一使用小写的 y\"的例子\n<例子1>\nSimpleDateFormat sdf = new SimpleDateFormat(\"YYYY-MM-dd\");\n", + "no_example": "### 不能被判定为\"日期格式化时，传入 pattern 中表示年份统一使用小写的 y\"的例子\n<例子1>\nSimpleDateFormat sdf = new SimpleDateFormat(\"yyyy-MM-dd\");\n" + }, + { + "id": 39, + "text": "不允许在程序任何地方中使用：1）java.sql.Date 2）java.sql.Time 3）java.sql.Timestamp。", + "detail": "缺陷类型：使用了 java.sql 包中的日期类；修复方案：使用 java.time 包中的日期类。", + "language": "Java", + "yes_example": "### 被判定为\"不允许在程序任何地方中使用：1）java.sql.Date 2）java.sql.Time 3）java.sql.Timestamp\"的例子\n<例子1>\njava.sql.Date sqlDate = new java.sql.Date(System.currentTimeMillis());\n", + "no_example": "### 不能被判定为\"不允许在程序任何地方中使用：1）java.sql.Date 2）java.sql.Time 3）java.sql.Timestamp\"的例子\n<例子1>\njava.time.LocalDate localDate = java.time.LocalDate.now();\n" + }, + { + "id": 40, + "text": "判断所有集合内部的元素是否为空，使用 isEmpty() 方法，而不是 size() == 0 的方式。", + "detail": "缺陷类型：集合判空方式错误；修复方案：使用 isEmpty() 方法。", + "language": "Java", + "yes_example": "### 被判定为\"判断所有集合内部的元素是否为空，使用 isEmpty() 方法，而不是 size() == 0 的方式\"的例子\n<例子1>\nList list = new ArrayList<>();\nif (list.size() == 0) {\n // 判空逻辑\n}\n", + "no_example": "### 不能被判定为\"判断所有集合内部的元素是否为空，使用 isEmpty() 方法，而不是 size() == 0 的方式\"的例子\n<例子1>\nList list = new ArrayList<>();\nif (list.isEmpty()) {\n // 判空逻辑\n}\n" + }, + { + "id": 41, + "text": "只要覆写 equals，就必须覆写 hashCode。", + "detail": "缺陷类型：未覆写 hashCode 方法；修复方案：同时覆写 equals 和 hashCode 方法。", + "language": "Java", + "yes_example": "### 被判定为\"只要覆写 equals，就必须覆写 hashCode\"的例子\n<例子1>\npublic class User {\n private String name;\n @Override\n public boolean equals(Object o) {\n if (this == o) return true;\n if (o == null || getClass() != o.getClass()) return false;\n User user = (User) o;\n return Objects.equals(name, user.name);\n }\n}\n", + "no_example": "### 不能被判定为\"只要覆写 equals，就必须覆写 hashCode\"的例子\n<例子1>\npublic class User {\n private String name;\n @Override\n public boolean equals(Object o) {\n if (this == o) return true;\n if (o == null || getClass() != o.getClass()) return false;\n User user = (User) o;\n return Objects.equals(name, user.name);\n }\n @Override\n public int hashCode() {\n return Objects.hash(name);\n }\n}\n" + }, + { + "id": 42, + "text": "使用 Map 的方法 keySet() / values() / entrySet() 返回集合对象时，不可以对其进行添加元素操作，否则会抛出 UnsupportedOperationException 异常。", + "detail": "缺陷类型：对 Map 的 keySet() / values() / entrySet() 返回的集合进行添加操作；修复方案：避免对这些集合进行添加操作。", + "language": "Java", + "yes_example": "### 被判定为\"使用 Map 的方法 keySet() / values() / entrySet() 返回集合对象时，不可以对其进行添加元素操作，否则会抛出 UnsupportedOperationException 异常\"的例子\n<例子1>\nMap map = new HashMap<>();\nmap.put(\"key1\", \"value1\");\nSet keys = map.keySet();\nkeys.add(\"key2\");\n", + "no_example": "### 不能被判定为\"使用 Map 的方法 keySet() / values() / entrySet() 返回集合对象时，不可以对其进行添加元素操作，否则会抛出 UnsupportedOperationException 异常\"的例子\n<例子1>\nMap map = new HashMap<>();\nmap.put(\"key1\", \"value1\");\nSet keys = map.keySet();\n// 不进行添加操作\n" + }, + { + "id": 43, + "text": "不要在 foreach 循环里进行元素的 remove / add 操作。remove 元素请使用 iterator 方式，如果并发操作，需要对 iterator", + "detail": "缺陷类型：在 foreach 循环中进行元素的 remove / add 操作；修复方案：使用 iterator 进行元素的 remove 操作。", + "language": "Java", + "yes_example": "### 被判定为\"不要在 foreach 循环里进行元素的 remove / add 操作。remove 元素请使用 iterator 方式，如果并发操作，需要对 iterator\"的例子\n<例子1>\nList list = new ArrayList<>(Arrays.asList(\"a\", \"b\", \"c\"));\nfor (String s : list) {\n if (s.equals(\"a\")) {\n list.remove(s);\n }\n}\n", + "no_example": "### 不能被判定为\"不要在 foreach 循环里进行元素的 remove / add 操作。remove 元素请使用 iterator 方式，如果并发操作，需要对 iterator\"的例子\n<例子1>\nList list = new ArrayList<>(Arrays.asList(\"a\", \"b\", \"c\"));\nIterator iterator = list.iterator();\nwhile (iterator.hasNext()) {\n String s = iterator.next();\n if (s.equals(\"a\")) {\n iterator.remove();\n }\n}\n" + }, + { + "id": 44, + "text": "类、类属性、类方法的注释必须使用 Javadoc 规范，使用 /** 内容 */ 格式，不得使用 // xxx方式。", + "detail": "缺陷类型：注释不符合 Javadoc 规范；修复方案：使用 Javadoc 规范的注释格式。", + "language": "Java", + "yes_example": "### 被判定为\"类、类属性、类方法的注释必须使用 Javadoc 规范，使用 /** 内容 */ 格式，不得使用 // xxx方式\"的例子\n<例子1>\npublic class Example {\n // 这是一个类注释\n private String name;\n // 这是一个属性注释\n public String getName() {\n return name;\n }\n // 这是一个方法注释\n}\n", + "no_example": "### 不能被判定为\"类、类属性、类方法的注释必须使用 Javadoc 规范，使用 /** 内容 */ 格式，不得使用 // xxx方式\"的例子\n<例子1>\n/**\n * 这是一个类注释\n */\npublic class Example {\n /**\n * 这是一个属性注释\n */\n private String name;\n /**\n * 这是一个方法注释\n */\n public String getName() {\n return name;\n }\n}\n" + }, + { + "id": 45, + "text": "所有的抽象方法（包括接口中的方法）必须要用 Javadoc 注释", + "detail": "缺陷类型：缺少 Javadoc 注释；修复方案：为所有的抽象方法（包括接口中的方法）添加 Javadoc 注释，除了返回值、参数异常说明外，还必须指出该方法做什么事情，实现什么功能。", + "language": "Java", + "yes_example": "### 被判定为缺少 Javadoc 注释的例子\n<例子1>\npublic interface MyInterface {\n void doSomething();\n}\n这段代码中的接口方法 doSomething() 没有 Javadoc 注释，所以被判定为缺少 Javadoc 注释。\n", + "no_example": "### 不能被判定为缺少 Javadoc 注释的例子\n<例子1>\n/**\n * 执行某个操作\n * @param param 参数说明\n * @return 返回值说明\n * @throws Exception 异常说明\n */\npublic interface MyInterface {\n void doSomething(String param) throws Exception;\n}\n这段代码中的接口方法 doSomething() 有完整的 Javadoc 注释，所以不能被判定为缺少 Javadoc 注释。\n" + }, + { + "id": 46, + "text": "方法内部单行注释和多行注释的使用规范", + "detail": "缺陷类型：注释使用不规范；修复方案：方法内部单行注释，在被注释语句上方另起一行，使用 // 注释。方法内部多行注释使用 /* */注释，注意与代码对齐。", + "language": "Java", + "yes_example": "### 被判定为注释使用不规范的例子\n<例子1>\npublic void exampleMethod() {\n int a = 1; // 初始化变量a\n int b = 2; /* 初始化变量b */\n}\n这段代码中的单行注释和多行注释没有按照规范使用，所以被判定为注释使用不规范。\n", + "no_example": "### 不能被判定为注释使用不规范的例子\n<例子1>\npublic void exampleMethod() {\n // 初始化变量a\n int a = 1;\n /*\n * 初始化变量b\n */\n int b = 2;\n}\n这段代码中的单行注释和多行注释按照规范使用，所以不能被判定为注释使用不规范。\n" + }, + { + "id": 47, + "text": "所有的枚举类型字段必须要有注释", + "detail": "缺陷类型：枚举类型字段缺少注释；修复方案：为所有的枚举类型字段添加注释，说明每个数据项的用途。", + "language": "Java", + "yes_example": "### 被判定为枚举类型字段缺少注释的例子\n<例子1>\npublic enum Status {\n ACTIVE,\n INACTIVE\n}\n这段代码中的枚举类型字段没有注释，所以被判定为枚举类型字段缺少注释。\n", + "no_example": "### 不能被判定为枚举类型字段缺少注释的例子\n<例子1>\npublic enum Status {\n /**\n * 活跃状态\n */\n ACTIVE,\n /**\n * 非活跃状态\n */\n INACTIVE\n}\n这段代码中的枚举类型字段有注释，所以不能被判定为枚举类型字段缺少注释。\n" + }, + { + "id": 48, + "text": "finally 块必须对资源对象、流对象进行关闭", + "detail": "缺陷类型：资源对象、流对象未在 finally 块中关闭；修复方案：在 finally 块中对资源对象、流对象进行关闭，有异常也要做 try-catch。", + "language": "Java", + "yes_example": "### 被判定为资源对象、流对象未在 finally 块中关闭的例子\n<例子1>\npublic void readFile() {\n FileInputStream fis = null;\n try {\n fis = new FileInputStream(\"file.txt\");\n // 读取文件内容\n } catch (IOException e) {\n e.printStackTrace();\n }\n}\n这段代码中的 FileInputStream 对象没有在 finally 块中关闭，所以被判定为资源对象、流对象未在 finally 块中关闭。\n", + "no_example": "### 不能被判定为资源对象、流对象未在 finally 块中关闭的例子\n<例子1>\npublic void readFile() {\n FileInputStream fis = null;\n try {\n fis = new FileInputStream(\"file.txt\");\n // 读取文件内容\n } catch (IOException e) {\n e.printStackTrace();\n } finally {\n if (fis != null) {\n try {\n fis.close();\n } catch (IOException e) {\n e.printStackTrace();\n }\n }\n }\n}\n这段代码中的 FileInputStream 对象在 finally 块中关闭，所以不能被判定为资源对象、流对象未在 finally 块中关闭。\n" + }, + { + "id": 49, + "text": "常量命名应该全部大写，单词间用下划线隔开", + "detail": "缺陷类型：常量命名不规范；修复方案：常量命名应该全部大写，单词间用下划线隔开，力求语义表达完整清楚，不要嫌名字长。", + "language": "Java", + "yes_example": "### 被判定为\"常量命名应该全部大写，单词间用下划线隔开\"的例子\n<例子1>\npublic static final int maxCount = 100;\n", + "no_example": "### 不能被判定为\"常量命名应该全部大写，单词间用下划线隔开\"的例子\n<例子1>\npublic static final int MAX_COUNT = 100;\n" + }, + { + "id": 50, + "text": "任何二目、三目运算符的左右两边都需要加一个空格", + "detail": "缺陷类型：运算符两边缺少空格；修复方案：任何二目、三目运算符的左右两边都需要加一个空格。", + "language": "Java", + "yes_example": "### 被判定为\"任何二目、三目运算符的左右两边都需要加一个空格\"的例子\n<例子1>\nint a=b+c;\n", + "no_example": "### 不能被判定为\"任何二目、三目运算符的左右两边都需要加一个空格\"的例子\n<例子1>\nint a = b + c;\n" + }, + { + "id": 1, + "text": "避免使用from import *", + "detail": "缺陷类型：避免使用from import *，导入所有内容会造成命名冲突；修复方案：每个使用到的子依赖需分别导入。", + "language": "Python", + "yes_example": "### 被判定为'避免使用from import *'的例子\n<例子1>\nfrom math import *\n", + "no_example": "### 不能被判定为'避免使用from import *'的例子\n<例子1>\nfrom math import sqrt, pi\n" + }, + { + "id": 2, + "text": "避免使用__import__()函数动态导入模块", + "detail": "缺陷类型：避免使用__import__()函数动态导入模块；修复方案：使用标准的import语句。", + "language": "Python", + "yes_example": "### 被判定为'避免使用__import__()函数动态导入模块'的例子\n<例子1>\nmodule = __import__('math')\n", + "no_example": "### 不能被判定为'避免使用__import__()函数动态导入模块'的例子\n<例子1>\nimport math\n" + }, + { + "id": 3, + "text": "导入语句未按标准库导入、相关第三方导入、本地应用/库特定导入的顺序分组", + "detail": "缺陷类型：导入语句未按标准库导入、相关第三方导入、本地应用/库特定导入的顺序分组；修复方案：按顺序分组导入语句。", + "language": "Python", + "yes_example": "### 被判定为'导入语句未按标准库导入、相关第三方导入、本地应用/库特定导入的顺序分组'的例子\n<例子1>\nimport requests\n\nimport mymodule\n\nimport os\n", + "no_example": "### 不能被判定为'导入语句未按标准库导入、相关第三方导入、本地应用/库特定导入的顺序分组'的例子\n<例子1>\nimport os\nimport requests\n\nimport mymodule\n" + }, + { + "id": 4, + "text": "避免未使用的函数形参", + "detail": "缺陷类型：存在未使用的函数形参；修复方案：移除未使用的函数形参。", + "language": "Python", + "yes_example": "### 被判定为'避免未使用的函数形参'的例子\n<例子1>\ndef func(a, b):\n return a\n", + "no_example": "### 不能被判定为'避免未使用的函数形参'的例子\n<例子1>\ndef func(a):\n return a\n" + }, + { + "id": 5, + "text": "使用is not None来检查一个变量是否不是None", + "detail": "缺陷类型：未使用is not None来检查一个变量是否不是None；修复方案：使用is not None来检查。", + "language": "Python", + "yes_example": "### 被判定为'使用is not None来检查一个变量是否不是None'的例子\n<例子1>\nif variable != None:\n pass\n", + "no_example": "### 不能被判定为'使用is not None来检查一个变量是否不是None'的例子\n<例子1>\nif variable is not None:\n pass\n" + }, + { + "id": 6, + "text": "避免使用==或!=来比较实例的等价性", + "detail": "缺陷类型：使用==或!=来比较实例的等价性；修复方案：应使用equals比较。", + "language": "Python", + "yes_example": "### 被判定为'避免使用==或!=来比较实例的等价性'的例子\n<例子1>\nif obj1 == obj2:\n pass\n", + "no_example": "### 不能被判定为'避免使用==或!=来比较实例的等价性'的例子\n<例子1>\nif obj1.equals(obj2):\n pass\n" + }, + { + "id": 7, + "text": "使用描述性变量名，避免使用单字母变量名", + "detail": "缺陷类型：使用单字母变量名；修复方案：使用描述性变量名。", + "language": "Python", + "yes_example": "### 被判定为'使用描述性变量名，避免使用单字母变量名'的例子\n<例子1>\nx = 10\n", + "no_example": "### 不能被判定为'使用描述性变量名，避免使用单字母变量名'的例子\n<例子1>\ncount = 10\n" + }, + { + "id": 8, + "text": "常量命名使用全大写字母，并用下划线分隔", + "detail": "缺陷类型：常量命名未使用全大写字母或未用下划线分隔；修复方案：常量命名使用全大写字母，并用下划线分隔。", + "language": "Python", + "yes_example": "### 被判定为'常量命名未使用全大写字母，并用下划线分隔'的例子\n<例子1>\npi = 3.14\n", + "no_example": "### 不能被判定为'常量命名未使用全大写字母，并用下划线分隔'的例子\n<例子1>\nPI = 3.14\n" + }, + { + "id": 9, + "text": "类名应该使用驼峰式命名（CamelCase）", + "detail": "缺陷类型：类名未使用驼峰式命名；修复方案：类名使用驼峰式命名。", + "language": "Python", + "yes_example": "### 被判定为'类名未使用驼峰式命名（CamelCase）'的例子\n<例子1>\nclass my_class:\n pass\n", + "no_example": "### 不能被判定为'类名未使用驼峰式命名（CamelCase）'的例子\n<例子1>\nclass MyClass:\n pass\n" + }, + { + "id": 10, + "text": "尽量使用with语句来管理资源", + "detail": "缺陷类型：未使用with语句来管理资源；修复方案：使用with语句来管理资源。", + "language": "Python", + "yes_example": "### 被判定为'未使用with语句来管理资源'的例子\n<例子1>\nfile = open('file.txt', 'r')\ncontent = file.read()\nfile.close()\n", + "no_example": "### 不能被判定为'未使用with语句来管理资源'的例子\n<例子1>\nwith open('file.txt', 'r') as file:\n content = file.read()\n" + }, + { + "id": 11, + "text": "避免使用except:来捕获所有异常，应该指定异常类型", + "detail": "缺陷类型：捕获所有异常；修复方案：指定具体的异常类型。", + "language": "Python", + "yes_example": "### 被判定为'避免使用except:来捕获所有异常，应该指定异常类型'的例子\n<例子1>\ntry:\n # some code\nexcept:\n handle_error()\n这段代码中使用了except:来捕获所有异常，所以这个被判定为'避免使用except:来捕获所有异常，应该指定异常类型'\n", + "no_example": "### 不能被判定为'避免使用except:来捕获所有异常，应该指定异常类型'的例子\n<例子1>\ntry:\n # some code\nexcept ValueError:\n handle_value_error()\n这段代码中指定了具体的异常类型ValueError，所以这个不能被判定为'避免使用except:来捕获所有异常，应该指定异常类型'\n" + }, + { + "id": 12, + "text": "尽量避免手动拼接字符串", + "detail": "缺陷类型：手动拼接字符串；修复方案：使用格式化字符串或join方法。", + "language": "Python", + "yes_example": "### 被判定为'尽量避免手动拼接字符串'的例子\n<例子1>\nname = 'John'\ngreeting = 'Hello, ' + name + '!'\n这段代码中使用了手动拼接字符串，所以这个被判定为'尽量避免手动拼接字符串'\n", + "no_example": "### 不能被判定为'尽量避免手动拼接字符串'的例子\n<例子1>\nname = 'John'\ngreeting = f'Hello, {name}!'\n这段代码中使用了格式化字符串，所以这个不能被判定为'尽量避免手动拼接字符串'\n" + }, + { + "id": 13, + "text": "尽量避免出现魔法字符和数字，声明为常量", + "detail": "缺陷类型：使用魔法字符和数字；修复方案：将其声明为常量。", + "language": "Python", + "yes_example": "### 被判定为'尽量避免出现魔法字符和数字，声明为常量'的例子\n<例子1>\nif status == 1:\n print('Active')\n这段代码中使用了魔法数字1，所以这个被判定为'尽量避免出现魔法字符和数字，声明为常量'\n", + "no_example": "### 不能被判定为'尽量避免出现魔法字符和数字，声明为常量'的例子\n<例子1>\nACTIVE_STATUS = 1\nif status == ACTIVE_STATUS:\n print('Active')\n这段代码中将魔法数字声明为了常量ACTIVE_STATUS，所以这个不能被判定为'尽量避免出现魔法字符和数字，声明为常量'\n" + }, + { + "id": 14, + "text": "boolean变量判断无需显式比较", + "detail": "缺陷类型：显式比较boolean变量；修复方案：直接使用boolean变量进行判断。", + "language": "Python", + "yes_example": "### 被判定为'boolean变量判断无需显式比较'的例子\n<例子1>\nflag = True\nif flag == True:\n print('Flag is true')\n这段代码中对boolean变量进行了显式比较，所以这个被判定为'boolean变量判断无需显式比较'\n", + "no_example": "### 不能被判定为'boolean变量判断无需显式比较'的例子\n<例子1>\nflag = True\nif flag:\n print('Flag is true')\n这段代码中直接使用了boolean变量进行判断，所以这个不能被判定为'boolean变量判断无需显式比较'\n" + }, + { + "id": 15, + "text": "使用isinstance()来检查对象的类型", + "detail": "缺陷类型：使用type()检查对象类型；修复方案：使用isinstance()函数。", + "language": "Python", + "yes_example": "### 被判定为'使用isinstance()来检查对象的类型'的例子\n<例子1>\nif type(obj) == list:\n print('obj is a list')\n这段代码中使用了type()来检查对象类型，所以这个被判定为'使用isinstance()来检查对象的类型'\n", + "no_example": "### 不能被判定为'使用isinstance()来检查对象的类型'的例子\n<例子1>\nif isinstance(obj, list):\n print('obj is a list')\n这段代码中使用了isinstance()来检查对象类型，所以这个不能被判定为'使用isinstance()来检查对象的类型'\n" + }, + { + "id": 16, + "text": "避免使用os.system()来调用外部命令", + "detail": "缺陷类型：使用os.system()调用外部命令；修复方案：使用subprocess模块。", + "language": "Python", + "yes_example": "### 被判定为'避免使用os.system()来调用外部命令'的例子\n<例子1>\nos.system('ls -l')\n这段代码中使用了os.system()来调用外部命令，所以这个被判定为'避免使用os.system()来调用外部命令'\n", + "no_example": "### 不能被判定为'避免使用os.system()来调用外部命令'的例子\n<例子1>\nimport subprocess\nsubprocess.run(['ls', '-l'])\n这段代码中使用了subprocess模块来调用外部命令，所以这个不能被判定为'避免使用os.system()来调用外部命令'\n" + }, + { + "id": 17, + "text": "只使用@property装饰器来创建只读属性，而非修改属性", + "detail": "缺陷类型：使用@property装饰器创建可修改属性；修复方案：只使用@property装饰器创建只读属性。", + "language": "Python", + "yes_example": "### 被判定为'只使用@property装饰器来创建只读属性，而非修改属性'的例子\n<例子1>\nclass MyClass:\n def __init__(self, value):\n self._value = value\n\n @property\n def value(self, new_value):\n self._value = new_value\n这段代码中使用@property装饰器创建了可修改属性，所以这个被判定为'只使用@property装饰器来创建只读属性，而非修改属性'\n", + "no_example": "### 不能被判定为'只使用@property装饰器来创建只读属性，而非修改属性'的例子\n<例子1>\nclass MyClass:\n def __init__(self, value):\n self._value = value\n\n @property\n def value(self):\n return self._value\n这段代码中使用@property装饰器创建了只读属性，所以这个不能被判定为'只使用@property装饰器来创建只读属性，而非修改属性'\n" + }, + { + "id": 18, + "text": "在使用索引或切片时，不要在方括号或冒号内加空格", + "detail": "缺陷类型：在索引或切片的方括号或冒号内加空格；修复方案：去掉方括号或冒号内的空格。", + "language": "Python", + "yes_example": "### 被判定为'在使用索引或切片时，不要在方括号或冒号内加空格'的例子\n<例子1>\nlist = [1, 2, 3, 4]\nsublist = list[ 1 : 3 ]\n这段代码中在索引或切片的方括号或冒号内加了空格，所以这个被判定为'在使用索引或切片时，不要在方括号或冒号内加空格'\n", + "no_example": "### 不能被判定为'在使用索引或切片时，不要在方括号或冒号内加空格'的例子\n<例子1>\nlist = [1, 2, 3, 4]\nsublist = list[1:3]\n这段代码中在索引或切片的方括号或冒号内没有加空格，所以这个不能被判定为'在使用索引或切片时，不要在方括号或冒号内加空格'\n" + }, + { + "id": 19, + "text": "在逗号、分号或冒号前不要加空格，但在它们之后要加空格", + "detail": "缺陷类型：在逗号、分号或冒号前加空格或在它们之后不加空格；修复方案：在逗号、分号或冒号前不要加空格，但在它们之后要加空格。", + "language": "Python", + "yes_example": "### 被判定为'在逗号、分号或冒号前不要加空格，但在它们之后要加空格'的例子\n<例子1>\nif x == 4 : print(x , y)\n这段代码中在逗号、分号或冒号前加了空格或在它们之后没有加空格，所以这个被判定为'在逗号、分号或冒号前不要加空格，但在它们之后要加空格'\n", + "no_example": "### 不能被判定为'在逗号、分号或冒号前不要加空格，但在它们之后要加空格'的例子\n<例子1>\nif x == 4: print(x, y)\n这段代码中在逗号、分号或冒号前没有加空格且在它们之后加了空格，所以这个不能被判定为'在逗号、分号或冒号前不要加空格，但在它们之后要加空格'\n" + }, + { + "id": 20, + "text": "对于二元操作符，两边都应有空格，例如 a = b + 1", + "detail": "缺陷类型：二元操作符两边没有空格；修复方案：在二元操作符两边加空格。", + "language": "Python", + "yes_example": "### 被判定为'对于二元操作符，两边都应有空格，例如 a = b + 1'的例子\n<例子1>\na=b+1\n这段代码中二元操作符两边没有空格，所以这个被判定为'对于二元操作符，两边都应有空格，例如 a = b + 1'\n", + "no_example": "### 不能被判定为'对于二元操作符，两边都应有空格，例如 a = b + 1'的例子\n<例子1>\na = b + 1\n这段代码中二元操作符两边有空格，所以这个不能被判定为'对于二元操作符，两边都应有空格，例如 a = b + 1'\n" + }, + { + "id": 21, + "text": "避免使用Python关键字作为变量名或函数名", + "detail": "缺陷类型：使用Python关键字作为变量名或函数名；修复方案：使用非关键字的名称。", + "language": "Python", + "yes_example": "### 被判定为'避免使用Python关键字作为变量名或函数名'的例子\n<例子1>\ndef class():\n pass\n\n<例子2>\nfor = 5\n", + "no_example": "### 不能被判定为'避免使用Python关键字作为变量名或函数名'的例子\n<例子1>\ndef my_function():\n pass\n\n<例子2>\nnumber = 5\n" + }, + { + "id": 22, + "text": "避免使用特殊字符作为变量名，例如$或@", + "detail": "缺陷类型：使用特殊字符作为变量名；修复方案：使用合法的变量名。", + "language": "Python", + "yes_example": "### 被判定为'避免使用特殊字符作为变量名，例如$或@'的例子\n<例子1>\nmy$var = 10\n\n<例子2>\n@var = 20\n", + "no_example": "### 不能被判定为'避免使用特殊字符作为变量名，例如$或@'的例子\n<例子1>\nmy_var = 10\n\n<例子2>\nvar_20 = 20\n" + }, + { + "id": 23, + "text": "避免使用raise来重新抛出当前的异常，这会丢失原始的栈跟踪", + "detail": "缺陷类型：使用raise重新抛出当前异常；修复方案：使用raise ... from ...语法。", + "language": "Python", + "yes_example": "### 被判定为'避免使用raise来重新抛出当前的异常，这会丢失原始的栈跟踪'的例子\n<例子1>\ntry:\n 1 / 0\nexcept ZeroDivisionError:\n raise\n", + "no_example": "### 不能被判定为'避免使用raise来重新抛出当前的异常，这会丢失原始的栈跟踪'的例子\n<例子1>\ntry:\n 1 / 0\nexcept ZeroDivisionError as e:\n raise RuntimeError('Error occurred') from e\n" + }, + { + "id": 24, + "text": "避免在except块中使用pass，这会捕获并忽略异常", + "detail": "缺陷类型：在except块中使用pass；修复方案：处理异常或记录日志。", + "language": "Python", + "yes_example": "### 被判定为'避免在except块中使用pass，这会捕获并忽略异常'的例子\n<例子1>\ntry:\n 1 / 0\nexcept ZeroDivisionError:\n pass\n", + "no_example": "### 不能被判定为'避免在except块中使用pass，这会捕获并忽略异常'的例子\n<例子1>\ntry:\n 1 / 0\nexcept ZeroDivisionError as e:\n logging.error('Error occurred: %s', e)\n" + }, + { + "id": 25, + "text": "避免使用assert语句来执行重要的运行时检查", + "detail": "缺陷类型：使用assert语句执行重要的运行时检查；修复方案：使用显式的条件检查和异常处理。", + "language": "Python", + "yes_example": "### 被判定为'避免使用assert语句来执行重要的运行时检查'的例子\n<例子1>\ndef divide(a, b):\n assert b != 0\n return a / b\n", + "no_example": "### 不能被判定为'避免使用assert语句来执行重要的运行时检查'的例子\n<例子1>\ndef divide(a, b):\n if b == 0:\n raise ValueError('b cannot be zero')\n return a / b\n" + }, + { + "id": 26, + "text": "避免使用eval()和exec()，这些函数可能会带来安全风险", + "detail": "缺陷类型：使用eval()和exec()函数；修复方案：使用安全的替代方案。", + "language": "Python", + "yes_example": "### 被判定为'避免使用eval()和exec()，这些函数可能会带来安全风险'的例子\n<例子1>\neval('print(1)')\n\n<例子2>\nexec('a = 1')\n", + "no_example": "### 不能被判定为'避免使用eval()和exec()，这些函数可能会带来安全风险'的例子\n<例子1>\ncompiled_code = compile('print(1)', '', 'exec')\nexec(compiled_code)\n" + }, + { + "id": 27, + "text": "避免使用open()函数的exec模式，这可能会带来安全风险", + "detail": "缺陷类型：使用open()函数的exec模式；修复方案：使用安全的文件操作模式。", + "language": "Python", + "yes_example": "### 被判定为'避免使用open()函数的exec模式，这可能会带来安全风险'的例子\n<例子1>\nopen('file.txt', 'w+').write('data')\n", + "no_example": "### 不能被判定为'避免使用open()函数的exec模式，这可能会带来安全风险'的例子\n<例子1>\nwith open('file.txt', 'w') as f:\n f.write('data')\n" + }, + { + "id": 28, + "text": "避免使用sys.exit()，应使用异常来控制程序的退出", + "detail": "缺陷类型：使用sys.exit()退出程序；修复方案：使用异常处理机制。", + "language": "Python", + "yes_example": "### 被判定为'避免使用sys.exit()，应使用异常来控制程序的退出'的例子\n<例子1>\nimport sys\nsys.exit(1)\n", + "no_example": "### 不能被判定为'避免使用sys.exit()，应使用异常来控制程序的退出'的例子\n<例子1>\nraise SystemExit(1)\n" + }, + { + "id": 29, + "text": "避免使用time.sleep()进行线程同步，应使用同步原语，如锁或事件", + "detail": "缺陷类型：使用time.sleep()进行线程同步；修复方案：使用同步原语。", + "language": "Python", + "yes_example": "### 被判定为'避免使用time.sleep()进行线程同步，应使用同步原语，如锁或事件'的例子\n<例子1>\nimport time\n\ndef worker():\n time.sleep(1)\n", + "no_example": "### 不能被判定为'避免使用time.sleep()进行线程同步，应使用同步原语，如锁或事件'的例子\n<例子1>\nimport threading\n\nevent = threading.Event()\n\ndef worker():\n event.wait()\n" + }, + { + "id": 30, + "text": "每行代码尽量不超过79个字符", + "detail": "缺陷类型：每行代码超过79个字符；修复方案：将长行代码拆分为多行。", + "language": "Python", + "yes_example": "### 被判定为'每行代码尽量不超过79个字符'的例子\n<例子1>\nprint('This is a very long line of code that exceeds the 79 characters limit')\n", + "no_example": "### 不能被判定为'每行代码尽量不超过79个字符'的例子\n<例子1>\nprint('This is a very long line of code that exceeds the 79 characters limit' +\n ' but it is split into two lines')\n" + }, + { + "id": 31, + "text": "模块级别的函数和类定义之间用两个空行分隔，类内部的方法定义之间用一个空行分隔", + "detail": "缺陷类型：模块级别的函数和类定义之间没有用两个空行分隔，类内部的方法定义之间没有用一个空行分隔；修复方案：按照规范添加空行。", + "language": "Python", + "yes_example": "### 被判定为'模块级别的函数和类定义之间用两个空行分隔，类内部的方法定义之间用一个空行分隔'的例子\n<例子1>\ndef func1():\n pass\ndef func2():\n pass\n\n<例子2>\nclass MyClass:\n def method1(self):\n pass\n def method2(self):\n pass\n", + "no_example": "### 不能被判定为'模块级别的函数和类定义之间用两个空行分隔，类内部的方法定义之间用一个空行分隔'的例子\n<例子1>\ndef func1():\n pass\n\n\ndef func2():\n pass\n\n<例子2>\nclass MyClass:\n def method1(self):\n pass\n\n def method2(self):\n pass\n" + }, + { + "id": 32, + "text": "使用小写字母和下划线分隔的方式命名变量和函数", + "detail": "缺陷类型：变量和函数命名不符合小写字母和下划线分隔的方式；修复方案：使用小写字母和下划线分隔的方式命名。", + "language": "Python", + "yes_example": "### 被判定为'使用小写字母和下划线分隔的方式命名变量和函数'的例子\n<例子1>\ndef myFunction():\n pass\n\n<例子2>\nmyVariable = 10\n", + "no_example": "### 不能被判定为'使用小写字母和下划线分隔的方式命名变量和函数'的例子\n<例子1>\ndef my_function():\n pass\n\n<例子2>\nmy_variable = 10\n" + }, + { + "id": 33, + "text": "不允许使用print()函数来记录日志，使用logging模块等来记录日志", + "detail": "缺陷类型：使用print()函数记录日志；修复方案：使用logging模块记录日志。", + "language": "Python", + "yes_example": "### 被判定为'不允许使用print()函数来记录日志，使用logging模块等来记录日志'的例子\n<例子1>\nprint('Error occurred')\n", + "no_example": "### 不能被判定为'不允许使用print()函数来记录日志，使用logging模块等来记录日志'的例子\n<例子1>\nimport logging\nlogging.error('Error occurred')\n" + } +] \ No newline at end of file diff --git a/metagpt/ext/cr/utils/__init__.py b/metagpt/ext/cr/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/metagpt/ext/cr/utils/cleaner.py b/metagpt/ext/cr/utils/cleaner.py new file mode 100644 index 000000000..3215737c1 --- /dev/null +++ b/metagpt/ext/cr/utils/cleaner.py @@ -0,0 +1,68 @@ +"""Cleaner.""" + +from unidiff import Hunk, PatchedFile, PatchSet + +from metagpt.logs import logger + + +def rm_patch_useless_part(patch: PatchSet, used_suffix: list[str] = ["java", "py"]) -> PatchSet: + new_patch = PatchSet("") + useless_files = [] + for pfile in patch: + suffix = str(pfile.target_file).split(".")[-1] + if suffix not in used_suffix or pfile.is_removed_file or "test" in pfile.target_file.casefold(): + useless_files.append(pfile.path) + continue + new_patch.append(pfile) + logger.info(f"total file num: {len(patch)}, used file num: {len(new_patch)}, useless_files: {useless_files}") + return new_patch + + +def add_line_num_on_patch(patch: PatchSet, start_line_num: int = 1) -> PatchSet: + new_patch = PatchSet("") + lineno = start_line_num + for pfile in patch: + new_pfile = PatchedFile( + source=pfile.source_file, + target=pfile.target_file, + source_timestamp=pfile.source_timestamp, + target_timestamp=pfile.target_timestamp, + ) + for hunk in pfile: + arr = [str(line) for line in hunk] + new_hunk = Hunk( + src_start=hunk.source_start, + src_len=hunk.source_length, + tgt_start=hunk.target_start, + tgt_len=hunk.target_length, + section_header=hunk.section_header, + ) + + for line in arr: + # if len(line) > 0 and line[0] in ["+", "-"]: + # line = f"{lineno} {line}" + # lineno += 1 + line = f"{lineno} {line}" + lineno += 1 + new_hunk.append(line) + new_pfile.append(new_hunk) + new_patch.append(new_pfile) + return new_patch + + +def get_code_block_from_patch(patch: PatchSet, code_start_line: str, code_end_line: str) -> str: + line_arr = str(patch).split("\n") + code_arr = [] + add_line_tag = False + for line in line_arr: + if line.startswith(f"{code_start_line} "): + add_line_tag = True + + if add_line_tag: + new_line = " ".join(line.split(" ")[1:]) # rm line-no tag + code_arr.append(new_line) + + if line.startswith(f"{code_end_line} "): + add_line_tag = False + + return "\n".join(code_arr) diff --git a/metagpt/ext/cr/utils/schema.py b/metagpt/ext/cr/utils/schema.py new file mode 100644 index 000000000..beb27a07f --- /dev/null +++ b/metagpt/ext/cr/utils/schema.py @@ -0,0 +1,20 @@ +from typing import Literal + +from pydantic import BaseModel, Field + + +class Point(BaseModel): + id: int = Field(default=0, description="ID of the point.") + text: str = Field(default="", description="Content of the point.") + language: Literal["Python", "Java"] = Field( + default="Python", description="The programming language that the point corresponds to." + ) + file_path: str = Field(default="", description="The file that the points come from.") + start_line: int = Field(default=0, description="The starting line number that the point refers to.") + end_line: int = Field(default=0, description="The ending line number that the point refers to.") + detail: str = Field(default="", description="File content from start_line to end_line.") + yes_example: str = Field(default="", description="yes of point examples") + no_example: str = Field(default="", description="no of point examples") + + def rag_key(self) -> str: + return self.text diff --git a/metagpt/prompts/di/data_analyst.py b/metagpt/prompts/di/data_analyst.py index 27b247fb3..08b8d0df8 100644 --- a/metagpt/prompts/di/data_analyst.py +++ b/metagpt/prompts/di/data_analyst.py @@ -40,6 +40,7 @@ Some text indicating your thoughts, such as how you should update the plan statu ... ] ``` +Notice: your output JSON data section must start with **```json [** """ BROWSER_INSTRUCTION = """ diff --git a/metagpt/prompts/di/role_zero.py b/metagpt/prompts/di/role_zero.py index ea25aab82..83ad5cd8e 100644 --- a/metagpt/prompts/di/role_zero.py +++ b/metagpt/prompts/di/role_zero.py @@ -52,4 +52,15 @@ Some text indicating your thoughts, such as how you should update the plan statu ... ] ``` +Notice: your output JSON data section must start with **```json [** +""" +JSON_REPAIR_PROMPT = """ +## json data +{json_data} + +## Output Format +```json +Formatted JSON data +``` +Help check if there are any formatting issues with the JSON data? If so, please help format it """ diff --git a/metagpt/prompts/di/swe_agent.py b/metagpt/prompts/di/swe_agent.py new file mode 100644 index 000000000..ed1f8a011 --- /dev/null +++ b/metagpt/prompts/di/swe_agent.py @@ -0,0 +1,189 @@ +""" +This code is adapted from the examples provided in the SWE-agent project. +You can find the original examples from the SWE-agent project here: +https://github.com/princeton-nlp/SWE-agent/tree/main/config/configs +""" + +SWE_AGENT_SYSTEM_TEMPLATE = """ +SETTING: You are an autonomous programmer, and you're working directly in the environment line with a special interface. + +The special interface consists of a file editor that shows you {WINDOW} lines of a file at a time. + +Please note that THE EDIT COMMAND REQUIRES PROPER INDENTATION. Pay attention to the original indentation when replacing the function. +If you'd like to add the line ' print(x)' you must fully write that out, with all those spaces before the code! Indentation is important and code that is not indented correctly will fail and require fixing before it can be run. +Always review your changes post-edit to ensure they accurately reflect your intentions. If the changes are not as desired, don't hesitate to issue another command to correct them. + +Your output should always contain a section of reasoning and a command described in JSON format. + +Use \\n to represent line breaks, ensuring the command conforms to the JSON format and is displayed on a single line. Except for the `edit` command, each parameter of the command needs to be enclosed in single quotes. +As shown in the example below: + +First I'll start by using ls to see what files are in the current directory. Then maybe we can look at some relevant files to see what they look like. + +```json +{{ + "command_name": "Bash.run", + "args": {{ + "cmd": "ls -a" + }} +}} +``` + +You should only include a *SINGLE* command in the command section and then wait for a response from the shell before continuing with more discussion and commands. Everything you include in the DISCUSSION section will be saved for future reference. +If you'd like to issue two commands at once, PLEASE DO NOT DO THAT! Please instead first submit just the first command, and then after receiving a response you'll be able to issue the second command. +Remember, YOU CAN ONLY ENTER ONE COMMAND AT A TIME. You should always wait for feedback after every command. + +You can use any bash commands you want (e.g., find, grep, cat, ls, cd) or any custom special tools (including `edit`) by calling Bash.run. Edit all the files you need. +You should carefully observe the behavior and results of the previous action, and avoid triggering repeated errors. + +However, the Bash.run does NOT support interactive session commands (e.g. python, vim), so please do not invoke them. + +In addition to the terminal, I also provide additional tools. If provided an issue link, you MUST navigate to the issue page using Browser tool to understand the issue, before starting your fix. + +# INSTRUCTIONS: +Your first action must be to check if the repository exists at the current path. If it exists, navigate to the repository path. If the repository doesn't exist, please download it and then navigate to it. +All subsequent actions must be performed within this repository path. Do not leave this directory to execute any actions at any time. +Your terminal session has started, and you can use any bash commands or the special interface to help you. Edit all the files you need. +""" + +MINIMAL_EXAMPLE = """ +## Example of a actions trajectory +User Requirement and Issue: Fix the bug in the repo. Because the environment is not available, you DO NOT need to run and modify any existing test case files or add new test case files to ensure that the bug is fixed. + +### Read and understand issue(Require): +{{ + "command_name": "Browser.goto", + "args": {{ + "url": "https://github.com/geekan/MetaGPT/issues/1275" + }} +}} +-> + +### Locate issue(Require): Locate the issue in the code by searching for the relevant file, function, or class and open the file to view the code. +{{ + "command_name": "Bash.run", + "args": {{ + "cmd": "cd /workspace/django__django_3.0" + }} +}} +-> + +Bash.run(cmd='search_dir_and_preview ASCIIUsernameValidator') +{{ + "command_name": "Bash.run", + "args": {{ + "cmd": "open /workspace/django__django_3.0/django/contrib/auth/validators.py" + }} +}} +-> + +### Fix the Bug(Require): Fix the bug in the code by editing the relevant function, class or code snippet. +{{ + "command_name": "Bash.run", + "args": {{ + "cmd": "edit 10:20 < + +### Submit the Changes(Require): Submit the changes to the repository. +{{ + "command_name": "Bash.run", + "args": {{ + "cmd": "submit" + }} +}} +Bash.run(cmd='submit') +-> +{{ + "command_name": "end", +}} +""" + + +IMPORTANT_TIPS = """ +1. If you run a command and it doesn't work, try running a different command. A command that did not work once will not work the second time unless you modify it! + +2. If you open a file and need to get to an area around a specific line that is not in the first 100 lines, say line 583, don't just use the scroll_down command multiple times. Instead, use the goto 583 command. It's much quicker. + +3. Always make sure to look at the currently open file and the current working directory (which appears right after the currently open file). The currently open file might be in a different directory than the working directory! Note that some commands, such as 'create', open files, so they might change the current open file. + +4. When editing files, it is easy to accidentally specify a wrong line number or to write code with incorrect indentation. Always check the code after you issue an edit to make sure that it reflects what you wanted to accomplish. If it didn't, issue another command to fix it. + +5. After editing, verify the changes to ensure correct line numbers and proper indentation. Adhere to PEP8 standards for Python code. + +6. NOTE ABOUT THE EDIT COMMAND: Indentation really matters! When editing a file, make sure to insert appropriate indentation before each line! Ensuring the code adheres to PEP8 standards. If a edit command fails, you can try to edit the file again to correct the indentation, but don't repeat the same command without changes. + +7. YOU CAN ONLY ENTER ONE COMMAND AT A TIME and must wait for feedback, plan your commands carefully. + +8. You cannot use any interactive session commands (e.g. python, vim) in this environment, but you can write scripts and run them. E.g. you can write a python script and then run it with `python .py`. + +9. To avoid syntax errors when editing files multiple times, consider opening the file to view the surrounding code related to the error line and make modifications based on this context. + +10. When using the `edit` command, remember it operates within a closed range. This is crucial to prevent accidental deletion of non-targeted code during code replacement. + +11. Ensure to observe the currently open file and the current working directory, which is displayed right after the open file. The open file might be in a different directory than the working directory. Remember, commands like 'create' open files and might alter the current open file. + +12. Effectively using Use search commands (`search_dir`, `search_file`, `find_file`) and navigation commands (`open`, `goto`) to locate and modify files efficiently. Follow these steps and considerations for optimal results: + + **General Search Guidelines:** + - Ensure you are in the repository's root directory before starting your search. + - Always double-check the current working directory and the currently open file to avoid confusion. + - Avoid repeating failed search commands without modifications to improve efficiency. + + **Strategies for Searching and Navigating Files:** + + 1. **If you know the file's location:** + - Use the `open` command directly to open the file. + - Use `search_file` to find the `search_term` within the currently open file. + - Alternatively, use the `goto` command to jump to the specified line. + - **Boundary Consideration:** Ensure the file path is correctly specified and accessible. + + 2. **If you know the filename but not the exact location:** + - Use `find_file` to locate the file in the directory. + - Use `open` to open the file once located. + - Use `search_file` to find the `search_term` within the file. + - Use `goto` to jump to the specified line if needed. + - **Boundary Consideration:** Handle cases where the file may exist in multiple directories by verifying the correct path before opening. + + 3. **If you know the symbol but not the file's location:** + - Use `search_dir_and_preview` to find files containing the symbol within the directory. + - Review the search results to identify the relevant file(s). + - Use `open` to open the identified file. + - Use `search_file` to locate the `search_term` within the open file. + - Use `goto` to jump to the specified line. + - **Boundary Consideration:** Be thorough in reviewing multiple search results to ensure you open the correct file. Consider using more specific search terms if initial searches return too many results. + + **Search Tips:** + - The `` for `search_dir_and_preview`, `find_file`, or `search_file` should be an existing class name, function name, or file name. + - Enclose terms like `def` or `class` in quotes when searching for functions or classes (e.g., `search_dir_and_preview 'def apow'` or `search_file 'class Pow'`). + - Use wildcard characters (`*`, `?`) in search terms to broaden or narrow down your search scope. + - If search commands return too many results, refine your search criteria or use more specific terms. + - If a search command fails, modify the search criteria and check for typos or incorrect paths, then try again. + - Based on feedback of observation or bash command in trajectory to guide adjustments in your search strategy. + +13. If the task results in succeed, fail, or NO PROGRESS, output `submit`. + +14. If provided an issue link, you MUST go to the issue page using Browser tool to understand the issue before starting your fix. + +15. When the edit fails, try to enlarge the starting line. +""" + +NEXT_STEP_TEMPLATE = f""" +# Example of Output +These examples are provided to demonstrate the output style that expected to be several stages including Locate issue, Fix the bug, Test the fix(Optional), and Submit the changes. It is included to show you how to correctly use the interface. You do not need to follow exactly what is done in the Example. The separator is "-----". +----- Beginning of Examples ----- +{MINIMAL_EXAMPLE} +----- End of Examples ----- + +# IMPORTANT TIPS +{IMPORTANT_TIPS} + +# Output Next Step +The current bash state is: +(Open file: {{open_file}}) +(Current directory: {{working_dir}}) + +Avoid repeating the same command. Instead, please think about the current situation and provide the next bash command to execute in JSON format:" + +""" diff --git a/metagpt/prompts/di/team_leader.py b/metagpt/prompts/di/team_leader.py index 9304fd24d..484727936 100644 --- a/metagpt/prompts/di/team_leader.py +++ b/metagpt/prompts/di/team_leader.py @@ -21,6 +21,7 @@ Note: 4. If the requirement is a common-sense, logical, or math problem, you should respond directly without assigning any task to team members. 5. If you think the requirement is not clear or ambiguous, you should ask the user for clarification immediately. Assign tasks only after all info is clear. 6. It is helpful for Engineer to have both the system design and the project schedule for writing the code, so include paths of both files (if available) and remind Engineer to definitely read them when publishing message to Engineer. +7. If the requirement is writing a TRD and software framework, you should assign it to Architect. When publishing message to Architect, you should directly copy the full original user requirement. """ FINISH_CURRENT_TASK_CMD = """ diff --git a/metagpt/provider/base_llm.py b/metagpt/provider/base_llm.py index db2757ec3..4489c56c5 100644 --- a/metagpt/provider/base_llm.py +++ b/metagpt/provider/base_llm.py @@ -65,7 +65,7 @@ class BaseLLM(ABC): # image url or image base64 url = image if image.startswith("http") else f"data:image/jpeg;base64,{image}" # it can with multiple-image inputs - content.append({"type": "image_url", "image_url": url}) + content.append({"type": "image_url", "image_url": {"url": url}}) return {"role": "user", "content": content} def _assistant_msg(self, msg: str) -> dict[str, str]: diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 0263da989..a41c8b0a6 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -40,8 +40,17 @@ from metagpt.utils.token_counter import ( ) -@register_provider([LLMType.OPENAI, LLMType.FIREWORKS, LLMType.OPEN_LLM, LLMType.MOONSHOT, LLMType.MISTRAL, LLMType.YI, - LLMType.OPEN_ROUTER]) +@register_provider( + [ + LLMType.OPENAI, + LLMType.FIREWORKS, + LLMType.OPEN_LLM, + LLMType.MOONSHOT, + LLMType.MISTRAL, + LLMType.YI, + LLMType.OPEN_ROUTER, + ] +) class OpenAILLM(BaseLLM): """Check https://platform.openai.com/examples for examples""" diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 8650f2640..f640d4a87 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -8,6 +8,8 @@ from metagpt.actions import WritePRD from metagpt.actions.design_api import WriteDesign from metagpt.roles.di.role_zero import RoleZero +from metagpt.tools.libs.software_development import write_trd_and_framework +from metagpt.utils.common import tool2name class Architect(RoleZero): @@ -29,9 +31,14 @@ class Architect(RoleZero): "libraries. Use same language as user requirement" ) - instruction: str = """Use WriteDesign tool to write a system design document""" + instruction: str = """Use WriteDesign tool to write a system design document if a system design is required; Use `write_trd_and_framework` tool to write a software framework if a software framework is required;""" max_react_loop: int = 1 # FIXME: Read and edit files requires more steps, consider later - tools: list[str] = ["Editor:write,read,write_content", "RoleZero", "WriteDesign"] + tools: list[str] = [ + "Editor:write,read,write_content", + "RoleZero", + "WriteDesign", + write_trd_and_framework.__name__, + ] def __init__(self, **kwargs) -> None: super().__init__(**kwargs) @@ -45,11 +52,11 @@ class Architect(RoleZero): self._watch({WritePRD}) def _update_tool_execution(self): - wd = WriteDesign() + write_design = WriteDesign() + self.tool_execution_map.update(tool2name(WriteDesign, ["run"], write_design.run)) self.tool_execution_map.update( { - "WriteDesign.run": wd.run, - "WriteDesign": wd.run, # alias - "run": wd.run, # alias + write_trd_and_framework.__name__: write_trd_and_framework, + "run": write_design.run, # alias } ) diff --git a/metagpt/roles/di/engineer2.py b/metagpt/roles/di/engineer2.py index 8ea823c74..01bf826f4 100644 --- a/metagpt/roles/di/engineer2.py +++ b/metagpt/roles/di/engineer2.py @@ -12,7 +12,7 @@ class Engineer2(RoleZero): goal: str = "Take on game, app, and web development" instruction: str = ENGINEER2_INSTRUCTION - tools: str = ["Plan", "Editor:write,read", "RoleZero", "ReviewAndRewriteCode"] + tools: list[str] = ["Plan", "Editor:write,read", "RoleZero", "ReviewAndRewriteCode"] def _update_tool_execution(self): review = ReviewAndRewriteCode() diff --git a/metagpt/roles/di/role_zero.py b/metagpt/roles/di/role_zero.py index 01f792ed0..a52d72c8e 100644 --- a/metagpt/roles/di/role_zero.py +++ b/metagpt/roles/di/role_zero.py @@ -4,7 +4,7 @@ import inspect import json import re import traceback -from typing import Callable, Literal, Tuple +from typing import Callable, Dict, List, Literal, Tuple from metagpt.strategy.task_type import TaskType from pydantic import model_validator @@ -12,7 +12,11 @@ from pydantic import model_validator from metagpt.actions import Action from metagpt.actions.di.run_command import RunCommand from metagpt.logs import logger -from metagpt.prompts.di.role_zero import CMD_PROMPT, ROLE_INSTRUCTION +from metagpt.prompts.di.role_zero import ( + CMD_PROMPT, + JSON_REPAIR_PROMPT, + ROLE_INSTRUCTION, +) from metagpt.roles import Role from metagpt.schema import AIMessage, Message, UserMessage from metagpt.strategy.experience_retriever import DummyExpRetriever, ExpRetriever @@ -22,6 +26,7 @@ from metagpt.tools.libs.editor import Editor from metagpt.tools.tool_recommend import BM25ToolRecommender, ToolRecommender from metagpt.tools.tool_registry import register_tool from metagpt.utils.common import CodeParser +from metagpt.utils.repair_llm_raw_output import RepairType, repair_llm_raw_output from metagpt.utils.report import ThoughtReporter @@ -170,16 +175,15 @@ class RoleZero(Role): if self.use_fixed_sop: return await super()._act() - try: - commands = json.loads(CodeParser.parse_code(block=None, lang="json", text=self.command_rsp)) - except Exception as e: - tb = traceback.format_exc() - print(tb) - error_msg = UserMessage(content=str(e)) - self.rc.memory.add(error_msg) + commands, ok = await self._parse_commands() + if not ok: + error_msg = commands return error_msg + logger.info(f"Commands: \n{commands}") outputs = await self._run_commands(commands) + logger.info(f"Commands outputs: \n{outputs}") self.rc.memory.add(UserMessage(content=outputs)) + return AIMessage( content=f"Complete run with outputs: {outputs}", sent_from=self.name, @@ -206,6 +210,35 @@ class RoleZero(Role): actions_taken += 1 return rsp # return output from the last action + async def _parse_commands(self) -> Tuple[List[Dict], bool]: + """Retrieves commands from the Large Language Model (LLM). + + This function attempts to retrieve a list of commands from the LLM by + processing the response (`self.command_rsp`). It handles potential errors + during parsing and LLM response formats. + + Returns: + A tuple containing: + - A boolean flag indicating success (True) or failure (False). + """ + try: + commands = CodeParser.parse_code(block=None, lang="json", text=self.command_rsp) + commands = json.loads(repair_llm_raw_output(output=commands, req_keys=[None], repair_type=RepairType.JSON)) + except json.JSONDecodeError: + commands = await self.llm.aask(msg=JSON_REPAIR_PROMPT.format(json_data=self.command_rsp)) + commands = json.loads(CodeParser.parse_code(block=None, lang="json", text=commands)) + except Exception as e: + tb = traceback.format_exc() + print(tb) + error_msg = UserMessage(content=str(e)) + self.rc.memory.add(error_msg) + return error_msg, False + + # 为了对LLM不按格式生成进行容错 + if isinstance(commands, dict): + commands = commands["commands"] if "commands" in commands else [commands] + return commands, True + async def _run_commands(self, commands) -> str: outputs = [] for cmd in commands: diff --git a/metagpt/roles/di/swe_agent.py b/metagpt/roles/di/swe_agent.py new file mode 100644 index 000000000..d0458a22f --- /dev/null +++ b/metagpt/roles/di/swe_agent.py @@ -0,0 +1,110 @@ +import json +import os + +from pydantic import Field + +from metagpt.logs import logger +from metagpt.prompts.di.swe_agent import ( + MINIMAL_EXAMPLE, + NEXT_STEP_TEMPLATE, + SWE_AGENT_SYSTEM_TEMPLATE, +) +from metagpt.roles.di.role_zero import RoleZero +from metagpt.tools.libs.git import git_create_pull, git_push +from metagpt.tools.libs.terminal import Bash + + +class SWEAgent(RoleZero): + name: str = "Swen" + profile: str = "Issue Solver" + goal: str = "Resolve GitHub issue" + _bash_window_size: int = 100 + _system_msg: str = SWE_AGENT_SYSTEM_TEMPLATE + system_msg: list[str] = [_system_msg.format(WINDOW=_bash_window_size)] + _instruction: str = NEXT_STEP_TEMPLATE + tools: list[str] = [ + "Bash", + "Browser:goto,scroll", + "RoleZero", + "git_push", + "git_create_pull", + ] + terminal: Bash = Field(default_factory=Bash, exclude=True) + output_diff: str = "" + max_react_loop: int = 40 + run_eval: bool = False + + async def _think(self) -> bool: + self._update_system_msg() + self._format_instruction() + res = await super()._think() + if self.run_eval: + await self._parse_commands_for_eval() + return res + + def _update_tool_execution(self): + self.tool_execution_map.update( + { + "Bash.run": self.terminal.run, + "git_push": git_push, + "git_create_pull": git_create_pull, + } + ) + + def _update_system_msg(self): + """ + Sets the system message for the SWE agent. + + Sets the `_bash_window_size` from the environment variable `WINDOW` if it exists. + Formats the `_system_msg` template with the current `_bash_window_size`. + """ + if os.getenv("WINDOW"): + self._bash_window_size = int(os.getenv("WINDOW")) + self.system_msg = [self._system_msg.format(WINDOW=self._bash_window_size)] + + def _format_instruction(self): + """ + Formats the instruction message for the SWE agent. + + Runs the "state" command in the terminal, parses its output as JSON, + and uses it to format the `_instruction` template. + """ + state_output = self.terminal.run("state") + bash_state = json.loads(state_output) + + self.instruction = self._instruction.format( + WINDOW=self._bash_window_size, examples=MINIMAL_EXAMPLE, **bash_state + ).strip() + + return self.instruction + + async def _parse_commands_for_eval(self): + """ + Handles actions based on parsed commands. + + Parses commands, checks for a "submit" action, and generates a patch using `git diff`. + Stores the cleaned patch in `output_diff`. Logs any exceptions. + + This function is specifically added for SWE bench evaluation. + """ + # only import when evaluation is needed + from metagpt.tools.swe_agent_commands.swe_agent_utils import extract_patch + + commands, ok = await self._parse_commands() + if not ok: + return + for cmd in commands: + if "end" != cmd.get("command_name", ""): + return + try: + diff_output = self.terminal.run("git diff --cached") + clear_diff = extract_patch(diff_output) + logger.info(f"Diff output: \n{clear_diff}") + if clear_diff: + self.output_diff = clear_diff + + except Exception as e: + logger.error(f"Error during submission: {e}") + + def _retrieve_experience(self) -> str: + return MINIMAL_EXAMPLE diff --git a/metagpt/roles/di/team_leader.py b/metagpt/roles/di/team_leader.py index 2932dd7f0..7915edcf6 100644 --- a/metagpt/roles/di/team_leader.py +++ b/metagpt/roles/di/team_leader.py @@ -14,7 +14,7 @@ from metagpt.tools.tool_registry import register_tool @register_tool(include_functions=["publish_team_message"]) class TeamLeader(RoleZero): - name: str = "Tim" + name: str = "Mike" profile: str = "Team Leader" system_msg: list[str] = [SYSTEM_PROMPT] diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index cc8c82bf1..5f70bf390 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -9,9 +9,10 @@ from metagpt.actions import UserRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.actions.requirement_analysis.requirement.pic2txt import Pic2Txt from metagpt.roles.di.role_zero import RoleZero from metagpt.roles.role import RoleReactMode -from metagpt.utils.common import any_to_name, any_to_str +from metagpt.utils.common import any_to_name, any_to_str, tool2name from metagpt.utils.git_repository import GitRepository @@ -32,9 +33,9 @@ class ProductManager(RoleZero): constraints: str = "utilize the same language as the user requirements for seamless communication" todo_action: str = any_to_name(WritePRD) - instruction: str = """Use WritePRD tool to write PRD""" + instruction: str = """Use WritePRD tool to write PRD if a PRD is required; Use `Pic2Txt` tool to write out an intact textual user requirements if an intact textual user requiremnt is required given some images alongside the contextual textual descriptions;""" max_react_loop: int = 1 # FIXME: Read and edit files requires more steps, consider later - tools: list[str] = ["Editor:write,read,write_content", "RoleZero", "WritePRD"] + tools: list[str] = ["Editor:write,read,write_content", "RoleZero", "WritePRD", Pic2Txt.__name__] def __init__(self, **kwargs) -> None: super().__init__(**kwargs) @@ -47,12 +48,9 @@ class ProductManager(RoleZero): def _update_tool_execution(self): wp = WritePRD() - self.tool_execution_map.update( - { - "WritePRD.run": wp.run, - "WritePRD": wp.run, # alias - } - ) + self.tool_execution_map.update(tool2name(WritePRD, ["run"], wp.run)) + pic2txt = Pic2Txt() + self.tool_execution_map.update(tool2name(Pic2Txt, ["run"], pic2txt.run)) async def _think(self) -> bool: """Decide what to do""" diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index 6356a0faf..63ca4fcbd 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -14,7 +14,458 @@ class DummyExpRetriever(ExpRetriever): """A dummy experience retriever that returns empty string.""" def retrieve(self, context: str = "") -> str: - return "" + return self.EXAMPLE + + EXAMPLE: str = "" + + +class TRDAllExpRetriever(ExpRetriever): + def retrieve(self, context: str = "") -> str: + return self.EXAMPLE + + EXAMPLE: str = """ +## example 1 +User Requirement: Given some user requirements, write a software framework. +Explanation: Given a complete user requirement, to write a TRD and software framework, you must follow all of the following steps to complete the TRD output required by the user: 1. Call 'write_trd' to generate TRD; 2. Call 'write_framework' to implement TRD into the software framework. +```json +[ + { + "command_name": "write_trd_and_framework", + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Execute `write_trd_and_framework` to write a TRD and software framework based on user requirements", + "args": { + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "additional_technical_requirements": "These are additional technical requirements, balabala..." + } + } +] +``` +## example 2 +User Requirement: Given some user requirements, write a software framework. +Explanation: Given a complete user requirement, to write a software framework, you must follow all of the following steps to complete the TRD output required by the user: 1. Call 'write_trd' to generate TRD; 2. Call 'write_framework' to implement TRD into the software framework. +```json +[ + { + "command_name": "write_trd", + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Execute `write_trd` to write the TRD based on user requirements", + "args": { + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + } + }, + { + "command_name": "write_framework", + "task_id": "2", + "dependent_task_ids": ["1"], + "instruction": "Execute `write_framework` to write the framework based on the TRD", + "args": { + "use_case_actors": "These are actors involved in the use case, balabala...", + "trd": " returned by `write_trd`", + "additional_technical_requirements": "These are additional technical requirements, balabala..." + } + } +] +``` +## example 3 +User Requirement: Given some user requirements, write a TRD, and implement the TRD within a software framework. +Explanation: + Given a complete requirement, 要写TRD需要follow如下步骤： + 1. 调用`CompressExternalInterfaces.run`，从acknowledgement中抽取external interfaces的信息； + 2. 按顺序执行如下步骤： + 2.1. 执行`DetectInteraction.run`; + 2.2. 执行`WriteTRD.run`; + 2.3. 执行`EvaluateTRD.run`; + 2.4. 检查`EvaluateTRD.run`的结果： + 2.4.1. 如果`EvaluateTRD.run`的结果被判定为pass，则执行步骤3； + 2.4.2. 如果`EvaluateTRD.run`的结果被判定为deny,则继续执行步骤2； + 3. 按顺序执行如下步骤： + 3.1. 执行`WriteFramework.run`; + 3.2. 执行`EvaluateFramework.run`; + 3.3. 检查`EvaluateFramework.run`的结果： + 3.3.1. 如果`EvaluateFramework.run`的结果被判定为pass，则执行步骤4； + 3.3.2. 如果`EvaluateFramework.run`的结果被判定为deny,则继续执行步骤3； + 3.3.3. 如果已经重复执行步骤3超过9次，则执行步骤4； + 4. 执行`save_framework`,将`WriteFramework.run`的结果保存下来； +```json +[ + { + "command_name": "CompressExternalInterfaces.run", + "args": { + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Execute `DetectInteraction.run` to extract external interfaces information from acknowledgement.", + "acknowledge": "## Interfaces\n balabala..." + } + }, + { + "command_name": "DetectInteraction.run", + "args": { + "task_id": "2", + "dependent_task_ids": ["1"], + "instruction": "Execute `DetectInteraction.run` to extract external interfaces information from acknowledgement.", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + } + }, + { + "command_name": "WriteTRD.run", + "args": { + "task_id": "3", + "dependent_task_ids": ["2"], + "instruction": "Execute `WriteTRD.run` to write TRD", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "available_external_interfaces": " returned by `CompressExternalInterfaces.run`", + "interaction_events": " returned by `DetectInteraction.run`" + } + }, + { + "command_name": "EvaluateTRD.run", + "args": { + "task_id": "4", + "dependent_task_ids": ["3"], + "instruction": "Execute `EvaluateTRD.run` to evaluate the TRD", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "available_external_interfaces": " returned by `CompressExternalInterfaces.run`", + "interaction_events": "", + "trd": " returned by `EvaluateTRD.run`" + } + }, + { + "command_name": "DetectInteraction.run", + "args": { + "task_id": "5", + "dependent_task_ids": ["4"], + "instruction": "Execute `DetectInteraction.run` to extract external interfaces information from acknowledgement.", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "evaluation_conclusion": " returned by `EvaluateTRD.run`" + } + }, + { + "command_name": "WriteTRD.run", + "args": { + "task_id": "6", + "dependent_task_ids": ["5"], + "instruction": "Execute `WriteTRD.run` to write TRD", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "available_external_interfaces": " returned by `CompressExternalInterfaces.run`", + "interaction_events": " returned by `DetectInteraction.run`", + "previous_version_trd": " returned by `WriteTRD.run`" + } + }, + { + "command_name": "EvaluateTRD.run", + "args": { + "task_id": "7", + "dependent_task_ids": ["6"], + "instruction": "Execute `EvaluateTRD.run` to evaluate the TRD", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "available_external_interfaces": " returned by `CompressExternalInterfaces.run`", + "interaction_events": " returned by `DetectInteraction.run`", + "trd": " returned by `WriteTRD.run`", + } + }, + { + "command_name": "WriteFramework.run", + "args": { + "task_id": "8", + "dependent_task_ids": ["7"], + "instruction": "Execute `WriteFramework.run` to write a software framework according to the TRD", + "use_case_actors": "These are actors involved in the use case, balabala...", + "trd": " returned by `WriteTRD.run`", + "acknowledge": "## Interfaces\n balabala...", + "additional_technical_requirements": "These are additional technical requirements, balabala...", + } + }, + { + "command_name": "EvaluateFramework.run", + "args": { + "task_id": "9", + "dependent_task_ids": ["8"], + "instruction": "Execute `EvaluateFramework.run` to evaluate the software framework returned by `WriteFramework.run`", + "use_case_actors": "These are actors involved in the use case, balabala...", + "trd": " returned by `WriteTRD.run`", + "acknowledge": "## Interfaces\n balabala...", + "legacy_output": " returned by `WriteFramework.run`", + "additional_technical_requirements": "These are additional technical requirements, balabala...", + } + }, + { + "command_name": "WriteFramework.run", + "args": { + "task_id": "10", + "dependent_task_ids": ["9"], + "instruction": "Execute `WriteFramework.run` to write a software framework according to the TRD", + "use_case_actors": "These are actors involved in the use case, balabala...", + "trd": " returned by `WriteTRD.run`", + "acknowledge": "## Interfaces\n balabala...", + "additional_technical_requirements": "These are additional technical requirements, balabala...", + } + }, + { + "command_name": "EvaluateFramework.run", + "args": { + "task_id": "11", + "dependent_task_ids": ["10"], + "instruction": "Execute `EvaluateFramework.run` to evaluate the software framework returned by `WriteFramework.run`", + "use_case_actors": "These are actors involved in the use case, balabala...", + "trd": " returned by `WriteTRD.run`", + "acknowledge": "## Interfaces\n balabala...", + "legacy_output": " returned by `WriteFramework.run`", + "additional_technical_requirements": "These are additional technical requirements, balabala...", + } + }, + { + "command_name": "save_framework", + "args": { + "task_id": "12", + "dependent_task_ids": ["11"], + "instruction": "Execute `save_framework` to save the software framework returned by `WriteFramework.run`", + "dir_data": " returned by `WriteFramework.run`", + } + } +] +``` + """ + + +class TRDToolExpRetriever(ExpRetriever): + """A TRD-related experience retriever that returns empty string.""" + + def retrieve(self, context: str = "") -> str: + return self.EXAMPLE + + EXAMPLE: str = """ +## example 1 +User Requirement: Given some user requirements, write a software framework. +Explanation: Given a complete user requirement, to write a TRD and software framework, you must follow all of the following steps to complete the TRD output required by the user: 1. Call 'write_trd' to generate TRD; 2. Call 'write_framework' to implement TRD into the software framework. +```json +[ + { + "command_name": "write_trd_and_framework", + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Execute `write_trd_and_framework` to write a TRD and software framework based on user requirements", + "args": { + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "additional_technical_requirements": "These are additional technical requirements, balabala..." + } + } +] + """ + # EXAMPLE: str = """ + # ## example 1 + # User Requirement: Given some user requirements, write a software framework. + # Explanation: Given a complete user requirement, to write a software framework, you must follow all of the following steps to complete the TRD output required by the user: 1. Call 'write_trd' to generate TRD; 2. Call 'write_framework' to implement TRD into the software framework. + # ```json + # [ + # { + # "command_name": "write_trd", + # "task_id": "1", + # "dependent_task_ids": [], + # "instruction": "Execute `write_trd` to write the TRD based on user requirements", + # "args": { + # "user_requirements": "This is user requirement balabala...", + # "use_case_actors": "These are actors involved in the use case, balabala...", + # } + # }, + # { + # "command_name": "write_framework", + # "task_id": "2", + # "dependent_task_ids": ["1"], + # "instruction": "Execute `write_framework` to write the framework based on the TRD", + # "args": { + # "use_case_actors": "These are actors involved in the use case, balabala...", + # "trd": " returned by `write_trd`", + # "additional_technical_requirements": "These are additional technical requirements, balabala..." + # } + # } + # ] + # ``` + # """ + + +class TRDExpRetriever(ExpRetriever): + """A TRD-related experience retriever that returns empty string.""" + + def retrieve(self, context: str = "") -> str: + return self.EXAMPLE + + EXAMPLE: str = """ + ## example 1 + User Requirement: Given some user requirements, write a TRD, and implement the TRD within a software framework. + Explanation: + Given a complete requirement, 要写TRD需要follow如下步骤： + 1. 调用`CompressExternalInterfaces.run`，从acknowledgement中抽取external interfaces的信息； + 2. 按顺序执行如下步骤： + 2.1. 执行`DetectInteraction.run`; + 2.2. 执行`WriteTRD.run`; + 2.3. 执行`EvaluateTRD.run`; + 2.4. 检查`EvaluateTRD.run`的结果： + 2.4.1. 如果`EvaluateTRD.run`的结果被判定为pass，则执行步骤3； + 2.4.2. 如果`EvaluateTRD.run`的结果被判定为deny,则继续执行步骤2； + 3. 按顺序执行如下步骤： + 3.1. 执行`WriteFramework.run`; + 3.2. 执行`EvaluateFramework.run`; + 3.3. 检查`EvaluateFramework.run`的结果： + 3.3.1. 如果`EvaluateFramework.run`的结果被判定为pass，则执行步骤4； + 3.3.2. 如果`EvaluateFramework.run`的结果被判定为deny,则继续执行步骤3； + 3.3.3. 如果已经重复执行步骤3超过9次，则执行步骤4； + 4. 执行`save_framework`,将`WriteFramework.run`的结果保存下来； + ```json + [ + { + "command_name": "CompressExternalInterfaces.run", + "args": { + "task_id": "1", + "dependent_task_ids": [], + "instruction": "Execute `DetectInteraction.run` to extract external interfaces information from acknowledgement.", + "acknowledge": "## Interfaces\n balabala..." + } + }, + { + "command_name": "DetectInteraction.run", + "args": { + "task_id": "2", + "dependent_task_ids": ["1"], + "instruction": "Execute `DetectInteraction.run` to extract external interfaces information from acknowledgement.", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + } + }, + { + "command_name": "WriteTRD.run", + "args": { + "task_id": "3", + "dependent_task_ids": ["2"], + "instruction": "Execute `WriteTRD.run` to write TRD", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "available_external_interfaces": " returned by `CompressExternalInterfaces.run`", + "interaction_events": " returned by `DetectInteraction.run`" + } + }, + { + "command_name": "EvaluateTRD.run", + "args": { + "task_id": "4", + "dependent_task_ids": ["3"], + "instruction": "Execute `EvaluateTRD.run` to evaluate the TRD", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "available_external_interfaces": " returned by `CompressExternalInterfaces.run`", + "interaction_events": "", + "trd": " returned by `EvaluateTRD.run`" + } + }, + { + "command_name": "DetectInteraction.run", + "args": { + "task_id": "5", + "dependent_task_ids": ["4"], + "instruction": "Execute `DetectInteraction.run` to extract external interfaces information from acknowledgement.", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "evaluation_conclusion": " returned by `EvaluateTRD.run`" + } + }, + { + "command_name": "WriteTRD.run", + "args": { + "task_id": "6", + "dependent_task_ids": ["5"], + "instruction": "Execute `WriteTRD.run` to write TRD", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "available_external_interfaces": " returned by `CompressExternalInterfaces.run`", + "interaction_events": " returned by `DetectInteraction.run`", + "previous_version_trd": " returned by `WriteTRD.run`" + } + }, + { + "command_name": "EvaluateTRD.run", + "args": { + "task_id": "7", + "dependent_task_ids": ["6"], + "instruction": "Execute `EvaluateTRD.run` to evaluate the TRD", + "user_requirements": "This is user requirement balabala...", + "use_case_actors": "These are actors involved in the use case, balabala...", + "available_external_interfaces": " returned by `CompressExternalInterfaces.run`", + "interaction_events": " returned by `DetectInteraction.run`", + "trd": " returned by `WriteTRD.run`", + } + }, + { + "command_name": "WriteFramework.run", + "args": { + "task_id": "8", + "dependent_task_ids": ["7"], + "instruction": "Execute `WriteFramework.run` to write a software framework according to the TRD", + "use_case_actors": "These are actors involved in the use case, balabala...", + "trd": " returned by `WriteTRD.run`", + "acknowledge": "## Interfaces\n balabala...", + "additional_technical_requirements": "These are additional technical requirements, balabala...", + } + }, + { + "command_name": "EvaluateFramework.run", + "args": { + "task_id": "9", + "dependent_task_ids": ["8"], + "instruction": "Execute `EvaluateFramework.run` to evaluate the software framework returned by `WriteFramework.run`", + "use_case_actors": "These are actors involved in the use case, balabala...", + "trd": " returned by `WriteTRD.run`", + "acknowledge": "## Interfaces\n balabala...", + "legacy_output": " returned by `WriteFramework.run`", + "additional_technical_requirements": "These are additional technical requirements, balabala...", + } + }, + { + "command_name": "WriteFramework.run", + "args": { + "task_id": "10", + "dependent_task_ids": ["9"], + "instruction": "Execute `WriteFramework.run` to write a software framework according to the TRD", + "use_case_actors": "These are actors involved in the use case, balabala...", + "trd": " returned by `WriteTRD.run`", + "acknowledge": "## Interfaces\n balabala...", + "additional_technical_requirements": "These are additional technical requirements, balabala...", + } + }, + { + "command_name": "EvaluateFramework.run", + "args": { + "task_id": "11", + "dependent_task_ids": ["10"], + "instruction": "Execute `EvaluateFramework.run` to evaluate the software framework returned by `WriteFramework.run`", + "use_case_actors": "These are actors involved in the use case, balabala...", + "trd": " returned by `WriteTRD.run`", + "acknowledge": "## Interfaces\n balabala...", + "legacy_output": " returned by `WriteFramework.run`", + "additional_technical_requirements": "These are additional technical requirements, balabala...", + } + }, + { + "command_name": "save_framework", + "args": { + "task_id": "12", + "dependent_task_ids": ["11"], + "instruction": "Execute `save_framework` to save the software framework returned by `WriteFramework.run`", + "dir_data": " returned by `WriteFramework.run`", + } + } + ] + ``` + """ TL_EXAMPLE = """ diff --git a/metagpt/strategy/thinking_command.py b/metagpt/strategy/thinking_command.py index 14fdf5950..f08afa448 100644 --- a/metagpt/strategy/thinking_command.py +++ b/metagpt/strategy/thinking_command.py @@ -100,7 +100,8 @@ def run_plan_command(role: Role, cmd: list[dict]): elif cmd["command_name"] == Command.FINISH_CURRENT_TASK.cmd_name: if role.planner.plan.is_plan_finished(): return - role.planner.plan.current_task.update_task_result(task_result=role.task_result) + if role.task_result: + role.planner.plan.current_task.update_task_result(task_result=role.task_result) role.planner.plan.finish_current_task() role.rc.working_memory.clear() diff --git a/metagpt/tools/libs/browser.py b/metagpt/tools/libs/browser.py index c6ea71bd5..864996e8c 100644 --- a/metagpt/tools/libs/browser.py +++ b/metagpt/tools/libs/browser.py @@ -122,6 +122,8 @@ class Browser: async def goto(self, url: str, timeout: float = 30000): """Navigate to a specific URL.""" + if self.page is None: + await self.start() async with self.reporter as reporter: await reporter.async_report(url, "url") await self.page.goto(url, timeout=timeout) diff --git a/metagpt/tools/libs/cr.py b/metagpt/tools/libs/cr.py new file mode 100644 index 000000000..7f9a4716c --- /dev/null +++ b/metagpt/tools/libs/cr.py @@ -0,0 +1,90 @@ +import json +from pathlib import Path +from typing import Optional + +import aiofiles +from unidiff import PatchSet + +import metagpt.ext.cr +from metagpt.ext.cr.actions.code_review import CodeReview as CodeReview_ +from metagpt.ext.cr.actions.modify_code import ModifyCode +from metagpt.ext.cr.utils.schema import Point +from metagpt.tools.libs.browser import Browser +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.report import EditorReporter + + +@register_tool(tags=["codereview"], include_functions=["review", "fix"]) +class CodeReview: + """Review and fix the patch content from the pull request URL or a file.""" + + async def review( + self, + patch_path: str, + cr_output_file: str, + cr_point_file: Optional[str] = None, + ) -> str: + """Review a PR and save code review comments. + + Args: + patch_path: The local path of the patch file or the url of the pull request. Example: "/data/xxx-pr-1.patch", "https://github.com/xx/XX/pull/1362" + cr_output_file: Output file path where code review comments will be saved. Example: "cr/xxx-pr-1.json" + cr_point_file: File path for specifying code review points. Defaults to a predefined file. + """ + patch = await self._get_patch_content(patch_path) + cr_point_file = cr_point_file if cr_point_file else Path(metagpt.ext.cr.__file__).parent / "points.json" + async with aiofiles.open(cr_point_file, "rb") as f: + cr_point_content = await f.read() + cr_points = [Point(**i) for i in json.loads(cr_point_content)] + + async with EditorReporter(enable_llm_stream=True) as reporter: + src_path = cr_output_file + cr_output_path = Path(cr_output_file) + await reporter.async_report( + {"type": "CodeReview", "src_path": src_path, "filename": cr_output_path.name}, "meta" + ) + comments = await CodeReview_().run(patch, cr_points) + cr_output_path.parent.mkdir(exist_ok=True, parents=True) + async with aiofiles.open(cr_output_path, "w") as f: + await f.write(json.dumps(comments, ensure_ascii=False)) + await reporter.async_report(cr_output_path) + + return f"The number of defects: {len(comments)} and the comments are stored in {cr_output_file}" + + async def fix( + self, + patch_path: str, + cr_file: str, + output_dir: str, + ) -> str: + """Fix the patch content based on code review comments. + + Args: + patch_path: The local path of the patch file or the url of the pull request. + cr_file: File path where code review comments are stored. + output_dir: File path where code review comments are stored. + """ + patch = await self._get_patch_content(patch_path) + async with aiofiles.open(cr_file, "r") as f: + comments = json.loads(await f.read()) + await ModifyCode(pr="").run(patch, comments, output_dir) + return f"The fixed patch files store in {output_dir}" + + async def _get_patch_content(self, patch_path): + if patch_path.startswith(("https://", "http://")): + # async with aiohttp.ClientSession(trust_env=True) as client: + # async with client.get(f"{patch_path}.diff", ) as resp: + # patch_file_content = await resp.text() + browser = Browser() + browser.proxy = {"server": "http://127.0.0.1:20172"} + async with browser: + await browser.goto(f"{patch_path}.diff") + patch_file_content = await browser.page.content() + + else: + async with aiofiles.open(patch_path) as f: + patch_file_content = await f.read() + await EditorReporter().async_report(patch_path) + + patch: PatchSet = PatchSet(patch_file_content) + return patch diff --git a/metagpt/tools/libs/git.py b/metagpt/tools/libs/git.py index 740cb81f9..4606b5815 100644 --- a/metagpt/tools/libs/git.py +++ b/metagpt/tools/libs/git.py @@ -14,7 +14,7 @@ from metagpt.tools.tool_registry import register_tool @register_tool(tags=["software development", "git", "Commit the changes and push to remote git repository."]) async def git_push( local_path: Union[str, Path], - access_token: str, + app_name: str, comments: str = "Commit", new_branch: str = "", ) -> "GitBranch": @@ -23,7 +23,7 @@ async def git_push( Args: local_path (Union[str, Path]): The path to the local Git repository. - access_token (str): The access token for authentication. Use `get_env` to get access token. + app_name (str): The name of the application where the repository is hosted. For example, "github", "gitlab", "bitbucket", etc. comments (str, optional): The commit message to use. Defaults to "Commit". new_branch (str, optional): The name of the new branch to create and push changes to. If not provided, changes will be pushed to the current branch. Defaults to "". @@ -36,11 +36,10 @@ async def git_push( Example: >>> url = "https://github.com/iorisa/snake-game.git" >>> local_path = await git_clone(url=url) - >>> from metagpt.tools.libs import get_env - >>> access_token = await get_env(key="access_token", app_name="github") # Read access token from enviroment variables. + >>> app_name="github" >>> comments = "Archive" >>> new_branch = "feature/new" - >>> branch = await git_push(local_path=local_path, access_token=access_token, comments=comments, new_branch=new_branch) + >>> branch = await git_push(local_path=local_path, app_name=app_name, comments=comments, new_branch=new_branch) >>> base = branch.base >>> head = branch.head >>> repo_name = branch.repo_name @@ -48,12 +47,15 @@ async def git_push( base branch:'master', head branch:'feature/new', repo_name:'iorisa/snake-game' """ + from metagpt.tools.libs import get_env from metagpt.utils.git_repository import GitRepository if not GitRepository.is_git_dir(local_path): raise ValueError("Invalid local git repository") repo = GitRepository(local_path=local_path, auto_init=False) + # Read access token from environment variables. + access_token = await get_env(key="access_token", app_name=app_name) branch = await repo.push(new_branch=new_branch, comments=comments, access_token=access_token) return branch @@ -63,7 +65,7 @@ async def git_create_pull( base: str, head: str, base_repo_name: str, - access_token: str, + app_name: str, head_repo_name: Optional[str] = None, title: Optional[str] = None, body: Optional[str] = None, @@ -76,43 +78,13 @@ async def git_create_pull( base (str): The base branch of the pull request. head (str): The head branch of the pull request. base_repo_name (str): The full repository name (user/repo) where the pull request will be created. - access_token (str): The access token for authentication. Use `get_env` to get access token. + app_name (str): The name of the application where the repository is hosted. For example, "github", "gitlab", "bitbucket", etc. head_repo_name (Optional[str], optional): The full repository name (user/repo) where the pull request will merge from. Defaults to None. title (Optional[str], optional): The title of the pull request. Defaults to None. body (Optional[str], optional): The body of the pull request. Defaults to None. issue (Optional[Issue], optional): The related issue of the pull request. Defaults to None. Example: - >>> # push and create pull - >>> url = "https://github.com/iorisa/snake-game.git" - >>> local_path = await git_clone(url=url) - >>> from metagpt.tools.libs import get_env - >>> access_token = await get_env(key="access_token", app_name="github") - >>> comments = "Archive" - >>> new_branch = "feature/new" - >>> branch = await git_push(local_path=local_path, access_token=access_token, comments=comments, new_branch=new_branch) - >>> base = branch.base - >>> head = branch.head - >>> repo_name = branch.repo_name - >>> print(f"base branch:'{base}', head branch:'{head}', repo_name:'{repo_name}'") - base branch:'master', head branch:'feature/new', repo_name:'iorisa/snake-game' - >>> title = "feat: modify http lib", - >>> body = "Change HTTP library used to send requests" - >>> pr = await git_create_pull( - >>> base_repo_name=repo_name, - >>> base=base, - >>> head=head, - >>> title=title, - >>> body=body, - >>> access_token=access_token, - >>> ) - >>> if isinstance(pr, PullRequest): - >>> print(pr) - PullRequest("feat: modify http lib") - >>> if isinstance(pr, str): - >>> print(f"Visit this url to create a new pull request: '{pr}'") - Visit this url to create a new pull request: 'https://github.com/iorisa/snake-game/compare/master...feature/new' - >>> # create pull request >>> base_repo_name = "geekan/MetaGPT" >>> head_repo_name = "ioris/MetaGPT" @@ -120,8 +92,7 @@ async def git_create_pull( >>> head = "feature/http" >>> title = "feat: modify http lib", >>> body = "Change HTTP library used to send requests" - >>> from metagpt.tools.libs import get_env - >>> access_token = await get_env(key="access_token", app_name="github") + >>> app_name = "github" >>> pr = await git_create_pull( >>> base_repo_name=base_repo_name, >>> head_repo_name=head_repo_name, @@ -129,7 +100,7 @@ async def git_create_pull( >>> head=head, >>> title=title, >>> body=body, - >>> access_token=access_token, + >>> app_name=app_name, >>> ) >>> if isinstance(pr, PullRequest): >>> print(pr) @@ -141,8 +112,11 @@ async def git_create_pull( Returns: PullRequest: The created pull request. """ + + from metagpt.tools.libs import get_env from metagpt.utils.git_repository import GitRepository + access_token = await get_env(key="access_token", app_name=app_name) return await GitRepository.create_pull( base=base, head=head, diff --git a/metagpt/tools/libs/software_development.py b/metagpt/tools/libs/software_development.py index a48e4a191..1f8538dfc 100644 --- a/metagpt/tools/libs/software_development.py +++ b/metagpt/tools/libs/software_development.py @@ -2,10 +2,28 @@ # -*- coding: utf-8 -*- from __future__ import annotations +import uuid +from datetime import datetime from pathlib import Path +from typing import Optional -from metagpt.const import ASSISTANT_ALIAS -from metagpt.logs import ToolLogItem, log_tool_output +from metagpt.actions.requirement_analysis.framework import ( + EvaluateFramework, + WriteFramework, + save_framework, +) +from metagpt.actions.requirement_analysis.trd import ( + CompressExternalInterfaces, + DetectInteraction, + EvaluateTRD, + WriteTRD, +) +from metagpt.const import ASSISTANT_ALIAS, DEFAULT_WORKSPACE_ROOT, TEST_DATA_PATH +from metagpt.context import Context +from metagpt.logs import ToolLogItem, log_tool_output, logger +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import aread +from metagpt.utils.cost_manager import CostManager async def import_git_repo(url: str) -> Path: @@ -42,3 +60,201 @@ async def import_git_repo(url: str) -> Path: log_tool_output(output=outputs, tool_name=import_git_repo.__name__) return ctx.repo.workdir + + +async def extract_external_interfaces(acknowledge: str) -> str: + """ + Extracts and compresses information about external system interfaces from a given acknowledgement text. + + Args: + acknowledge (str): A natural text of acknowledgement containing details about external system interfaces. + + Returns: + str: A compressed version of the information about external system interfaces. + + Example: + >>> acknowledge = "## Interfaces\\n..." + >>> external_interfaces = await extract_external_interfaces(acknowledge=acknowledge) + >>> print(external_interfaces) + ```json\n[\n{\n"id": 1,\n"inputs": {... + """ + compress_acknowledge = CompressExternalInterfaces() + return await compress_acknowledge.run(acknowledge=acknowledge) + + +async def mock_asearch_acknowledgement(use_case_actors: str): + return await aread(filename=TEST_DATA_PATH / "requirements/1.acknowledge.md") + + +@register_tool(tags=["system design", "write trd", "Write a TRD"]) +async def write_trd( + use_case_actors: str, + user_requirements: str, + investment: float = 10, + context: Optional[Context] = None, +) -> str: + """ + Handles the writing of a Technical Requirements Document (TRD) based on user requirements. + + Args: + user_requirements (str): The new/incremental user requirements. + use_case_actors (str): Description of the actors involved in the use case. + investment (float): Budget. Automatically stops optimizing TRD when the budget is overdrawn. + context (Context, optional): The context configuration. Default is None. + Returns: + str: The newly created TRD. + + Example: + >>> # Given a new user requirements, write out a new TRD. + >>> user_requirements = "Write a 'snake game' TRD." + >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;" + >>> investment = 10.0 + >>> trd = await write_trd( + >>> user_requirements=user_requirements, + >>> use_case_actors=use_case_actors, + >>> investment=investment, + >>> ) + >>> print(trd) + ## Technical Requirements Document\n ... + """ + context = context or Context(cost_manager=CostManager(max_budget=investment)) + compress_acknowledge = CompressExternalInterfaces() + acknowledgement = await mock_asearch_acknowledgement(use_case_actors) # Replaced by acknowledgement_repo later. + external_interfaces = await compress_acknowledge.run(acknowledge=acknowledgement) + detect_interaction = DetectInteraction(context=context) + w_trd = WriteTRD(context=context) + evaluate_trd = EvaluateTRD(context=context) + is_pass = False + evaluation_conclusion = "" + interaction_events = "" + trd = "" + while not is_pass and (context.cost_manager.total_cost < context.cost_manager.max_budget): + interaction_events = await detect_interaction.run( + user_requirements=user_requirements, + use_case_actors=use_case_actors, + legacy_interaction_events=interaction_events, + evaluation_conclusion=evaluation_conclusion, + ) + trd = await w_trd.run( + user_requirements=user_requirements, + use_case_actors=use_case_actors, + available_external_interfaces=external_interfaces, + evaluation_conclusion=evaluation_conclusion, + interaction_events=interaction_events, + previous_version_trd=trd, + ) + evaluation = await evaluate_trd.run( + user_requirements=user_requirements, + use_case_actors=use_case_actors, + trd=trd, + interaction_events=interaction_events, + ) + is_pass = evaluation.is_pass + evaluation_conclusion = evaluation.conclusion + + return trd + + +@register_tool(tags=["system design", "write software framework", "Write a software framework based on a TRD"]) +async def write_framework( + use_case_actors: str, + trd: str, + additional_technical_requirements: str, + output_dir: Optional[str] = "", + investment: float = 20.0, + context: Optional[Context] = None, + max_loop: int = 20, +) -> str: + """ + Run the action to generate a software framework based on the provided TRD and related information. + + Args: + use_case_actors (str): Description of the use case actors involved. + trd (str): Technical Requirements Document detailing the requirements. + additional_technical_requirements (str): Any additional technical requirements. + output_dir (str, optional): Path to save the software framework files. Default is en empty string. + investment (float): Budget. Automatically stops optimizing TRD when the budget is overdrawn. + context (Context, optional): The context configuration. Default is None. + max_loop(int, optional): Acts as a safety exit valve when cost statistics fail. Default is 20. + + Returns: + str: The generated software framework as a string of pathnames. + + Example: + >>> use_case_actors = "- Actor: game player;\\n- System: snake game; \\n- External System: game center;" + >>> trd = "## TRD\\n..." + >>> additional_technical_requirements = "Using Java language, ..." + >>> investment = 15.0 + >>> framework = await write_framework( + >>> use_case_actors=use_case_actors, + >>> trd=trd, + >>> additional_technical_requirements=constraint, + >>> investment=investment, + >>> ) + >>> print(framework) + [{"path":"balabala", "filename":"...", ... + """ + context = context or Context(cost_manager=CostManager(max_budget=investment)) + write_framework = WriteFramework(context=context) + evaluate_framework = EvaluateFramework(context=context) + is_pass = False + framework = "" + evaluation_conclusion = "" + acknowledgement = await mock_asearch_acknowledgement(use_case_actors) # Replaced by acknowledgement_repo later. + loop_count = 0 + output_dir = ( + Path(output_dir) + if output_dir + else DEFAULT_WORKSPACE_ROOT / (datetime.now().strftime("%Y%m%d%H%M%ST") + uuid.uuid4().hex[0:8]) + ) + file_list = [] + while not is_pass and (context.cost_manager.total_cost < context.cost_manager.max_budget): + try: + framework = await write_framework.run( + use_case_actors=use_case_actors, + trd=trd, + acknowledge=acknowledgement, + legacy_output=framework, + evaluation_conclusion=evaluation_conclusion, + additional_technical_requirements=additional_technical_requirements, + ) + except Exception as e: + logger.info(f"{e}") + break + evaluation = await evaluate_framework.run( + use_case_actors=use_case_actors, + trd=trd, + acknowledge=acknowledgement, + legacy_output=framework, + additional_technical_requirements=additional_technical_requirements, + ) + is_pass = evaluation.is_pass + evaluation_conclusion = evaluation.conclusion + loop_count += 1 + logger.info(f"Loop {loop_count}") + if context.cost_manager.total_cost < 1 and loop_count > max_loop: + break + file_list = await save_framework(dir_data=framework, trd=trd, output_dir=output_dir) + logger.info(f"Output:\n{file_list}") + + return "## Software Framework" + "".join([f"\n- {i}" for i in file_list]) + + +@register_tool(tags=["system design", "write trd and framework", "Write a TRD and the framework"]) +async def write_trd_and_framework( + use_case_actors: str, + user_requirements: str, + additional_technical_requirements: str, + investment: float = 50.0, + output_dir: Optional[str] = "", + context: Optional[Context] = None, +) -> str: + context = context or Context(cost_manager=CostManager(max_budget=investment)) + trd = await write_trd(use_case_actors=use_case_actors, user_requirements=user_requirements, context=context) + return await write_framework( + use_case_actors=use_case_actors, + trd=trd, + additional_technical_requirements=additional_technical_requirements, + output_dir=output_dir, + context=context, + ) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index faf2893a7..bcf039a5e 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -2,6 +2,7 @@ import subprocess import threading from queue import Queue +from metagpt.const import DEFAULT_WORKSPACE_ROOT, SWE_SETUP_PATH from metagpt.tools.tool_registry import register_tool from metagpt.utils.report import END_MARKER_VALUE, TerminalReporter @@ -26,7 +27,7 @@ class Terminal: stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - executable="/bin/bash" + executable="/bin/bash", ) self.stdout_queue = Queue() self.observer = TerminalReporter() @@ -129,3 +130,96 @@ class Terminal: self.process.stdin.close() self.process.terminate() self.process.wait() + + +@register_tool(include_functions=["run"]) +class Bash(Terminal): + """ + A class to run bash commands directly and provides custom shell functions. + All custom functions in this class can ONLY be called via the `Bash.run` method. + """ + + def __init__(self): + """init""" + super().__init__() + self.run_command(f"cd {DEFAULT_WORKSPACE_ROOT}") + self.run_command(f"source {SWE_SETUP_PATH}") + + def run(self, cmd) -> str: + """ + Executes a bash command. + + Args: + cmd (str): The bash command to execute. + + Returns: + str: The output of the command. + + This method allows for executing standard bash commands as well as + utilizing several custom shell functions defined in the environment. + + Custom Shell Functions: + + - open [] + Opens the file at the given path in the editor. If line_number is provided, + the window will move to include that line. + Arguments: + path (str): The path to the file to open. + line_number (int, optional): The line number to move the window to. + If not provided, the window will start at the top of the file. + + - goto + Moves the window to show . + Arguments: + line_number (int): The line number to move the window to. + + - scroll_down + Moves the window down {WINDOW} lines. + + - scroll_up + Moves the window up {WINDOW} lines. + + - create + Creates and opens a new file with the given name. + Arguments: + filename (str): The name of the file to create. + + - submit + Submits your current code. it can only be executed once, the last action before the `end`. + + - search_dir_and_preview [] + Searches for search_term in all files in dir and gives their code preview + with line numbers. If dir is not provided, searches in the current directory. + Arguments: + search_term (str): The term to search for. + dir (str, optional): The directory to search in. Defaults to the current directory. + + - search_file [] + Searches for search_term in file. If file is not provided, searches in the current open file. + Arguments: + search_term (str): The term to search for. + file (str, optional): The file to search in. Defaults to the current open file. + + - find_file [] + Finds all files with the given name in dir. If dir is not provided, searches in the current directory. + Arguments: + file_name (str): The name of the file to search for. + dir (str, optional): The directory to search in. Defaults to the current directory. + + - edit : < + EOF + Line numbers start from 1. Replaces lines through (inclusive) with the given text in the open file. + The replacement text is terminated by a line with only EOF on it. All of the will be entered, so make + sure your indentation is formatted properly. Python files will be checked for syntax errors after the edit. If the system + detects a syntax error, the edit will not be executed. Simply try to edit the file again, but make sure to read the error + message and modify the edit command you issue accordingly. Issuing the same command a second time will just lead to the same + error message again. All code modifications made via the 'edit' command must strictly follow the PEP8 standard. + Arguments: + start_line (int): The line number to start the edit at, starting from 1. + end_line (int): The line number to end the edit at (inclusive), starting from 1. + replacement_text (str): The text to replace the current selection with, must conform to PEP8 standards. + + Note: Make sure to use these functions as per their defined arguments and behaviors. + """ + return self.run_command(cmd) diff --git a/metagpt/tools/swe_agent_commands/__init__.py b/metagpt/tools/swe_agent_commands/__init__.py new file mode 100644 index 000000000..c0d3e2a60 --- /dev/null +++ b/metagpt/tools/swe_agent_commands/__init__.py @@ -0,0 +1,7 @@ +""" +This folder is borrowed from princeton-nlp/SWE-agent +You can find the original repository here: +https://github.com/princeton-nlp/SWE-agent/tree/main/config/commands +We are using a modified version from OpenDevin: +https://github.com/OpenDevin/OpenDevin/tree/main/opendevin/runtime/plugins/swe_agent_commands +""" diff --git a/metagpt/tools/swe_agent_commands/_setup_default_env.sh b/metagpt/tools/swe_agent_commands/_setup_default_env.sh new file mode 100644 index 000000000..8fb4a379e --- /dev/null +++ b/metagpt/tools/swe_agent_commands/_setup_default_env.sh @@ -0,0 +1,20 @@ +# _setup_default_env.sh +# Default Mode from SWE-Bench +# https://github.com/princeton-nlp/SWE-agent/blob/ca54d5556b9db4f4f2be21f09530ce69a72c0305/config/configs/default_sys-env_window100-detailed_cmd_format-last_5_history-1_demos.yaml + +export WINDOW=100 +export OVERLAP=2 +export CURRENT_LINE=0 +export CURRENT_FILE='' +export SEARCH_RESULTS=() +export SEARCH_FILES=() +export SEARCH_INDEX=0 + +state() { + local working_dir="$PWD" + if [ ! -e "$CURRENT_FILE" ]; then + echo '{"open_file": "n/a", "working_dir": "'$working_dir'"}' + else + echo '{"open_file": "'$(realpath "$CURRENT_FILE")'", "working_dir": "'$working_dir'"}' + fi +} diff --git a/metagpt/tools/swe_agent_commands/_split_string b/metagpt/tools/swe_agent_commands/_split_string new file mode 100755 index 000000000..ecc363e71 --- /dev/null +++ b/metagpt/tools/swe_agent_commands/_split_string @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import sys + + +def print_flake8_output(input_string, show_line_numbers=False): + for value in input_string.split("\n"): + parts = value.split() + if not show_line_numbers: + print(f"- {' '.join(parts[1:])}") + else: + line_nums = ":".join(parts[0].split(":")[1:]) + print(f"- {line_nums} {' '.join(parts[1:])}") + + +if __name__ == "__main__": + lint_output = sys.argv[1] + print_flake8_output(lint_output) diff --git a/metagpt/tools/swe_agent_commands/_split_string.py b/metagpt/tools/swe_agent_commands/_split_string.py new file mode 100755 index 000000000..ecc363e71 --- /dev/null +++ b/metagpt/tools/swe_agent_commands/_split_string.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import sys + + +def print_flake8_output(input_string, show_line_numbers=False): + for value in input_string.split("\n"): + parts = value.split() + if not show_line_numbers: + print(f"- {' '.join(parts[1:])}") + else: + line_nums = ":".join(parts[0].split(":")[1:]) + print(f"- {line_nums} {' '.join(parts[1:])}") + + +if __name__ == "__main__": + lint_output = sys.argv[1] + print_flake8_output(lint_output) diff --git a/metagpt/tools/swe_agent_commands/defaults.sh b/metagpt/tools/swe_agent_commands/defaults.sh new file mode 100644 index 000000000..d416dcbf5 --- /dev/null +++ b/metagpt/tools/swe_agent_commands/defaults.sh @@ -0,0 +1,192 @@ +_print() { + local total_lines=$(awk 'END {print NR}' $CURRENT_FILE) + echo "[File: $(realpath $CURRENT_FILE) ($total_lines lines total)]" + lines_above=$(jq -n "$CURRENT_LINE - $WINDOW/2" | jq '[0, .] | max | floor') + lines_below=$(jq -n "$total_lines - $CURRENT_LINE - $WINDOW/2" | jq '[0, .] | max | round') + if [ $lines_above -gt 0 ]; then + echo "($lines_above more lines above)" + fi + cat $CURRENT_FILE | grep -n $ | head -n $(jq -n "[$CURRENT_LINE + $WINDOW/2, $WINDOW/2] | max | floor") | tail -n $(jq -n "$WINDOW") + if [ $lines_below -gt 0 ]; then + echo "($lines_below more lines below)" + fi +} + +_constrain_line() { + if [ -z "$CURRENT_FILE" ] + then + echo "No file open. Use the open command first." + return + fi + local max_line=$(awk 'END {print NR}' $CURRENT_FILE) + local half_window=$(jq -n "$WINDOW/2" | jq 'floor') + export CURRENT_LINE=$(jq -n "[$CURRENT_LINE, $max_line - $half_window] | min") + export CURRENT_LINE=$(jq -n "[$CURRENT_LINE, $half_window] | max") +} + +# @yaml +# signature: open [] +# docstring: opens the file at the given path in the editor. If line_number is provided, the window will be move to include that line +# arguments: +# path: +# type: string +# description: the path to the file to open +# required: true +# line_number: +# type: integer +# description: the line number to move the window to (if not provided, the window will start at the top of the file) +# required: false +open() { + if [ -z "$1" ] + then + echo "Usage: open " + return + fi + # Check if the second argument is provided + if [ -n "$2" ]; then + # Check if the provided argument is a valid number + if ! [[ $2 =~ ^[0-9]+$ ]]; then + echo "Usage: open []" + echo "Error: must be a number" + return # Exit if the line number is not valid + fi + local max_line=$(awk 'END {print NR}' $1) + if [ $2 -gt $max_line ]; then + echo "Warning: ($2) is greater than the number of lines in the file ($max_line)" + echo "Warning: Setting to $max_line" + local line_number=$(jq -n "$max_line") # Set line number to max if greater than max + elif [ $2 -lt 1 ]; then + echo "Warning: ($2) is less than 1" + echo "Warning: Setting to 1" + local line_number=$(jq -n "1") # Set line number to 1 if less than 1 + else + local OFFSET=$(jq -n "$WINDOW/6" | jq 'floor') + local line_number=$(jq -n "[$2 + $WINDOW/2 - $OFFSET, 1] | max | floor") + fi + else + local line_number=$(jq -n "$WINDOW/2") # Set default line number if not provided + fi + + if [ -f "$1" ]; then + export CURRENT_FILE=$(realpath $1) + export CURRENT_LINE=$line_number + _constrain_line + _print + elif [ -d "$1" ]; then + echo "Error: $1 is a directory. You can only open files. Use cd or ls to navigate directories." + else + echo "File $1 not found" + fi +} + +# @yaml +# signature: goto +# docstring: moves the window to show +# arguments: +# line_number: +# type: integer +# description: the line number to move the window to +# required: true +goto() { + if [ $# -gt 1 ]; then + echo "goto allows only one line number at a time." + return + fi + if [ -z "$CURRENT_FILE" ] + then + echo "No file open. Use the open command first." + return + fi + if [ -z "$1" ] + then + echo "Usage: goto " + return + fi + if ! [[ $1 =~ ^[0-9]+$ ]] + then + echo "Usage: goto " + echo "Error: must be a number" + return + fi + local max_line=$(awk 'END {print NR}' $CURRENT_FILE) + if [ $1 -gt $max_line ] + then + echo "Error: must be less than or equal to $max_line" + return + fi + local OFFSET=$(jq -n "$WINDOW/6" | jq 'floor') + export CURRENT_LINE=$(jq -n "[$1 + $WINDOW/2 - $OFFSET, 1] | max | floor") + _constrain_line + _print +} + +# @yaml +# signature: scroll_down +# docstring: moves the window down {WINDOW} lines +scroll_down() { + if [ -z "$CURRENT_FILE" ] + then + echo "No file open. Use the open command first." + return + fi + export CURRENT_LINE=$(jq -n "$CURRENT_LINE + $WINDOW - $OVERLAP") + _constrain_line + _print +} + +# @yaml +# signature: scroll_up +# docstring: moves the window down {WINDOW} lines +scroll_up() { + if [ -z "$CURRENT_FILE" ] + then + echo "No file open. Use the open command first." + return + fi + export CURRENT_LINE=$(jq -n "$CURRENT_LINE - $WINDOW + $OVERLAP") + _constrain_line + _print +} + +# @yaml +# signature: create +# docstring: creates and opens a new file with the given name +# arguments: +# filename: +# type: string +# description: the name of the file to create +# required: true +create() { + if [ -z "$1" ]; then + echo "Usage: create " + return + fi + + # Check if the file already exists + if [ -e "$1" ]; then + echo "Error: File '$1' already exists." + open "$1" + return + fi + + # Create the file an empty new line + printf "\n" > "$1" + # Use the existing open command to open the created file + open "$1" +} + +# @yaml +# signature: submit +# docstring: submits your current code. the last action before the `end`, it can only be executed once. +submit() { + # Check if the patch file exists and is non-empty + if [ -s "$SWE_CMD_WORK_DIR/test.patch" ]; then + # Apply the patch in reverse + git apply -R < "$SWE_CMD_WORK_DIR/test.patch" + fi + + git add -A + echo "<>" +} diff --git a/metagpt/tools/swe_agent_commands/edit_linting.sh b/metagpt/tools/swe_agent_commands/edit_linting.sh new file mode 100644 index 000000000..e6d675ada --- /dev/null +++ b/metagpt/tools/swe_agent_commands/edit_linting.sh @@ -0,0 +1,165 @@ +# @yaml +# signature: |- +# edit : < +# EOF +# docstring: Line numbers start from 1. Replaces lines through (inclusive) with the given text in the open file. The replacement text is terminated by a line with only EOF on it. All of the will be entered, so make sure your indentation is formatted properly. Python files will be checked for syntax errors after the edit. If the system detects a syntax error, the edit will not be executed. Simply try to edit the file again, but make sure to read the error message and modify the edit command you issue accordingly. Issuing the same command a second time will just lead to the same error message again. All code modifications made via the 'edit' command must strictly follow the PEP8 standard. +# end_name: EOF +# arguments: +# start_line: +# type: integer +# description: the line number to start the edit at, start from 1. +# required: true +# end_line: +# type: integer +# description: the line number to end the edit at (inclusive), start from 1. +# required: true +# replacement_text: +# type: string +# description: the text to replace the current selection with must conform to PEP8 standards. +# required: true +edit() { + if [ -z "$CURRENT_FILE" ] + then + echo 'No file open. Use the `open` command first.' + return + fi + + local start_line="$(echo $1: | cut -d: -f1)" + local end_line="$(echo $1: | cut -d: -f2)" + + if [ -z "$start_line" ] || [ -z "$end_line" ] + then + echo "Usage: edit :" + return + fi + + local re='^[0-9]+$' + if ! [[ $start_line =~ $re ]]; then + echo "Usage: edit :" + echo "Error: start_line must be a number" + return + fi + if ! [[ $end_line =~ $re ]]; then + echo "Usage: edit :" + echo "Error: end_line must be a number" + return + fi + + # Run linter for original file + if [[ $CURRENT_FILE == *.py ]]; then + original_lint_output=$(flake8 --isolated --select=F821,F822,F831,E112,E113,E999,E902 "$CURRENT_FILE" 2>&1) + else + # do nothing + original_lint_output="" + fi + + + # Bash array starts at 0, so let's adjust + local start_line=$((start_line - 1)) + local end_line=$((end_line)) + + local line_count=0 + local replacement=() + while IFS= read -r line + do + replacement+=("$line") + ((line_count++)) + done + + # Create a backup of the current file + cp "$CURRENT_FILE" "$SWE_CMD_WORK_DIR/$(basename "$CURRENT_FILE")_backup" + + # Read the file line by line into an array + mapfile -t lines < "$CURRENT_FILE" + local new_lines=("${lines[@]:0:$start_line}" "${replacement[@]}" "${lines[@]:$((end_line))}") + # Write the new stuff directly back into the original file + printf "%s\n" "${new_lines[@]}" >| "$CURRENT_FILE" + + # Run linter + if [[ $CURRENT_FILE == *.py ]]; then + lint_output=$(flake8 --isolated --select=F821,F822,F831,E112,E113,E999,E902 "$CURRENT_FILE" 2>&1) + else + # do nothing + lint_output="" + fi + + # Create temporary files + temp_original=$(mktemp) + temp_modified=$(mktemp) + + # Remove line numbers and save cleaned outputs to temporary files + echo "$original_lint_output" | sed 's/:[0-9]\+:[0-9]\+:/:LINE:COL:/g' > "$temp_original" + echo "$lint_output" | sed 's/:[0-9]\+:[0-9]\+:/:LINE:COL:/g' > "$temp_modified" + + + # Compare the temporary files + if cmp -s "$temp_original" "$temp_modified"; then + lint_output="" + else + echo "Linter output for the original file:" + cat "$temp_original" + # print linter result + echo "Linter output for the modified file:" + cat "$temp_modified" + fi + + # Clean up temporary files + rm "$temp_original" "$temp_modified" + + # if there is no output, then the file is good + if [ -z "$lint_output" ]; then + export CURRENT_LINE=$start_line + _constrain_line + _print + + echo "File updated. Please review the changes and make sure they are correct (correct indentation, no duplicate lines, etc). Edit the file again if necessary." + else + echo "Your proposed edit has introduced new syntax error(s). Please understand the fixes and retry your edit command." + echo "" + echo "ERRORS:" + _split_string "$lint_output" + echo "" + + # Save original values + original_current_line=$CURRENT_LINE + original_window=$WINDOW + + # Update values + export CURRENT_LINE=$(( (line_count / 2) + start_line )) # Set to "center" of edit + export WINDOW=$((line_count + 10)) # Show +/- 5 lines around edit + + echo "This is how your edit would have looked if applied" + echo "-------------------------------------------------" + _constrain_line + _print + echo "-------------------------------------------------" + echo "" + + + # Restoring CURRENT_FILE to original contents. + cp "$SWE_CMD_WORK_DIR/$(basename "$CURRENT_FILE")_backup" "$CURRENT_FILE" + + export CURRENT_LINE=$(( ((end_line - start_line + 1) / 2) + start_line )) + export WINDOW=$((end_line - start_line + 10)) + + echo "This is the original code before your edit" + echo "-------------------------------------------------" + _constrain_line + _print + echo "-------------------------------------------------" +# + + # Restore original values + export CURRENT_LINE=$original_current_line + export WINDOW=$original_window + + echo "Your changes have NOT been applied. Please fix your edit command and try again." + echo "You either need to 1) Specify the correct start/end line arguments or 2) Correct your edit code." + echo "DO NOT re-run the same failed edit command. Running it again will lead to the same error." + fi + + + # Remove backup file + rm -f "$SWE_CMD_WORK_DIR/$(basename "$CURRENT_FILE")_backup" +} diff --git a/metagpt/tools/swe_agent_commands/search.sh b/metagpt/tools/swe_agent_commands/search.sh new file mode 100644 index 000000000..b973b2d12 --- /dev/null +++ b/metagpt/tools/swe_agent_commands/search.sh @@ -0,0 +1,245 @@ +# @yaml +# signature: search_dir_and_preview [] +# docstring: searches for search_term in all files in dir and give their code preview with line number if you think need a first look. The output will vary depending on the length of the search results, but the file path, line number & corresponding code or number of occurrences will always be output. If dir is not provided, searches in the current directory +# arguments: +# search_term: +# type: string +# description: the term to search for +# required: true +# dir: +# type: string +# description: the directory to search in (if not provided, searches in the current directory) +# required: false +search_dir_and_preview() { + if [ $# -eq 1 ]; then + local search_term="$1" + local dir="./" + elif [ $# -eq 2 ]; then + local search_term="$1" + if [ -d "$2" ]; then + local dir="$2" + else + echo "Directory $2 not found" + return + fi + else + echo "Usage: search_dir_and_preview []" + return + fi + dir=$(realpath "$dir") + local matches=$(find "$dir" -type f -path '*.py' -exec grep -nIH -- "$search_term" {} + | cut -d: -f1 | sort | uniq -c) +< 100, print an error + if [ $num_files -gt 100 ]; then + echo "More than $num_files files matched for \"$search_term\" in $dir. Please narrow your search." + return + fi + + match_with_cnt=$(echo "$matches" | awk '{$2=$2; gsub(/^\.+\/+/, "./", $2); print $2 " ("$1" matches)"}') +< [] +# docstring: searches for search_term in file. If file is not provided, searches in the current open file +# arguments: +# search_term: +# type: string +# description: the term to search for +# required: true +# file: +# type: string +# description: the file to search in (if not provided, searches in the current open file) +# required: false +search_file() { + # Check if the first argument is provided + if [ -z "$1" ]; then + echo "Usage: search_file []" + return + fi + # Check if the second argument is provided + if [ -n "$2" ]; then + # Check if the provided argument is a valid file + if [ -f "$2" ]; then + local file="$2" # Set file if valid + else + echo "Usage: search_file []" + echo "Error: File name $2 not found. Please provide a valid file name." + return # Exit if the file is not valid + fi + else + # Check if a file is open + if [ -z "$CURRENT_FILE" ]; then + echo "No file open. Use the open command first." + return # Exit if no file is open + fi + local file="$CURRENT_FILE" # Set file to the current open file + fi + local search_term="$1" + file=$(realpath "$file") + # Use grep to directly get the desired formatted output + local matches=$(grep -nH -- "$search_term" "$file") + # Check if no matches were found + if [ -z "$matches" ]; then + echo "No matches found for \"$search_term\" in $file" + return + fi + # Calculate total number of matches + local num_matches=$(echo "$matches" | wc -l | awk '{$1=$1; print $0}') + + # calculate total number of lines matched + local num_lines=$(echo "$matches" | cut -d: -f1 | sort | uniq | wc -l | awk '{$1=$1; print $0}') + # if num_lines is > 100, print an error + if [ $num_lines -gt 100 ]; then + echo "More than $num_lines lines matched for \"$search_term\" in $file. Please narrow your search." + return + fi + + # Print the total number of matches and the matches themselves + echo "Found $num_matches matches for \"$search_term\" in $file:" + echo "$matches" | cut -d: -f1-2 | sort -u -t: -k2,2n | while IFS=: read -r filename line_number; do + echo "Line $line_number:$(sed -n "${line_number}p" "$file")" + done + echo "End of matches for \"$search_term\" in $file" +} + +# @yaml +# signature: find_file [] +# docstring: finds all files with the given name in dir. If dir is not provided, searches in the current directory +# arguments: +# file_name: +# type: string +# description: the name of the file to search for +# required: true +# dir: +# type: string +# description: the directory to search in (if not provided, searches in the current directory) +# required: false +find_file() { + if [ $# -eq 1 ]; then + local file_name="$1" + local dir="./" + elif [ $# -eq 2 ]; then + local file_name="$1" + if [ -d "$2" ]; then + local dir="$2" + else + echo "Directory $2 not found" + return + fi + else + echo "Usage: find_file []" + return + fi + + dir=$(realpath "$dir") + local matches=$(find "$dir" -type f -name "$file_name") + # if no matches, return + if [ -z "$matches" ]; then + echo "No matches found for \"$file_name\" in $dir" + return + fi + # Calculate total number of matches + local num_matches=$(echo "$matches" | wc -l | awk '{$1=$1; print $0}') + echo "Found $num_matches matches for \"$file_name\" in $dir:" + echo "$matches" | awk '{print $0}' +} diff --git a/metagpt/tools/swe_agent_commands/setup_default.sh b/metagpt/tools/swe_agent_commands/setup_default.sh new file mode 100644 index 000000000..dc3b335df --- /dev/null +++ b/metagpt/tools/swe_agent_commands/setup_default.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +pip install flake8 + +# Default Mode from SWE-Bench +# https://github.com/princeton-nlp/SWE-agent/blob/ca54d5556b9db4f4f2be21f09530ce69a72c0305/config/configs/default_sys-env_window100-detailed_cmd_format-last_5_history-1_demos.yaml#L103-L106 +SCRIPT_PATH="${BASH_SOURCE[0]}" # use BASH_SOURCE to avoid the influence of `source *.sh which cause CUR_DIR=/bin` +CUR_DIR=$(dirname $(readlink -f $SCRIPT_PATH)) +REPO_ROOT_DIR=$CUR_DIR"/../../.." +source $REPO_ROOT_DIR/metagpt/tools/swe_agent_commands/_setup_default_env.sh + +# make _split_string (py) available +export PATH=$PATH:$REPO_ROOT_DIR/metagpt/tools/swe_agent_commands + +source $REPO_ROOT_DIR/metagpt/tools/swe_agent_commands/defaults.sh +source $REPO_ROOT_DIR/metagpt/tools/swe_agent_commands/search.sh +source $REPO_ROOT_DIR/metagpt/tools/swe_agent_commands/edit_linting.sh + +export SWE_CMD_WORK_DIR="$REPO_ROOT_DIR/workspace/swe_agent_workdir" +#sudo chmod 777 $REPO_ROOT_DIR/workspace/swe_agent_workdir diff --git a/metagpt/tools/swe_agent_commands/swe_agent_utils.py b/metagpt/tools/swe_agent_commands/swe_agent_utils.py new file mode 100644 index 000000000..9e293f4d2 --- /dev/null +++ b/metagpt/tools/swe_agent_commands/swe_agent_utils.py @@ -0,0 +1,38 @@ +from pathlib import Path + +import numpy as np +from datasets import load_dataset, load_from_disk + + +def extract_patch(command_output): + patch_lines = [] + recording = False + for line in command_output.split("\n"): + if line.startswith("diff --git"): + recording = True + if recording: + patch_lines.append(line) + return "\n".join(patch_lines) + + +def load_hf_dataset(dataset_name_or_path: str, cache_dir, split: str = "test", existing_ids: list = []): + data_dir = cache_dir / dataset_name_or_path + if Path(data_dir).exists(): + dataset = load_from_disk(data_dir) + else: + dataset = load_dataset(dataset_name_or_path) + dataset.save_to_disk(data_dir) + print(dataset) + if split not in dataset: + raise ValueError(f"Invalid split {split} for dataset {dataset_name_or_path}") + dataset = dataset[split] + np.array(list(map(len, dataset["instance_id"]))) + + if existing_ids: + dataset = dataset.filter( + lambda x: x["instance_id"] not in existing_ids, + desc="Filtering out existing ids", + load_from_cache_file=False, + ) + + return dataset diff --git a/metagpt/tools/tool_recommend.py b/metagpt/tools/tool_recommend.py index 0c596707a..b0fd0f39b 100644 --- a/metagpt/tools/tool_recommend.py +++ b/metagpt/tools/tool_recommend.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import traceback from typing import Any import numpy as np @@ -9,11 +10,13 @@ from rank_bm25 import BM25Okapi from metagpt.llm import LLM from metagpt.logs import logger +from metagpt.prompts.di.role_zero import JSON_REPAIR_PROMPT from metagpt.schema import Plan from metagpt.tools import TOOL_REGISTRY from metagpt.tools.tool_data_type import Tool from metagpt.tools.tool_registry import validate_tool_names from metagpt.utils.common import CodeParser +from metagpt.utils.repair_llm_raw_output import RepairType, repair_llm_raw_output TOOL_INFO_PROMPT = """ ## Capabilities @@ -134,8 +137,25 @@ class ToolRecommender(BaseModel): topk=topk, ) rsp = await LLM().aask(prompt, stream=False) - rsp = CodeParser.parse_code(text=rsp) - ranked_tools = json.loads(rsp) + + # 临时方案，待role zero的版本完成可将本注释内的代码直接替换掉 + # -------------开始--------------- + try: + ranked_tools = CodeParser.parse_code(block=None, lang="json", text=rsp) + ranked_tools = json.loads( + repair_llm_raw_output(output=ranked_tools, req_keys=[None], repair_type=RepairType.JSON) + ) + except json.JSONDecodeError: + ranked_tools = await self.llm.aask(msg=JSON_REPAIR_PROMPT.format(json_data=rsp)) + ranked_tools = json.loads(CodeParser.parse_code(block=None, lang="json", text=ranked_tools)) + except Exception: + tb = traceback.format_exc() + print(tb) + + # 为了对LLM不按格式生成进行容错 + if isinstance(ranked_tools, dict): + ranked_tools = list(ranked_tools.values())[0] + # -------------结束--------------- valid_tools = validate_tool_names(ranked_tools) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 72dc6ab94..e8f150556 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -26,7 +26,7 @@ import sys import traceback from io import BytesIO from pathlib import Path -from typing import Any, Callable, List, Literal, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union from urllib.parse import quote, unquote import aiofiles @@ -1013,3 +1013,34 @@ async def save_json_to_markdown(content: str, output_filename: str | Path): logger.warning(f"An unexpected error occurred: {e}") return await awrite(filename=output_filename, data=json_to_markdown(m)) + + +def tool2name(cls, methods: List[str], entry) -> Dict[str, Any]: + """ + Generates a mapping of class methods to a given entry with class name as a prefix. + + Args: + cls: The class from which the methods are derived. + methods (List[str]): A list of method names as strings. + entry (Any): The entry to be mapped to each method. + + Returns: + Dict[str, Any]: A dictionary where keys are method names prefixed with the class name and + values are the given entry. If the number of methods is less than 2, + the dictionary will contain a single entry with the class name as the key. + + Example: + >>> class MyClass: + >>> pass + >>> + >>> tool2name(MyClass, ['method1', 'method2'], 'some_entry') + {'MyClass.method1': 'some_entry', 'MyClass.method2': 'some_entry'} + + >>> tool2name(MyClass, ['method1'], 'some_entry') + {'MyClass': 'some_entry', 'MyClass.method1': 'some_entry'} + """ + class_name = cls.__name__ + mappings = {f"{class_name}.{i}": entry for i in methods} + if len(mappings) < 2: + mappings[class_name] = entry + return mappings diff --git a/metagpt/utils/di_graph_repository.py b/metagpt/utils/di_graph_repository.py index fee706ece..f8fabfbdc 100644 --- a/metagpt/utils/di_graph_repository.py +++ b/metagpt/utils/di_graph_repository.py @@ -23,8 +23,8 @@ from metagpt.utils.graph_repository import SPO, GraphRepository class DiGraphRepository(GraphRepository): """Graph repository based on DiGraph.""" - def __init__(self, name: str, **kwargs): - super().__init__(name=name, **kwargs) + def __init__(self, name: str | Path, **kwargs): + super().__init__(name=str(name), **kwargs) self._repo = networkx.DiGraph() async def insert(self, subject: str, predicate: str, object_: str): @@ -112,8 +112,14 @@ class DiGraphRepository(GraphRepository): async def load(self, pathname: str | Path): """Load a directed graph repository from a JSON file.""" data = await aread(filename=pathname, encoding="utf-8") - m = json.loads(data) + self.load_json(data) + + def load_json(self, val: str): + if not val: + return self + m = json.loads(val) self._repo = networkx.node_link_graph(m) + return self @staticmethod async def load_from(pathname: str | Path) -> GraphRepository: @@ -126,9 +132,7 @@ class DiGraphRepository(GraphRepository): GraphRepository: A new instance of the graph repository loaded from the specified JSON file. """ pathname = Path(pathname) - name = pathname.with_suffix("").name - root = pathname.parent - graph = DiGraphRepository(name=name, root=root) + graph = DiGraphRepository(name=pathname.stem, root=pathname.parent) if pathname.exists(): await graph.load(pathname=pathname) return graph diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index 0ba2daa89..63e2f8736 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -32,6 +32,8 @@ TOKEN_COSTS = { "gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03}, "gpt-4-vision-preview": {"prompt": 0.01, "completion": 0.03}, # TODO add extra image price calculator "gpt-4-1106-vision-preview": {"prompt": 0.01, "completion": 0.03}, + "gpt-4o": {"prompt": 0.005, "completion": 0.015}, + "gpt-4o-2024-05-13": {"prompt": 0.005, "completion": 0.015}, "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0}, "glm-3-turbo": {"prompt": 0.0007, "completion": 0.0007}, # 128k version, prompt + completion tokens=0.005￥/k-tokens "glm-4": {"prompt": 0.014, "completion": 0.014}, # 128k version, prompt + completion tokens=0.1￥/k-tokens @@ -207,6 +209,8 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0125"): "gpt-4-1106-preview", "gpt-4-vision-preview", "gpt-4-1106-vision-preview", + "gpt-4o-2024-05-13", + "gpt-4o", }: tokens_per_message = 3 # # every reply is primed with <|start|>assistant<|message|> tokens_per_name = 1 diff --git a/requirements.txt b/requirements.txt index 83a904156..23806eb63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ PyYAML==6.0.1 # sentence_transformers==2.2.2 setuptools==65.6.3 tenacity==8.2.3 -tiktoken==0.6.0 +tiktoken==0.7.0 tqdm==4.66.2 #unstructured[local-inference] # selenium>4 diff --git a/tests/data/requirements/1.acknowledge.md b/tests/data/requirements/1.acknowledge.md new file mode 100644 index 000000000..61de5f4b8 --- /dev/null +++ b/tests/data/requirements/1.acknowledge.md @@ -0,0 +1,189 @@ +## Interfaces +- 用户登录 + - Description: 用户从小程序/微应用发起请求，需要验证用户的合法身份才能正常处理。 + - ID: 1 + - HTTP METHOD: GET + - Endpoint: `/sup/login.json` + - Input Parameters: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |authCode|用户临时免登授权码|String(64)|√|| + |loginTypeEnum|登录类型|String(20)|√|| + |authCorpId|用户所在企业/组织id|String(64)||微应用免登时传递| + |app|应用标识|String(3)|√|| + - Returns: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |success|业务处理成功与否，成功true，否则false|boolean|√|只判断这个属性即可| + |message|错误信息，可以用来提示|string|√|| + |code|返回状态码|string|√|| + |data|用户的sessionId|string|√|| +- 根据sessionId查询用户详细信息 + - Description: 查询当前用户的详细信息，如 staffId，unionId，name，avatar等信息 + - ID: 2 + - HTTP METHOD: GET + - Endpoint: `/sup/user.json` + - Input Parameters: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |NDA_SESSION|用户sessionId|String(64)|√|| + - Returns: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |success|业务处理成功与否，成功true，否则false|boolean|√|只判断这个属性即可| + |message|错误信息，可以用来提示|string|√|| + |code|返回状态码|string|√|| + |data|用户的详细信息|object|√|| + |-> corpId|当前用户企业 钉钉ID(小程序端会拿不到该信息)|string|√|| + |-> corpName|当前用户企业名称(小程序端会拿不到该信息)|string|√|| + |-> staffId|员工在当前企业内的唯一标识，也称staffId(小程序端会拿不到该信息)|string|√|| + |-> unionId|员工在当前开发者企业账号范围内的唯一标识，系统生成，固定值，不会改变。|string|√|| + |-> name|当前用户的名称(小程序端会拿不到该信息)|string|√|| + |-> avatar|头像图片URL|string|√|| +- 查询国家情况描述 + - Description: 根据国家code查询国家情况描述 + - ID: 3 + - HTTP METHOD: GET + - Endpoint: `/sup/country/detail.json` + - Input Parameters: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |countryCode|国家code|string|√|| + - Returns: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |success|业务处理成功true，否则false|boolean|√|只判断这个属性即可| + |message|错误信息，可以用来提示|string|√|| + |code|返回状态码|string|√|| + |data|国家情况描述|object|√|| + |-> id|id|integer|√|| + |-> countryName|国家名称|string|√|| + |-> countryCode|国家code|string|√|| + |-> detail|产品法规分析|string|√|| +- 查询产品法规分析（法律意见详情） + - Description: 根据国家和业务线查询产品法规分析 + - ID: 4 + - HTTP METHOD: GET + - Endpoint: `/sup/legal/detail.json` + - Input Parameters: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |countryCode|国家code|string|√|| + |businessCode|业务线code|string|√|| + - Returns: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |success|业务处理成功true，否则false|boolean|√|只判断这个属性即可| + |message|错误信息，可以用来提示|string|√|| + |code|返回状态码|string|√|| + |data|法律意见详情|object|√|| + |-> id|id|integer|√|| + |-> countryName|国家名称|string|√|| + |-> countryCode|国家code|string|√|| + |-> businessLine|业务线|string|√|| + |-> businessCode|业务线code|string|√|| + |-> detail|产品法规分析|string|√|| + |-> signEntity|签约主体|string|√|| +- 查询法律意见总数 + - Description: 法律意见总数查询 + - ID: 5 + - HTTP METHOD: GET + - Endpoint: `/sup/legal/count.json` + - Input Parameters: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + - Returns: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |success|业务处理成功true，否则false|boolean|√|只判断这个属性即可| + |message|错误信息，可以用来提示|string|√|| + |code|返回状态码|string|√|| + |data|总数|integer|√|| +- 查询所有国家和业务线信息列表 + - Description: 查询所有国家和业务线信息列表 + - ID: 6 + - HTTP METHOD: GET + - Endpoint: `/sup/legal/country/list.json` + - Input Parameters: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + - Returns: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |success|业务处理成功true，否则false|boolean|√|只判断这个属性即可| + |message|错误信息，可以用来提示|string|√|| + |code|返回状态码|string|√|| + |data|所有数据列表|list of object|√|| + |-> country|国家code|string|√|| + |-> business|业务线code|string|√|| + |-> dataType|数据类型|string|√|| + |-> businessName|业务线名|string|√|| + |-> countryName|国家名|string|√|| + |-> businessNameEn|业务线名(英文)|string|√|| +- 调用法务中台antlaw接口 + - ID: 7 +- 国家/区域导游详情 & 法律意见详情 查询 + - Description：根据国家code查询国家/区域导游信息详情 + - ID: 8 + - HTTP METHOD: GET + - Endpoint: `/contract/country/navigate.json` + - Input Parameters: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |countryCode|国家code|string|√|| + - Returns: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |success|业务处理成功true，否则false|boolean|√|只判断这个属性即可| + |message|错误信息，可以用来提示|string|√|| + |code|返回状态码|string|√|| + |data|国家/区域导游详情|object|√|| + |-> country||||| + |-> -> id|id|integer|√|| + |-> -> country|国家code|string|√|| + |-> -> countryName|国家中文名称|string|√|| + |-> -> countryNameEn|国家英文名称|string|√|| + |-> -> content|国家导游中文详情json数组，具体格式见下示例|list of object|√|| + |-> -> -> title|标题|object|√|| + |-> -> -> -> title|中文标题|string||| + |-> -> -> -> titleEn|英文标题|string||| + |-> -> -> contentList|标题下面的文字描述列表|list of object|√|| + |-> -> -> -> detail|内容中文详情|string|√|| + |-> -> -> -> detailEn|内容英文详情|string|√|| + |-> -> -> -> url|超链接|string||| + |-> legal|法务信息|object||| + |-> -> country|国家code|string|√|| + |-> -> businessList|业务线列表|list of object||| + |-> -> -> id|id|integer||新增时不传，修改时传递| + |-> -> -> business|业务线code|string|√|| + |-> -> -> businessName|业务线中文名称|string|√|| + |-> -> -> businessNameEn|业务线英文名称|string|√|| + |-> -> -> content|业务线json，具体如下|object|√|| + |-> -> -> -> detailEn|具体的详情英文内容|string|√|| + |-> -> -> -> detail|具体的详情内容|string|√|| +- 国家/区域导游列表分页查询 + - Description: 分页查询国家/区域列表 + - ID: 9 + - HTTP METHOD: GET + - Endpoint: `/contract/country/list.json` + - Input Parameters: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |pageSize|分页大小|integer|√|>=1| + |pageNum|分页大小|integer|√|>=1| + |country|国家code|string||| + |business|业务线code|string||| + - Returns: + |名称|描述|类型(长度)|必选|备注| + | :- | :- | :-: | :- | :- | + |success|业务处理成功true，否则false|boolean|√|只判断这个属性即可| + |message|错误信息，可以用来提示|string|√|| + |code|返回状态码|string|√|| + |data|国家/区域导游详情|list of object|√|| + |-> id|id|integer|√|| + |-> country|国家code|string|√|| + |-> countryName|国家中文名称|string|√|| + |-> countryNameEn|国家英文名称|string|√|| + |-> gmtCreate|创建时间|string|√|| + |-> gmtModified|更新时间|string|√|| + |total|数据总量|integer|√|| diff --git a/tests/data/requirements/1.actors.json b/tests/data/requirements/1.actors.json new file mode 100644 index 000000000..aa7bf5012 --- /dev/null +++ b/tests/data/requirements/1.actors.json @@ -0,0 +1,5 @@ +{ + "法务查询者": "Actor", + "国际小超人钉钉小程序": "System", + "法务中台": "External System" +} \ No newline at end of file diff --git a/tests/data/requirements/1.constraint.java.md b/tests/data/requirements/1.constraint.java.md new file mode 100644 index 000000000..1c7b0f902 --- /dev/null +++ b/tests/data/requirements/1.constraint.java.md @@ -0,0 +1,5 @@ +- 基于dingtalk框架开发，用java语言； +- 人机交互发生在法务查询者和国际小超人钉钉小程序之间； +- 接口类的功能要放到implement子类中实现; +- 法务中台网址：`https://mock.apipark.cn/m1/4717294-4369585-default` +- 写代码时，不要单元测试代码; \ No newline at end of file diff --git a/tests/data/requirements/1.constraint.md b/tests/data/requirements/1.constraint.md new file mode 100644 index 000000000..307ee3c89 --- /dev/null +++ b/tests/data/requirements/1.constraint.md @@ -0,0 +1,7 @@ +- Using pure javascript without any third-party package, 法务查询者与国际小超人钉钉小程序之间UI用web； +- 在README.md中列出你使用到的javascript工具，已经相应的配置文件名； +- 法务中台网址：`https://mock.apipark.cn/m1/4717294-4369585-default`, 只有国际小超人钉钉小程序能访问; +- 写代码时，不要单元测试代码; +- 如果使用了接口 ID 6, 它的返回结果要去重复项; +- 不需要实现登录相关操作； +- 不需要实现切换语言的功能； \ No newline at end of file diff --git a/tests/data/requirements/1.json b/tests/data/requirements/1.json new file mode 100644 index 000000000..8e608bbca --- /dev/null +++ b/tests/data/requirements/1.json @@ -0,0 +1,5 @@ +[ + "【按国家名维度搜索】\n法务查询者在国际小超人钉钉小程序搜索框中进行检索时采用 typeahead，只能下拉选择法务中台中有的国家名。", + "法务查询者从国际小超人钉钉小程序UI侧的国家名称列表中选中国家名，进入国家详情界面。\n在国家详情界面里，法务查询者从国家详情中的业务线名列表中选出业务线名。", + "国际小超人钉钉小程序用国家代码和业务代码做参数，查询法律意见详情，然后将结果展示给法务查询者。" +] \ No newline at end of file diff --git a/tests/data/requirements/1.original_requirement.txt b/tests/data/requirements/1.original_requirement.txt new file mode 100644 index 000000000..dd3a9a0c2 --- /dev/null +++ b/tests/data/requirements/1.original_requirement.txt @@ -0,0 +1,16 @@ +3.2.首页 + +首页有两个分区，上面部分是法律意见检索栏。 + +法务查询者第一次进入国际小超人钉钉小程序展示引导页，以后进入不再展示，点击「我知道了」引导页消失。 + +【首页】 +![](1.png) +【按国家名维度搜索】 + +法务查询者在国际小超人钉钉小程序的搜索框中进行检索时采用typeahead，只能下拉选择法务中台中有的国家名称。 +![](2.png) +【检索结果】 + +法务查询者可根据国际小超人钉钉小程序UI上的滚筒切换业务线 +![](3.png) \ No newline at end of file diff --git a/tests/data/requirements/1.txt b/tests/data/requirements/1.txt new file mode 100644 index 000000000..615614098 --- /dev/null +++ b/tests/data/requirements/1.txt @@ -0,0 +1,25 @@ +## Textual User Requirements + +### 3.2. 首页 + +首页有两个分区，上面部分是法律意见检索栏。 + +法务查询者第一次进入国际小超人钉钉小程序展示引导页，以后进入不再展示，点击「我知道了」引导页消失。 + +#### 首页 +![首页](1.png) +这是一个名为“法务小超人”的移动应用程序的界面截图。界面顶部显示了应用名称和一个可切换语言的按钮“English”。在界面中间部分，有一个标题“法律意见查询”，以及一个搜索框，提示输入国家名称以查询法律意见。下方显示已收录法律意见8394篇。界面下半部分是“法务 Q&A”部分，列出了一些法律相关的选项，例如“国际法务接入口人”、“国内法务接入口人”、“国际法律协议合同办理指引”和“国内法律协议合同办理指引”。界面底部有三个导航按钮，分别是“首页”、“模板”和“我的”。 + +#### 按国家名维度搜索 +法务查询者在国际小超人钉钉小程序的搜索框中进行检索时采用typeahead，只能下拉选择法务中台中有的国家名称。 +![按国家名维度搜索](2.png) +在这张图像中，用户正在一个名为“法律意见查询”的应用中进行国家名称的搜索。用户在搜索框中输入国家名称时，系统会提供下拉建议。这些建议基于 typeahead 功能，从法务中台中筛选出匹配的国家名称供用户选择。目前，搜索结果包含了“中国”和“菲律宾”两个具体的国家名称，其它显示为“国家名”。用户可以通过下拉菜单快速选择所需的国家名称。 + +#### 检索结果 +法务查询者可根据国际小超人钉钉小程序UI上的滚筒切换业务线 +![检索结果](3.png) +这张图片展示了一个移动应用的界面，界面标题为“法律意见详情”。用户可以根据具体情况切换业务线。界面中有多个字段，包括“国家名称”、“国家情况描述”、“业务线”、“产品法规分析”和“签约主体”。第一张截图显示了详细的法律情报信息，包含区域名称、区域情况描述、业务线和产品法规概述等字段。第二张截图显示了“法律意见详情”界面，其中列出了国家名称、国家情况描述、业务线、产品法规分析和签约主体。第三张截图与第二张相似，但显示了选项的可选择状态。最下方有“取消”和“确定”的按钮。 +法务查询者从国家详情中的业务线名列表中选出要查看的业务线。 + +#### 查看法律意见详情 +国际小超人钉钉小程序用国家代码和业务代码做参数，查询法律意见详情，然后将法律意见详情展示给法务查询者。 \ No newline at end of file diff --git a/tests/data/requirements/pic/1.png b/tests/data/requirements/pic/1.png new file mode 100644 index 000000000..58fca1e94 Binary files /dev/null and b/tests/data/requirements/pic/1.png differ diff --git a/tests/data/requirements/pic/2.1.png b/tests/data/requirements/pic/2.1.png new file mode 100644 index 000000000..f50e729e4 Binary files /dev/null and b/tests/data/requirements/pic/2.1.png differ diff --git a/tests/data/requirements/pic/2.png b/tests/data/requirements/pic/2.png new file mode 100644 index 000000000..babecbccf Binary files /dev/null and b/tests/data/requirements/pic/2.png differ diff --git a/tests/data/requirements/pic/3.png b/tests/data/requirements/pic/3.png new file mode 100644 index 000000000..f5a9d1b28 Binary files /dev/null and b/tests/data/requirements/pic/3.png differ diff --git a/tests/data/requirements/pic/4.png b/tests/data/requirements/pic/4.png new file mode 100644 index 000000000..86a6bf217 Binary files /dev/null and b/tests/data/requirements/pic/4.png differ diff --git a/tests/data/requirements/pic/5.png b/tests/data/requirements/pic/5.png new file mode 100644 index 000000000..d594baee5 Binary files /dev/null and b/tests/data/requirements/pic/5.png differ diff --git a/tests/metagpt/actions/requirement_analysis/__init__.py b/tests/metagpt/actions/requirement_analysis/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/metagpt/actions/requirement_analysis/requirement/__init__.py b/tests/metagpt/actions/requirement_analysis/requirement/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/metagpt/actions/requirement_analysis/requirement/test_pic2txt.py b/tests/metagpt/actions/requirement_analysis/requirement/test_pic2txt.py new file mode 100644 index 000000000..e5875b6ac --- /dev/null +++ b/tests/metagpt/actions/requirement_analysis/requirement/test_pic2txt.py @@ -0,0 +1,26 @@ +import pytest + +from metagpt.actions.requirement_analysis.requirement.pic2txt import Pic2Txt +from metagpt.const import TEST_DATA_PATH +from metagpt.utils.common import aread + + +@pytest.mark.asyncio +async def test_pic2txt(context): + images = [ + TEST_DATA_PATH / "requirements/pic/1.png", + TEST_DATA_PATH / "requirements/pic/2.png", + TEST_DATA_PATH / "requirements/pic/3.png", + ] + textual_user_requirements = await aread(filename=TEST_DATA_PATH / "requirements/1.original_requirement.txt") + + action = Pic2Txt(context=context) + rsp = await action.run( + image_paths=images, + textual_user_requirement=textual_user_requirements, + ) + assert rsp + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/environment/mgx_env/run_mgx_env.py b/tests/metagpt/environment/mgx_env/run_mgx_env.py index 0d5287412..93c134bb4 100644 --- a/tests/metagpt/environment/mgx_env/run_mgx_env.py +++ b/tests/metagpt/environment/mgx_env/run_mgx_env.py @@ -1,16 +1,19 @@ import asyncio import os +import re import threading +import time from metagpt.environment.mgx.mgx_env import MGXEnv from metagpt.roles import Architect, Engineer, ProductManager, ProjectManager from metagpt.roles.di.data_analyst import DataAnalyst from metagpt.roles.di.engineer2 import Engineer2 +from metagpt.roles.di.swe_agent import SWEAgent from metagpt.roles.di.team_leader import TeamLeader from metagpt.schema import Message -async def main(requirement="", enable_human_input=False, use_fixed_sop=False): +async def main(requirement="", enable_human_input=False, use_fixed_sop=False, allow_idle_time=30): if use_fixed_sop: engineer = Engineer(n_borg=5, use_code_review=False) else: @@ -26,35 +29,56 @@ async def main(requirement="", enable_human_input=False, use_fixed_sop=False): engineer, # QaEngineer(), DataAnalyst(tools=[""]), + SWEAgent(), ] ) if enable_human_input: # simulate human sending messages in chatbox - send_human_input(env) + stop_event = threading.Event() + human_input_thread = send_human_input(env, stop_event) if requirement: env.publish_message(Message(content=requirement)) - # env.publish_message(Message(content=requirement, send_to={"David"}), user_defined_recipient="David") + # user_defined_recipient = "Alex" + # env.publish_message(Message(content=requirement, send_to={user_defined_recipient}), user_defined_recipient=user_defined_recipient) - while not env.is_idle: - await env.run() + allow_idle_time = allow_idle_time if enable_human_input else 1 + start_time = time.time() + while time.time() - start_time < allow_idle_time: + if not env.is_idle: + await env.run() + start_time = time.time() # reset start time + + if enable_human_input: + print("No more human input, terminating, press ENTER for a full termination.") + stop_event.set() + human_input_thread.join() -def send_human_input(env): +def send_human_input(env, stop_event): """ Simulate sending message in chatbox Note in local environment, the message is consumed only after current round of env.run is finished """ def send_messages(): - while True: + while not stop_event.is_set(): message = input("Enter a message any time: ") - env.publish_message(Message(content=message)) + user_defined_recipient = re.search(r"@(\w+)", message) + if user_defined_recipient: + recipient_name = user_defined_recipient.group(1) + print(f"{recipient_name} will receive the message") + env.publish_message( + Message(content=message, send_to={recipient_name}), user_defined_recipient=recipient_name + ) + else: + env.publish_message(Message(content=message)) # Start a thread for sending messages send_thread = threading.Thread(target=send_messages, args=()) send_thread.start() + return send_thread GAME_REQ = "create a 2048 game" @@ -100,6 +124,14 @@ clone https://github.com/garylin2099/simple_calculator, checkout a new branch na Commit your changes and push, finally, create a PR to the master branch of https://github.com/mannaandpoem/simple_calculator. """ +TL_CHAT1 = """Summarize the paper for me""" # expecting clarification +TL_CHAT2 = """Solve the issue at this link""" # expecting clarification +TL_CHAT3 = """Who is the first man landing on Moon""" # expecting answering directly +TL_CHAT4 = """Find all zeros in the indicated finite field of the given polynomial with coefficients in that field. x^5 + 3x^3 + x^2 + 2x in Z_5""" # expecting answering directly +TL_CHAT5 = """Find the degree for the given field extension Q(sqrt(2), sqrt(3), sqrt(18)) over Q.""" # expecting answering directly +TL_CHAT6 = """Statement 1 | A ring homomorphism is one to one if and only if the kernel is {{0}},. Statement 2 | Q is an ideal in R""" # expecting answering directly +TL_CHAT7 = """Jean has 30 lollipops. Jean eats 2 of the lollipops. With the remaining lollipops, Jean wants to package 2 lollipops in one bag. How many bags can Jean fill?""" # expecting answering directly + if __name__ == "__main__": # NOTE: Add access_token to test github issue fixing diff --git a/tests/metagpt/roles/di/run_swe_agent_for_benchmark.py b/tests/metagpt/roles/di/run_swe_agent_for_benchmark.py new file mode 100644 index 000000000..54b3623a4 --- /dev/null +++ b/tests/metagpt/roles/di/run_swe_agent_for_benchmark.py @@ -0,0 +1,113 @@ +import asyncio +import json +from datetime import datetime + +from metagpt.config2 import config +from metagpt.const import DEFAULT_WORKSPACE_ROOT, METAGPT_ROOT +from metagpt.logs import logger +from metagpt.roles.di.swe_agent import SWEAgent +from metagpt.tools.libs.terminal import Terminal +from metagpt.tools.swe_agent_commands.swe_agent_utils import load_hf_dataset + +# Specify by yourself +TEST_REPO_DIR = METAGPT_ROOT / "data" / "test_repo" +DATA_DIR = METAGPT_ROOT / "data/hugging_face" + +INSTANCE_TEMPLATE = """ +## User Requirement +Fix the bug in the repo. Because the environment is not available, you DO NOT need to run and modify any existing test case files or add new test case files to ensure that the bug is fixed. + +We're currently solving the following issue within our repository. You can use any bash commands or the special interface to help you. Here's the issue and hints text: +## ISSUE +{issue} + +## HINTS +hints text is the comment under issue: +{hints_text} + +The repository may already exist at the path `{repo_path}`. If it doesn't, please download the repository to this path. +Your first action must be to navigate to the repository path `{repo_path}`. +This issue occurred in version {version}, with the corresponding base commit being {base_commit}. You need to switch to the code version associated with this commit. +All subsequent actions must be performed within this repository path. Do not leave this directory to execute any actions at any time. + +# INSTRUCTIONS: +Now, you're going to solve this issue on your own from the perspective of a programmer. Your terminal session has started and you're in the repository's root directory. You can use any bash commands or the special interface to help you. Edit all the files you need. +Remember, YOU CAN ONLY ENTER ONE COMMAND AT A TIME. You should always wait for feedback after every command. +""" + + +def check_instance_status(instance, swe_result_dir): + output_file = swe_result_dir / "all_preds.jsonl" + res = True + # 先检查all_preds.jsonl文件是否存在 + if not output_file.exists(): + return res + with open(output_file, "r") as fp: + for line in fp: + existing_instance = json.loads(line.strip()) + if existing_instance["instance_id"] == instance["instance_id"]: + return False + return True + + +async def run(instance, swe_result_dir): + if not check_instance_status(instance, swe_result_dir): + logger.info(f"Instance {instance['instance_id']} already exists, skipping execution.") + return + + repo_path = TEST_REPO_DIR / (instance["repo"].replace("-", "_").replace("/", "__") + "_" + instance["version"]) + + # 前处理 + terminal = Terminal() + terminal.run_command(f"cd {repo_path} && git reset --hard && git clean -n -d && git clean -f -d") + terminal.run_command("BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}')") + logger.info(terminal.run_command("echo $BRANCH")) + logger.info(terminal.run_command('git checkout "$BRANCH"')) + logger.info(terminal.run_command("git branch")) + + user_requirement_and_issue = INSTANCE_TEMPLATE.format( + issue=instance["problem_statement"], + hints_text=instance["hints_text"], + repo_path=repo_path, + version=instance["version"], + base_commit=instance["base_commit"], + ) + + logger.info(f"**** Starting to run {instance['instance_id']}****") + swe_agent = SWEAgent() + swe_agent.run_eval = True + await swe_agent.run(user_requirement_and_issue) + save_predictions(swe_agent, instance, swe_result_dir) + logger.info(f"**** Finished running {instance['instance_id']}****") + + +def save_predictions(swe_agent: SWEAgent, instance, swe_result_dir): + output_file = swe_result_dir / "all_preds.jsonl" + instance["model_name_or_path"] = swe_agent.config.llm.model + instance["model_patch"] = swe_agent.output_diff + + logger.info(f"Preparing to save predictions to {output_file}") + + # Save the predictions to a JSONL file + with open(output_file, "a+") as fp: + print(json.dumps(instance), file=fp, flush=True) + + logger.info(f"Saved prediction of {instance['instance_id']} to {output_file}") + + +async def async_main(): + dataset_path = "manna-ai/SWE-bench_Nano" # "princeton-nlp/SWE-bench_Lite" #"manna-ai/SWE-bench_Nano" + + dataset = load_hf_dataset(dataset_name_or_path=dataset_path, cache_dir=DATA_DIR, split="test") + date_time = datetime.now().strftime("%m%d") + _round = "first" + # _round = "second" + exp_name = f"nano_mgx_{date_time}_{_round}" + swe_result_dir = DEFAULT_WORKSPACE_ROOT / f"result_{config.llm.model.replace('/', '_')}" / exp_name + swe_result_dir.mkdir(parents=True, exist_ok=True) + for instance in dataset: + await run(instance, swe_result_dir) + + +if __name__ == "__main__": + asyncio.run(async_main()) diff --git a/tests/metagpt/roles/di/run_swe_agent_open_source_issue.py b/tests/metagpt/roles/di/run_swe_agent_open_source_issue.py new file mode 100644 index 000000000..ec87dd7e2 --- /dev/null +++ b/tests/metagpt/roles/di/run_swe_agent_open_source_issue.py @@ -0,0 +1,44 @@ +import asyncio + +from metagpt.logs import logger +from metagpt.roles.di.swe_agent import SWEAgent + +FIX_ISSUE1 = """ +Write a fix for this issue: https://github.com/langchain-ai/langchain/issues/20453, +you can fix it on this repo https://github.com/garylin2099/langchain +""" +# + "checkout a branch named test-fix, commit your changes, push, +# and create a PR to the master branch of https://github.com/iorisa/langchain" +# """ +FIX_ISSUE2 = """ +Write a fix for this issue https://github.com/geekan/MetaGPT/issues/1275. +You can fix it on the v0.8-release branch of this repo https://github.com/garylin2099/MetaGPT +""" +# + "during fixing, checkout a branch named test-fix-1275, commit your changes, push, +# and create a PR to the v0.8-release branch of https://github.com/garylin2099/MetaGPT" + +FIX_ISSUE3 = """ +Write a fix for this issue https://github.com/geekan/MetaGPT/issues/1262. +You can fix it on this repo https://github.com/garylin2099/MetaGPT +""" +# during fixing, checkout a branch named test-fix-1262, commit your changes, push, +# and create a PR to https://github.com/garylin2099/MetaGPT +# """ +FIX_ISSUE_SIMPLE = """ +Write a fix for this issue: https://github.com/mannaandpoem/simple_calculator/issues/1, +you can fix it on this repo https://github.com/garylin2099/simple_calculator +""" +# checkout a branch named test, commit your changes, push, and create a PR to the master branch of original repo. +# """ + + +NO_ENV_TIP = """ +Because the environment is not available, you DO NOT need to run and modify any existing test case files or +add new test case files to ensure that the bug is fixed. +""" +if __name__ == "__main__": + swe_agent = SWEAgent() + logger.info("**** Starting run ****") + user_requirement_and_issue = FIX_ISSUE1 + NO_ENV_TIP + asyncio.run(swe_agent.run(user_requirement_and_issue)) + logger.info("**** Finished running ****")