From c49b832deecfb9d5ab1455d0db238e03e9300740 Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 21 Nov 2023 20:34:37 +0800 Subject: [PATCH] add trigger repair_llm_output for open llm --- metagpt/actions/action.py | 21 +- metagpt/config.py | 1 + metagpt/roles/role.py | 14 +- metagpt/utils/repair_llm_raw_output.py | 246 ++++++++++++++++++ tests/metagpt/utils/test_custom_decoder.py | 45 ++++ .../utils/test_repair_llm_raw_output.py | 203 +++++++++++++++ 6 files changed, 515 insertions(+), 15 deletions(-) create mode 100644 metagpt/utils/repair_llm_raw_output.py create mode 100644 tests/metagpt/utils/test_repair_llm_raw_output.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 790295d55..f9e4f926b 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -16,6 +16,8 @@ from metagpt.llm import LLM from metagpt.logs import logger from metagpt.utils.common import OutputParser from metagpt.utils.custom_decoder import CustomDecoder +from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, RepairType,\ + retry_parse_json_text, extract_content_from_output class Action(ABC): @@ -49,7 +51,7 @@ class Action(ABC): system_msgs.append(self.prefix) return await self.llm.aask(prompt, system_msgs) - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) + # @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) async def _aask_v1( self, prompt: str, @@ -65,22 +67,19 @@ class Action(ABC): content = await self.llm.aask(prompt, system_msgs) logger.debug(content) output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) + output_class_fields = list(output_class.schema()["properties"].keys()) # Custom ActionOutput's fields if format == "json": - pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]" - matches = re.findall(pattern, content, re.DOTALL) - - for match in matches: - if match: - content = match - break - - parsed_data = CustomDecoder(strict=False).decode(content) + content = repair_llm_raw_output(content, req_keys=output_class_fields + ["[/CONTENT]"]) + content = extract_content_from_output(content) + content = repair_llm_raw_output(content, req_keys=[None], repair_type=RepairType.JSON) # req_keys mocked + logger.info(f"extracted CONTENT from content:\n{content}") + parsed_data = retry_parse_json_text(content) else: # using markdown parser parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) - logger.debug(parsed_data) + logger.debug(f"parsed_data:\n{parsed_data}") instruct_content = output_class(**parsed_data) return ActionOutput(content, instruct_content) diff --git a/metagpt/config.py b/metagpt/config.py index 3f9e742bd..a4c43c28a 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -93,6 +93,7 @@ class Config(metaclass=Singleton): self.mermaid_engine = self._get("MERMAID_ENGINE", "nodejs") self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") + self.repair_llm_output = self._get("REPAIR_LLM_OUTPUT", False) self.prompt_format = self._get("PROMPT_FORMAT", "markdown") def _init_with_config_files_and_env(self, configs: dict, yaml_file): diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index b96c361c0..140910f0a 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -19,6 +19,8 @@ from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory from metagpt.schema import Message +from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output + PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -49,6 +51,7 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi {name}: {result} """ + class RoleReactMode(str, Enum): REACT = "react" BY_ORDER = "by_order" @@ -58,6 +61,7 @@ class RoleReactMode(str, Enum): def values(cls): return [item.value for item in cls] + class RoleSetting(BaseModel): """Role Settings""" name: str @@ -79,11 +83,11 @@ class RoleContext(BaseModel): env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) - state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None + state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) - react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes + react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes max_react_loop: int = 1 class Config: @@ -127,8 +131,9 @@ class Role: i = action("", llm=self._llm) else: if self._setting.is_human and not isinstance(action.llm, HumanProvider): - logger.warning(f"is_human attribute does not take effect," - f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances") + logger.warning(f"is_human attribute does not take effect, " + f"as Role's {str(action)} was initialized using LLM, " + f"try passing in Action classes instead of initialized instances") i = action i.set_prefix(self._get_prefix(), self.profile) self._actions.append(i) @@ -193,6 +198,7 @@ class Role: n_states=len(self._states) - 1, previous_state=self._rc.state) # print(prompt) next_state = await self._llm.aask(prompt) + next_state = extract_state_value_from_output(next_state) logger.debug(f"{prompt=}") if (not next_state.isdigit() and next_state != "-1") \ or int(next_state) not in range(-1, len(self._states)): diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py new file mode 100644 index 000000000..a65e4be80 --- /dev/null +++ b/metagpt/utils/repair_llm_raw_output.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : repair llm raw output with particular conditions + +import copy +from enum import Enum +from typing import Union +import regex as re + +from metagpt.logs import logger +from metagpt.config import CONFIG +from metagpt.utils.custom_decoder import CustomDecoder + + +class RepairType(Enum): + CS = "case sensitivity" + SCM = "special character missing" # Usually the req_key appear in pairs like `[key] xx [/key]` + RKPM = "required key pair missing" # condition like `[key] xx` which lacks `[/key]` + JSON = "json format" + + +def repair_case_sensitivity(output: str, req_key: str) -> str: + """ + usually, req_key is the key name of expected json or markdown content, it won't appear in the value part. + fix target string `"Shared Knowledge": ""` but `"Shared knowledge": ""` actually + """ + if req_key in output: + return output + + output_lower = output.lower() + req_key_lower = req_key.lower() + if req_key_lower in output_lower: + # find the sub-part index, and replace it with raw req_key + lidx = output_lower.find(req_key_lower) + source = output[lidx: lidx + len(req_key_lower)] + output = output.replace(source, req_key) + logger.info(f"repair_case_sensitivity: {req_key}") + + return output + + +def repair_special_character_missing(output: str, req_key: str) -> str: + """ + fix target string `[CONTENT]xxx[/CONTENT]` lacks [/CONTENT] + """ + sc_arr = ["/"] + + if req_key in output: + return output + + for sc in sc_arr: + req_key_pure = req_key.replace(sc, "") + appear_cnt = output.count(req_key_pure) + if req_key_pure in output and appear_cnt > 1: + # req_key with special_character usually in the tail side + ridx = output.rfind(req_key_pure) + output = f"{output[:ridx]}{req_key}{output[ridx + len(req_key_pure):]}" + logger.info(f"repair_special_character_missing: {req_key}") + + return output + + +def repair_required_key_pair_missing(output: str, req_key: str) -> str: + """ + implement the req_key pair in the begin or end of the content + req_key format + 1. `[req_key]`, and its pair `[/req_key]` + 2. `[/req_key]`, and its pair `[req_key]` + """ + if req_key.startswith("[") and req_key.endswith("]"): + if "/" in req_key: + left_key = req_key.replace("/", "") # `[/req_key]` -> `[req_key]` + right_key = req_key + else: + left_key = req_key + right_key = f"{req_key[0]}/{req_key[1:]}" # `[req_key]` -> `[/req_key]` + + if left_key not in output: + output = left_key + output + if right_key not in output: + output = output + right_key + + return output + + +def repair_json_format(output: str) -> str: + """ + fix extra `[` or `}` in the end + """ + output = output.strip() + + if output.startswith("[{"): + output = output[1:] + logger.info(f"repair_json_format: {'[{'}") + elif output.endswith("}]"): + output = output[:-1] + logger.info(f"repair_json_format: {'}]'}") + elif output.startswith("{") and output.startswith("]"): + output = output[:-1] + "}" + + return output + + +def _repair_llm_raw_output(output: str, req_key: str, repair_type: RepairType = None) -> str: + repair_types = [repair_type] if repair_type else [item for item in RepairType if item not in [RepairType.JSON]] + for repair_type in repair_types: + if repair_type == RepairType.CS: + output = repair_case_sensitivity(output, req_key) + elif repair_type == RepairType.SCM: + output = repair_special_character_missing(output, req_key) + elif repair_type == RepairType.JSON: + output = repair_json_format(output) + elif repair_type == RepairType.RKPM: + output = repair_required_key_pair_missing(output, req_key) + return output + + +def repair_llm_raw_output(output: str, req_keys: list[str], repair_type: RepairType = None) -> str: + """ + in open-source llm model, it usually can't follow the instruction well, the output may be incomplete, + so here we try to repair it and use all repair methods by default. + typical case + 1. case sensitivity + target: "Original Requirements" + output: "Original requirements" + 2. special character missing + target: [/CONTENT] + output: [CONTENT] + 3. json format + target: { xxx } + output: { xxx }] + """ + if not CONFIG.repair_llm_output: + return output + + # do the repairation usually for non-openai models + for req_key in req_keys: + output = _repair_llm_raw_output(output=output, + req_key=req_key, + repair_type=repair_type) + return output + + +def repair_invalid_json(output: str, error: str) -> str: + """ + repair the situation like there are extra chars like + error examples + example 1. json.decoder.JSONDecodeError: Expecting ',' delimiter: line 154 column 1 (char 2765) + example 2. xxx.JSONDecodeError: Expecting property name enclosed in double quotes: line 14 column 1 (char 266) + """ + pattern = r"line ([0-9]+)" + + matches = re.findall(pattern, error, re.DOTALL) + if len(matches) > 0: + line_no = int(matches[0]) - 1 + + # due to CustomDecoder can handle `"": ''` or `'': ""`, so convert `"""` -> `"`, `'''` -> `'` + output = output.replace('"""', '"').replace("'''", '"') + arr = output.split("\n") + line = arr[line_no].strip() + # different general problems + if line.endswith("],"): + # problem, redundant char `]` + line = line.replace("]", "") + elif line.endswith("},"): + # problem, redundant char `}` + line = line.replace("}", "") + elif '",' not in line: + line = f'{line}",' + elif "," not in line: + # problem, miss char `,` at the end. + line = f"{line}," + + arr[line_no] = line + output = "\n".join(arr) + logger.info(f"repair_invalid_json, raw error: {error}") + + return output + + +def retry_parse_json_text(output: str, retry: int = 5) -> Union[list, dict]: + """ + repair the json-text situation like there are extra chars like [']', '}'] + """ + parsed_data = {} + for idx in range(retry): + raw_output = copy.deepcopy(output) + + try: + parsed_data = CustomDecoder(strict=False).decode(output) + break + except Exception as exp: + if not CONFIG.repair_llm_output: + # if repair_llm_output is False, break from the retry loop + break + + logger.warning(f"decode content into json failed, try to fix it. exp: {exp}") + error = str(exp) + output = repair_invalid_json(output, error) + + return parsed_data + + +def extract_content_from_output(content: str, right_key: str = "[/CONTENT]"): + """ extract xxx from [CONTENT](xxx)[/CONTENT] using regex pattern """ + def re_extract_content(cont: str, pattern: str) -> str: + matches = re.findall(pattern, cont, re.DOTALL) + for match in matches: + if match: + cont = match + break + return cont.strip() + + raw_content = copy.deepcopy(content) + pattern = r"\[CONTENT\]([\s\S]*)\[/CONTENT\]" + new_content = re_extract_content(raw_content, pattern) + + if not new_content.startswith("{"): + # TODO find a more general pattern + # # for `[CONTENT]xxx[CONTENT]xxxx[/CONTENT] situation + logger.warning(f"extract_content try another pattern: {pattern}") + raw_content = copy.deepcopy(new_content + right_key) + # # pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]" + new_content = re_extract_content(raw_content, pattern) + else: + if right_key in new_content: + idx = new_content.find(right_key) + new_content = new_content[:idx] + + return new_content + + +def extract_state_value_from_output(content: str) -> str: + """ + For openai models, they will always return state number. But for open llm models, the instruction result maybe a + long text contain target number, so here add a extraction to improve success rate. + + Args: + content (str): llm's output from `Role._think` + """ + content = content.strip() # deal the output cases like " 0", "0\n" and so on. + pattern = r"([0-9])" # TODO find the number using a more proper method not just extract from content using pattern + matches = re.findall(pattern, content, re.DOTALL) + matches = list(set(matches)) + state = matches[0] if len(matches) > 0 else "-1" + return state diff --git a/tests/metagpt/utils/test_custom_decoder.py b/tests/metagpt/utils/test_custom_decoder.py index c7b14ad59..4af7a6cdc 100644 --- a/tests/metagpt/utils/test_custom_decoder.py +++ b/tests/metagpt/utils/test_custom_decoder.py @@ -6,6 +6,7 @@ @File : test_custom_decoder.py """ +import pytest from metagpt.utils.custom_decoder import CustomDecoder @@ -37,6 +38,46 @@ def test_parse_single_quote(): parsed_data = decoder.decode(input_data) assert 'a"\n b' in parsed_data + input_data = """{ + 'a': " + b +" +} +""" + with pytest.raises(Exception): + parsed_data = decoder.decode(input_data) + + input_data = """{ + 'a': ' + b +' +} +""" + with pytest.raises(Exception): + parsed_data = decoder.decode(input_data) + + +def test_parse_double_quote(): + decoder = CustomDecoder(strict=False) + + input_data = """{ + "a": " + b +" +} +""" + parsed_data = decoder.decode(input_data) + assert parsed_data["a"] == "\n b\n" + + input_data = """{ + "a": ' + b +' +} +""" + parsed_data = decoder.decode(input_data) + assert parsed_data["a"] == "\n b\n" + def test_parse_triple_double_quote(): # Create a custom JSON decoder @@ -54,6 +95,10 @@ def test_parse_triple_double_quote(): parsed_data = decoder.decode(input_data) assert parsed_data["a"] == "b" + input_data = "{\"\"\"a\"\"\": '''b'''}" + parsed_data = decoder.decode(input_data) + assert parsed_data["a"] == "b" + def test_parse_triple_single_quote(): # Create a custom JSON decoder diff --git a/tests/metagpt/utils/test_repair_llm_raw_output.py b/tests/metagpt/utils/test_repair_llm_raw_output.py new file mode 100644 index 000000000..39a7343e7 --- /dev/null +++ b/tests/metagpt/utils/test_repair_llm_raw_output.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : unittest of repair_llm_raw_output + +import pytest + +from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, RepairType, repair_invalid_json,\ + extract_content_from_output, retry_parse_json_text + + +def test_repair_case_sensitivity(): + raw_output = """{ + "Original requirements": "Write a 2048 game", + "search Information": "", + "competitive Quadrant charT": "quadrantChart + Campaign A: [0.3, 0.6]", + "requirement analysis": "The 2048 game should be simple to play" +}""" + target_output = """{ + "Original Requirements": "Write a 2048 game", + "Search Information": "", + "Competitive Quadrant Chart": "quadrantChart + Campaign A: [0.3, 0.6]", + "Requirement Analysis": "The 2048 game should be simple to play" +}""" + req_keys = ["Original Requirements", "Search Information", "Competitive Quadrant Chart", "Requirement Analysis"] + output = repair_llm_raw_output(output=raw_output, + req_keys=req_keys) + assert output == target_output + + +def test_repair_special_character_missing(): + raw_output = """[CONTENT] + "Anything UNCLEAR": "No unclear requirements or information." +[CONTENT]""" + + target_output = """[CONTENT] + "Anything UNCLEAR": "No unclear requirements or information." +[/CONTENT]""" + req_keys = ["[/CONTENT]"] + output = repair_llm_raw_output(output=raw_output, + req_keys=req_keys) + assert output == target_output + + +def test_required_key_pair_missing(): + raw_output = "[CONTENT] xxx" + target_output = "[CONTENT] xxx[/CONTENT]" + + output = repair_llm_raw_output(output=raw_output, + req_keys=["[/CONTENT]"]) + assert output == target_output + + raw_output = "xxx[/CONTENT]" + target_output = "[CONTENT]xxx[/CONTENT]" + + output = repair_llm_raw_output(output=raw_output, + req_keys=["[CONTENT]"]) + assert output == target_output + + +def test_repair_json_format(): + raw_output = "{ xxx }]" + target_output = "{ xxx }" + + output = repair_llm_raw_output(output=raw_output, + req_keys=[None], + repair_type=RepairType.JSON) + assert output == target_output + + +def test_retry_parse_json_text(): + invalid_json_text = """{ +"Original Requirements": "Create a 2048 game", +"Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis" +], +"Requirement Analysis": "The requirements are clear and well-defined" +}""" + target_json = { + "Original Requirements": "Create a 2048 game", + "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", + "Requirement Analysis": "The requirements are clear and well-defined" + } + output = retry_parse_json_text(invalid_json_text) + assert output == target_json + + invalid_json_text = """{ +"Original Requirements": "Create a 2048 game", +"Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis" +}, +"Requirement Analysis": "The requirements are clear and well-defined" +}""" + target_json = { + "Original Requirements": "Create a 2048 game", + "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", + "Requirement Analysis": "The requirements are clear and well-defined" + } + output = retry_parse_json_text(invalid_json_text) + assert output == target_json + + +def test_extract_content_from_output(): + output = 'Sure! Here is the properly formatted JSON output based on the given context:\n\n[CONTENT]\n{\n"' \ + 'Required Python third-party packages": [\n"pygame==2.0.4",\n"pytest"\n],\n"Required Other language ' \ + 'third-party packages": [\n"No third-party packages are required."\n],\n"Full API spec": "\nopenapi: ' \ + '3.0.0\n\ndescription: A JSON object representing the game state.\n\npaths:\n game:\n get:\n ' \ + 'summary: Get the current game state.\n responses:\n 200:\n description: Game state.' \ + '\n\n moves:\n post:\n summary: Make a move.\n requestBody:\n description: Move to be ' \ + 'made.\n content:\n applicationjson:\n schema:\n type: object\n ' \ + ' properties:\n x:\n type: integer\n y:\n ' \ + ' type: integer\n tile:\n type: object\n ' \ + 'properties:\n value:\n type: integer\n x:\n ' \ + ' type: integer\n y:\n type: integer\n\n ' \ + 'undo-move:\n post:\n summary: Undo the last move.\n responses:\n 200:\n ' \ + ' description: Undone move.\n\n end-game:\n post:\n summary: End the game.\n responses:\n ' \ + ' 200:\n description: Game ended.\n\n start-game:\n post:\n summary: Start a new ' \ + 'game.\n responses:\n 200:\n description: Game started.\n\n game-over:\n get:\n ' \ + ' summary: Check if the game is over.\n responses:\n 200:\n description: Game ' \ + 'over.\n 404:\n description: Game not over.\n\n score:\n get:\n summary: Get the ' \ + 'current score.\n responses:\n 200:\n description: Score.\n\n tile:\n get:\n ' \ + 'summary: Get a specific tile.\n parameters:\n tile_id:\n type: integer\n ' \ + 'description: ID of the tile to get.\n responses:\n 200:\n description: Tile.\n\n ' \ + 'tiles:\n get:\n summary: Get all tiles.\n responses:\n 200:\n description: ' \ + 'Tiles.\n\n level:\n get:\n summary: Get the current level.\n responses:\n 200:\n ' \ + ' description: Level.\n\n level-up:\n post:\n summary: Level up.\n responses:\n ' \ + '200:\n description: Level up successful.\n\n level-down:\n post:\n summary: Level ' \ + 'down.\n responses:\n 200:\n description: Level down successful.\n\n restart:\n ' \ + 'post:\n summary: Restart the game.\n responses:\n 200:\n description: Game ' \ + 'restarted.\n\n help:\n get:\n summary: Get help.\n responses:\n 200:\n ' \ + 'description: Help.\n\n version:\n get:\n summary: Get the version of the game.\n ' \ + 'responses:\n 200:\n description: Version.\n\n}\n\n"Logic Analysis": [\n"game.py",' \ + '\n"Contains the game logic."\n],\n"Task list": [\n"game.py",\n"Contains the game logic and should be ' \ + 'done first."\n],\n"Shared Knowledge": "\n\'game.py\' contains the game logic.\n",\n"Anything ' \ + 'UNCLEAR": "How to start the game."\n]\n\n[/CONTENT] Great! Your JSON output is properly formatted ' \ + 'and correctly includes all the required sections. Here\'s a breakdown of what each section ' \ + 'contains:\n\nRequired Python third-party packages:\n\n* pygame==2.0.4\n* pytest\n\nRequired Other ' \ + 'language third-party packages:\n\n* No third-party packages are required.\n\nFull API spec:\n\n* ' \ + 'openapi: 3.0.0\n* description: A JSON object representing the game state.\n* paths:\n + game: ' \ + 'Get the current game state.\n + moves: Make a move.\n + undo-move: Undo the last move.\n + ' \ + 'end-game: End the game.\n + start-game: Start a new game.\n + game-over: Check if the game is ' \ + 'over.\n + score: Get the current score.\n + tile: Get a specific tile.\n + tiles: Get all tiles.\n ' \ + '+ level: Get the current level.\n + level-up: Level up.\n + level-down: Level down.\n + restart: ' \ + 'Restart the game.\n + help: Get help.\n + version: Get the version of the game.\n\nLogic ' \ + 'Analysis:\n\n* game.py contains the game logic.\n\nTask list:\n\n* game.py contains the game logic ' \ + 'and should be done first.\n\nShared Knowledge:\n\n* \'game.py\' contains the game logic.\n\nAnything ' \ + 'UNCLEAR:\n\n* How to start the game.\n\nGreat job! This JSON output should provide a clear and ' \ + 'comprehensive overview of the project\'s requirements and dependencies.' + output = extract_content_from_output(output) + assert output.startswith('{\n"Required Python third-party packages') + + output = 'Sure, I would be happy to help! Here is the information you provided, formatted as a JSON object ' \ + 'inside the [CONTENT] tag:\n\n[CONTENT]\n{\n"Original Requirements": "Create a 2048 game",\n"Search ' \ + 'Information": "Search results for 2048 game",\n"Requirements": [\n"Create a game with the same rules ' \ + 'as the original 2048 game",\n"Implement a user interface that is easy to use and understand",\n"Add a ' \ + 'scoreboard to track the player progress",\n"Allow the player to undo and redo moves",\n"Implement a ' \ + 'game over screen to display the final score"\n],\n"Product Goals": [\n"Create a fun and engaging game ' \ + 'experience for the player",\n"Design a user interface that is visually appealing and easy to use",\n"' \ + 'Optimize the game for performance and responsiveness"\n],\n"User Stories": [\n"As a player, I want to ' \ + 'be able to move tiles around the board to combine numbers",\n"As a player, I want to be able to undo ' \ + 'and redo moves to correct mistakes",\n"As a player, I want to see the final score and game over screen' \ + ' when I win"\n],\n"Competitive Analysis": [\n"Competitor A: 2048 game with a simple user interface and' \ + ' basic graphics",\n"Competitor B: 2048 game with a more complex user interface and better graphics",' \ + '\n"Competitor C: 2048 game with a unique twist on the rules and a more challenging gameplay experience"' \ + '\n],\n"Competitive Quadrant Chart": "quadrantChart\\n\ttitle Reach and engagement of campaigns\\n\t\t' \ + 'x-axis Low Reach --> High Reach\\n\t\ty-axis Low Engagement --> High Engagement\\n\tquadrant-1 We ' \ + 'should expand\\n\tquadrant-2 Need to promote\\n\tquadrant-3 Re-evaluate\\n\tquadrant-4 May be ' \ + 'improved\\n\tCampaign A: [0.3, 0.6]\\n\tCampaign B: [0.45, 0.23]\\n\tCampaign C: [0.57, 0.69]\\n\t' \ + 'Campaign D: [0.78, 0.34]\\n\tCampaign E: [0.40, 0.34]\\n\tCampaign F: [0.35, 0.78]"\n],\n"Requirement ' \ + 'Analysis": "The requirements are clear and well-defined, but there may be some ambiguity around the ' \ + 'specific implementation details",\n"Requirement Pool": [\n["P0", "Implement a game with the same ' \ + 'rules as the original 2048 game"],\n["P1", "Add a scoreboard to track the player progress"],\n["P2", ' \ + '"Allow the player to undo and redo moves"]\n],\n"UI Design draft": "The UI should be simple and easy ' \ + 'to use, with a clean and visually appealing design. The game board should be the main focus of the ' \ + 'UI, with clear and concise buttons for the player to interact with.",\n"Anything UNCLEAR": ""\n}\n' \ + '[/CONTENT]\n\nI hope this helps! Let me know if you have any further questions or if there anything ' \ + 'else I can do to assist you.' + output = extract_content_from_output(output) + assert output.startswith('{\n"Original Requirements"') + + output = """ Sure, I'd be happy to help! Here's the JSON output for the given context:\n\n[CONTENT]\n{ +"Implementation approach": "We will use the open-source framework PyGame to create a 2D game engine, which will + provide us with a robust and efficient way to handle game logic and rendering. PyGame is widely used in the game + development community and has a large number of resources and tutorials available online.",\n"Python package name": + "pygame_2048",\n"File list": ["main.py", "game.py", "constants.py", "ui.py"],\n"Data structures and interface + definitions": '\nclassDiagram\n class Game{\n +int score\n +list tiles\n +function + move_tile(tile, int dx, int dy)\n +function undo_move()\n +function get_highest_score()\n }\n + class Tile{\n +int value\n +int x\n +int y\n }\n ...\n Game "1" -- "1" Food: has\n', +\n"Program call flow": '\nsequenceDiagram\n participant M as Main\n participant G as Game\n ...\n G->>M: + end game\n',\n"Anything UNCLEAR": "The requirement is clear to me."\n}\n[/CONTENT] Here's the JSON output for the + given context, wrapped inside the [CONTENT][/CONTENT] format:\n\n[CONTENT]\n{\n"Implementation approach": "We will + use the open-source framework PyGame to create a 2D game engine, which will provide us with a robust and efficient + way to handle game logic and rendering. PyGame is widely used in the game development community and has a large + number of resources and tutorials available online.",\n"Python package name": "pygame_2048",\n"File list": + ["main.py", "game.py", "constants.py", "ui.py"],\n"Data structures and interface definitions": '\nclassDiagram\n + class Game{\n +int score\n +list tiles\n +function move_tile(tile, int dx, int dy)\n ++function undo_move()\n +function get_highest_score()\n }\n class Tile{\n +int value\n +int x\n + +int y\n }\n ...\n Game "1" -- "1" Food: has\n',\n"Program call flow": '\nsequenceDiagram\n participant + M as Main\n participant G as Game\n ...\n G->>M: end game\n',\n"Anything UNCLEAR": "The requirement is + clear to me."\n}\n[/CONTENT] Great! Your JSON output is well-formatted and provides all the necessary + information for a developer to understand the design and implementation of the 2048 game. +""" + output = extract_content_from_output(output) + assert output.startswith('{\n"Implementation approach"') and "[/CONTENT]" not in output