From c233699275930e15b11ef64633d4383ac1fc6ba4 Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 21 Nov 2023 20:33:58 +0800 Subject: [PATCH 01/16] add aiohttp encapsulation --- metagpt/utils/ahttp_client.py | 59 ++++++++++++++++++++++++ tests/metagpt/utils/test_ahttp_client.py | 38 +++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 metagpt/utils/ahttp_client.py create mode 100644 tests/metagpt/utils/test_ahttp_client.py diff --git a/metagpt/utils/ahttp_client.py b/metagpt/utils/ahttp_client.py new file mode 100644 index 000000000..d4f9f94e5 --- /dev/null +++ b/metagpt/utils/ahttp_client.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : pure async http_client + +from typing import Optional, Any, Mapping, Union + +from aiohttp.client import DEFAULT_TIMEOUT +import aiohttp + + +async def apost(url: str, + params: Optional[Mapping[str, str]] = None, + json: Any = None, + data: Any = None, + headers: Optional[dict] = None, + as_json: bool = False, + encoding: str = "utf-8", + timeout: int = DEFAULT_TIMEOUT.total) -> Union[str, dict]: + async with aiohttp.ClientSession() as session: + async with session.post( + url=url, + params=params, + json=json, + data=data, + headers=headers, + timeout=timeout + ) as resp: + if as_json: + data = await resp.json() + else: + data = await resp.read() + data = data.decode(encoding) + return data + + +async def apost_stream(url: str, + params: Optional[Mapping[str, str]] = None, + json: Any = None, + data: Any = None, + headers: Optional[dict] = None, + encoding: str = "utf-8", + timeout: int = DEFAULT_TIMEOUT.total) -> Any: + """ + usage: + result = astream(url="xx") + async for line in result: + deal_with(line) + """ + async with aiohttp.ClientSession() as session: + async with session.post( + url=url, + params=params, + json=json, + data=data, + headers=headers, + timeout=timeout + ) as resp: + async for line in resp.content: + yield line.decode(encoding) diff --git a/tests/metagpt/utils/test_ahttp_client.py b/tests/metagpt/utils/test_ahttp_client.py new file mode 100644 index 000000000..15159423a --- /dev/null +++ b/tests/metagpt/utils/test_ahttp_client.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : unittest of ahttp_client + +import pytest + +from metagpt.utils.ahttp_client import apost, apost_stream + + +@pytest.mark.asyncio +async def test_apost(): + result = await apost( + url="https://www.baidu.com/" + ) + assert "百度一下" in result + + result = await apost( + url="http://aider.meizu.com/app/weather/listWeather", + data={"cityIds": "101240101"}, + as_json=True + ) + assert result["code"] == "200" + + +@pytest.mark.asyncio +async def test_apost_stream(): + result = apost_stream( + url="https://www.baidu.com/" + ) + async for line in result: + assert len(line) >= 0 + + result = apost_stream( + url="http://aider.meizu.com/app/weather/listWeather", + data={"cityIds": "101240101"} + ) + async for line in result: + assert len(line) >= 0 From c49b832deecfb9d5ab1455d0db238e03e9300740 Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 21 Nov 2023 20:34:37 +0800 Subject: [PATCH 02/16] add trigger repair_llm_output for open llm --- metagpt/actions/action.py | 21 +- metagpt/config.py | 1 + metagpt/roles/role.py | 14 +- metagpt/utils/repair_llm_raw_output.py | 246 ++++++++++++++++++ tests/metagpt/utils/test_custom_decoder.py | 45 ++++ .../utils/test_repair_llm_raw_output.py | 203 +++++++++++++++ 6 files changed, 515 insertions(+), 15 deletions(-) create mode 100644 metagpt/utils/repair_llm_raw_output.py create mode 100644 tests/metagpt/utils/test_repair_llm_raw_output.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 790295d55..f9e4f926b 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -16,6 +16,8 @@ from metagpt.llm import LLM from metagpt.logs import logger from metagpt.utils.common import OutputParser from metagpt.utils.custom_decoder import CustomDecoder +from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, RepairType,\ + retry_parse_json_text, extract_content_from_output class Action(ABC): @@ -49,7 +51,7 @@ class Action(ABC): system_msgs.append(self.prefix) return await self.llm.aask(prompt, system_msgs) - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) + # @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) async def _aask_v1( self, prompt: str, @@ -65,22 +67,19 @@ class Action(ABC): content = await self.llm.aask(prompt, system_msgs) logger.debug(content) output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) + output_class_fields = list(output_class.schema()["properties"].keys()) # Custom ActionOutput's fields if format == "json": - pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]" - matches = re.findall(pattern, content, re.DOTALL) - - for match in matches: - if match: - content = match - break - - parsed_data = CustomDecoder(strict=False).decode(content) + content = repair_llm_raw_output(content, req_keys=output_class_fields + ["[/CONTENT]"]) + content = extract_content_from_output(content) + content = repair_llm_raw_output(content, req_keys=[None], repair_type=RepairType.JSON) # req_keys mocked + logger.info(f"extracted CONTENT from content:\n{content}") + parsed_data = retry_parse_json_text(content) else: # using markdown parser parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) - logger.debug(parsed_data) + logger.debug(f"parsed_data:\n{parsed_data}") instruct_content = output_class(**parsed_data) return ActionOutput(content, instruct_content) diff --git a/metagpt/config.py b/metagpt/config.py index 3f9e742bd..a4c43c28a 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -93,6 +93,7 @@ class Config(metaclass=Singleton): self.mermaid_engine = self._get("MERMAID_ENGINE", "nodejs") self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") + self.repair_llm_output = self._get("REPAIR_LLM_OUTPUT", False) self.prompt_format = self._get("PROMPT_FORMAT", "markdown") def _init_with_config_files_and_env(self, configs: dict, yaml_file): diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index b96c361c0..140910f0a 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -19,6 +19,8 @@ from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory from metagpt.schema import Message +from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output + PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -49,6 +51,7 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi {name}: {result} """ + class RoleReactMode(str, Enum): REACT = "react" BY_ORDER = "by_order" @@ -58,6 +61,7 @@ class RoleReactMode(str, Enum): def values(cls): return [item.value for item in cls] + class RoleSetting(BaseModel): """Role Settings""" name: str @@ -79,11 +83,11 @@ class RoleContext(BaseModel): env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) - state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None + state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) - react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes + react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes max_react_loop: int = 1 class Config: @@ -127,8 +131,9 @@ class Role: i = action("", llm=self._llm) else: if self._setting.is_human and not isinstance(action.llm, HumanProvider): - logger.warning(f"is_human attribute does not take effect," - f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances") + logger.warning(f"is_human attribute does not take effect, " + f"as Role's {str(action)} was initialized using LLM, " + f"try passing in Action classes instead of initialized instances") i = action i.set_prefix(self._get_prefix(), self.profile) self._actions.append(i) @@ -193,6 +198,7 @@ class Role: n_states=len(self._states) - 1, previous_state=self._rc.state) # print(prompt) next_state = await self._llm.aask(prompt) + next_state = extract_state_value_from_output(next_state) logger.debug(f"{prompt=}") if (not next_state.isdigit() and next_state != "-1") \ or int(next_state) not in range(-1, len(self._states)): diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py new file mode 100644 index 000000000..a65e4be80 --- /dev/null +++ b/metagpt/utils/repair_llm_raw_output.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : repair llm raw output with particular conditions + +import copy +from enum import Enum +from typing import Union +import regex as re + +from metagpt.logs import logger +from metagpt.config import CONFIG +from metagpt.utils.custom_decoder import CustomDecoder + + +class RepairType(Enum): + CS = "case sensitivity" + SCM = "special character missing" # Usually the req_key appear in pairs like `[key] xx [/key]` + RKPM = "required key pair missing" # condition like `[key] xx` which lacks `[/key]` + JSON = "json format" + + +def repair_case_sensitivity(output: str, req_key: str) -> str: + """ + usually, req_key is the key name of expected json or markdown content, it won't appear in the value part. + fix target string `"Shared Knowledge": ""` but `"Shared knowledge": ""` actually + """ + if req_key in output: + return output + + output_lower = output.lower() + req_key_lower = req_key.lower() + if req_key_lower in output_lower: + # find the sub-part index, and replace it with raw req_key + lidx = output_lower.find(req_key_lower) + source = output[lidx: lidx + len(req_key_lower)] + output = output.replace(source, req_key) + logger.info(f"repair_case_sensitivity: {req_key}") + + return output + + +def repair_special_character_missing(output: str, req_key: str) -> str: + """ + fix target string `[CONTENT]xxx[/CONTENT]` lacks [/CONTENT] + """ + sc_arr = ["/"] + + if req_key in output: + return output + + for sc in sc_arr: + req_key_pure = req_key.replace(sc, "") + appear_cnt = output.count(req_key_pure) + if req_key_pure in output and appear_cnt > 1: + # req_key with special_character usually in the tail side + ridx = output.rfind(req_key_pure) + output = f"{output[:ridx]}{req_key}{output[ridx + len(req_key_pure):]}" + logger.info(f"repair_special_character_missing: {req_key}") + + return output + + +def repair_required_key_pair_missing(output: str, req_key: str) -> str: + """ + implement the req_key pair in the begin or end of the content + req_key format + 1. `[req_key]`, and its pair `[/req_key]` + 2. `[/req_key]`, and its pair `[req_key]` + """ + if req_key.startswith("[") and req_key.endswith("]"): + if "/" in req_key: + left_key = req_key.replace("/", "") # `[/req_key]` -> `[req_key]` + right_key = req_key + else: + left_key = req_key + right_key = f"{req_key[0]}/{req_key[1:]}" # `[req_key]` -> `[/req_key]` + + if left_key not in output: + output = left_key + output + if right_key not in output: + output = output + right_key + + return output + + +def repair_json_format(output: str) -> str: + """ + fix extra `[` or `}` in the end + """ + output = output.strip() + + if output.startswith("[{"): + output = output[1:] + logger.info(f"repair_json_format: {'[{'}") + elif output.endswith("}]"): + output = output[:-1] + logger.info(f"repair_json_format: {'}]'}") + elif output.startswith("{") and output.startswith("]"): + output = output[:-1] + "}" + + return output + + +def _repair_llm_raw_output(output: str, req_key: str, repair_type: RepairType = None) -> str: + repair_types = [repair_type] if repair_type else [item for item in RepairType if item not in [RepairType.JSON]] + for repair_type in repair_types: + if repair_type == RepairType.CS: + output = repair_case_sensitivity(output, req_key) + elif repair_type == RepairType.SCM: + output = repair_special_character_missing(output, req_key) + elif repair_type == RepairType.JSON: + output = repair_json_format(output) + elif repair_type == RepairType.RKPM: + output = repair_required_key_pair_missing(output, req_key) + return output + + +def repair_llm_raw_output(output: str, req_keys: list[str], repair_type: RepairType = None) -> str: + """ + in open-source llm model, it usually can't follow the instruction well, the output may be incomplete, + so here we try to repair it and use all repair methods by default. + typical case + 1. case sensitivity + target: "Original Requirements" + output: "Original requirements" + 2. special character missing + target: [/CONTENT] + output: [CONTENT] + 3. json format + target: { xxx } + output: { xxx }] + """ + if not CONFIG.repair_llm_output: + return output + + # do the repairation usually for non-openai models + for req_key in req_keys: + output = _repair_llm_raw_output(output=output, + req_key=req_key, + repair_type=repair_type) + return output + + +def repair_invalid_json(output: str, error: str) -> str: + """ + repair the situation like there are extra chars like + error examples + example 1. json.decoder.JSONDecodeError: Expecting ',' delimiter: line 154 column 1 (char 2765) + example 2. xxx.JSONDecodeError: Expecting property name enclosed in double quotes: line 14 column 1 (char 266) + """ + pattern = r"line ([0-9]+)" + + matches = re.findall(pattern, error, re.DOTALL) + if len(matches) > 0: + line_no = int(matches[0]) - 1 + + # due to CustomDecoder can handle `"": ''` or `'': ""`, so convert `"""` -> `"`, `'''` -> `'` + output = output.replace('"""', '"').replace("'''", '"') + arr = output.split("\n") + line = arr[line_no].strip() + # different general problems + if line.endswith("],"): + # problem, redundant char `]` + line = line.replace("]", "") + elif line.endswith("},"): + # problem, redundant char `}` + line = line.replace("}", "") + elif '",' not in line: + line = f'{line}",' + elif "," not in line: + # problem, miss char `,` at the end. + line = f"{line}," + + arr[line_no] = line + output = "\n".join(arr) + logger.info(f"repair_invalid_json, raw error: {error}") + + return output + + +def retry_parse_json_text(output: str, retry: int = 5) -> Union[list, dict]: + """ + repair the json-text situation like there are extra chars like [']', '}'] + """ + parsed_data = {} + for idx in range(retry): + raw_output = copy.deepcopy(output) + + try: + parsed_data = CustomDecoder(strict=False).decode(output) + break + except Exception as exp: + if not CONFIG.repair_llm_output: + # if repair_llm_output is False, break from the retry loop + break + + logger.warning(f"decode content into json failed, try to fix it. exp: {exp}") + error = str(exp) + output = repair_invalid_json(output, error) + + return parsed_data + + +def extract_content_from_output(content: str, right_key: str = "[/CONTENT]"): + """ extract xxx from [CONTENT](xxx)[/CONTENT] using regex pattern """ + def re_extract_content(cont: str, pattern: str) -> str: + matches = re.findall(pattern, cont, re.DOTALL) + for match in matches: + if match: + cont = match + break + return cont.strip() + + raw_content = copy.deepcopy(content) + pattern = r"\[CONTENT\]([\s\S]*)\[/CONTENT\]" + new_content = re_extract_content(raw_content, pattern) + + if not new_content.startswith("{"): + # TODO find a more general pattern + # # for `[CONTENT]xxx[CONTENT]xxxx[/CONTENT] situation + logger.warning(f"extract_content try another pattern: {pattern}") + raw_content = copy.deepcopy(new_content + right_key) + # # pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]" + new_content = re_extract_content(raw_content, pattern) + else: + if right_key in new_content: + idx = new_content.find(right_key) + new_content = new_content[:idx] + + return new_content + + +def extract_state_value_from_output(content: str) -> str: + """ + For openai models, they will always return state number. But for open llm models, the instruction result maybe a + long text contain target number, so here add a extraction to improve success rate. + + Args: + content (str): llm's output from `Role._think` + """ + content = content.strip() # deal the output cases like " 0", "0\n" and so on. + pattern = r"([0-9])" # TODO find the number using a more proper method not just extract from content using pattern + matches = re.findall(pattern, content, re.DOTALL) + matches = list(set(matches)) + state = matches[0] if len(matches) > 0 else "-1" + return state diff --git a/tests/metagpt/utils/test_custom_decoder.py b/tests/metagpt/utils/test_custom_decoder.py index c7b14ad59..4af7a6cdc 100644 --- a/tests/metagpt/utils/test_custom_decoder.py +++ b/tests/metagpt/utils/test_custom_decoder.py @@ -6,6 +6,7 @@ @File : test_custom_decoder.py """ +import pytest from metagpt.utils.custom_decoder import CustomDecoder @@ -37,6 +38,46 @@ def test_parse_single_quote(): parsed_data = decoder.decode(input_data) assert 'a"\n b' in parsed_data + input_data = """{ + 'a': " + b +" +} +""" + with pytest.raises(Exception): + parsed_data = decoder.decode(input_data) + + input_data = """{ + 'a': ' + b +' +} +""" + with pytest.raises(Exception): + parsed_data = decoder.decode(input_data) + + +def test_parse_double_quote(): + decoder = CustomDecoder(strict=False) + + input_data = """{ + "a": " + b +" +} +""" + parsed_data = decoder.decode(input_data) + assert parsed_data["a"] == "\n b\n" + + input_data = """{ + "a": ' + b +' +} +""" + parsed_data = decoder.decode(input_data) + assert parsed_data["a"] == "\n b\n" + def test_parse_triple_double_quote(): # Create a custom JSON decoder @@ -54,6 +95,10 @@ def test_parse_triple_double_quote(): parsed_data = decoder.decode(input_data) assert parsed_data["a"] == "b" + input_data = "{\"\"\"a\"\"\": '''b'''}" + parsed_data = decoder.decode(input_data) + assert parsed_data["a"] == "b" + def test_parse_triple_single_quote(): # Create a custom JSON decoder diff --git a/tests/metagpt/utils/test_repair_llm_raw_output.py b/tests/metagpt/utils/test_repair_llm_raw_output.py new file mode 100644 index 000000000..39a7343e7 --- /dev/null +++ b/tests/metagpt/utils/test_repair_llm_raw_output.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : unittest of repair_llm_raw_output + +import pytest + +from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, RepairType, repair_invalid_json,\ + extract_content_from_output, retry_parse_json_text + + +def test_repair_case_sensitivity(): + raw_output = """{ + "Original requirements": "Write a 2048 game", + "search Information": "", + "competitive Quadrant charT": "quadrantChart + Campaign A: [0.3, 0.6]", + "requirement analysis": "The 2048 game should be simple to play" +}""" + target_output = """{ + "Original Requirements": "Write a 2048 game", + "Search Information": "", + "Competitive Quadrant Chart": "quadrantChart + Campaign A: [0.3, 0.6]", + "Requirement Analysis": "The 2048 game should be simple to play" +}""" + req_keys = ["Original Requirements", "Search Information", "Competitive Quadrant Chart", "Requirement Analysis"] + output = repair_llm_raw_output(output=raw_output, + req_keys=req_keys) + assert output == target_output + + +def test_repair_special_character_missing(): + raw_output = """[CONTENT] + "Anything UNCLEAR": "No unclear requirements or information." +[CONTENT]""" + + target_output = """[CONTENT] + "Anything UNCLEAR": "No unclear requirements or information." +[/CONTENT]""" + req_keys = ["[/CONTENT]"] + output = repair_llm_raw_output(output=raw_output, + req_keys=req_keys) + assert output == target_output + + +def test_required_key_pair_missing(): + raw_output = "[CONTENT] xxx" + target_output = "[CONTENT] xxx[/CONTENT]" + + output = repair_llm_raw_output(output=raw_output, + req_keys=["[/CONTENT]"]) + assert output == target_output + + raw_output = "xxx[/CONTENT]" + target_output = "[CONTENT]xxx[/CONTENT]" + + output = repair_llm_raw_output(output=raw_output, + req_keys=["[CONTENT]"]) + assert output == target_output + + +def test_repair_json_format(): + raw_output = "{ xxx }]" + target_output = "{ xxx }" + + output = repair_llm_raw_output(output=raw_output, + req_keys=[None], + repair_type=RepairType.JSON) + assert output == target_output + + +def test_retry_parse_json_text(): + invalid_json_text = """{ +"Original Requirements": "Create a 2048 game", +"Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis" +], +"Requirement Analysis": "The requirements are clear and well-defined" +}""" + target_json = { + "Original Requirements": "Create a 2048 game", + "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", + "Requirement Analysis": "The requirements are clear and well-defined" + } + output = retry_parse_json_text(invalid_json_text) + assert output == target_json + + invalid_json_text = """{ +"Original Requirements": "Create a 2048 game", +"Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis" +}, +"Requirement Analysis": "The requirements are clear and well-defined" +}""" + target_json = { + "Original Requirements": "Create a 2048 game", + "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", + "Requirement Analysis": "The requirements are clear and well-defined" + } + output = retry_parse_json_text(invalid_json_text) + assert output == target_json + + +def test_extract_content_from_output(): + output = 'Sure! Here is the properly formatted JSON output based on the given context:\n\n[CONTENT]\n{\n"' \ + 'Required Python third-party packages": [\n"pygame==2.0.4",\n"pytest"\n],\n"Required Other language ' \ + 'third-party packages": [\n"No third-party packages are required."\n],\n"Full API spec": "\nopenapi: ' \ + '3.0.0\n\ndescription: A JSON object representing the game state.\n\npaths:\n game:\n get:\n ' \ + 'summary: Get the current game state.\n responses:\n 200:\n description: Game state.' \ + '\n\n moves:\n post:\n summary: Make a move.\n requestBody:\n description: Move to be ' \ + 'made.\n content:\n applicationjson:\n schema:\n type: object\n ' \ + ' properties:\n x:\n type: integer\n y:\n ' \ + ' type: integer\n tile:\n type: object\n ' \ + 'properties:\n value:\n type: integer\n x:\n ' \ + ' type: integer\n y:\n type: integer\n\n ' \ + 'undo-move:\n post:\n summary: Undo the last move.\n responses:\n 200:\n ' \ + ' description: Undone move.\n\n end-game:\n post:\n summary: End the game.\n responses:\n ' \ + ' 200:\n description: Game ended.\n\n start-game:\n post:\n summary: Start a new ' \ + 'game.\n responses:\n 200:\n description: Game started.\n\n game-over:\n get:\n ' \ + ' summary: Check if the game is over.\n responses:\n 200:\n description: Game ' \ + 'over.\n 404:\n description: Game not over.\n\n score:\n get:\n summary: Get the ' \ + 'current score.\n responses:\n 200:\n description: Score.\n\n tile:\n get:\n ' \ + 'summary: Get a specific tile.\n parameters:\n tile_id:\n type: integer\n ' \ + 'description: ID of the tile to get.\n responses:\n 200:\n description: Tile.\n\n ' \ + 'tiles:\n get:\n summary: Get all tiles.\n responses:\n 200:\n description: ' \ + 'Tiles.\n\n level:\n get:\n summary: Get the current level.\n responses:\n 200:\n ' \ + ' description: Level.\n\n level-up:\n post:\n summary: Level up.\n responses:\n ' \ + '200:\n description: Level up successful.\n\n level-down:\n post:\n summary: Level ' \ + 'down.\n responses:\n 200:\n description: Level down successful.\n\n restart:\n ' \ + 'post:\n summary: Restart the game.\n responses:\n 200:\n description: Game ' \ + 'restarted.\n\n help:\n get:\n summary: Get help.\n responses:\n 200:\n ' \ + 'description: Help.\n\n version:\n get:\n summary: Get the version of the game.\n ' \ + 'responses:\n 200:\n description: Version.\n\n}\n\n"Logic Analysis": [\n"game.py",' \ + '\n"Contains the game logic."\n],\n"Task list": [\n"game.py",\n"Contains the game logic and should be ' \ + 'done first."\n],\n"Shared Knowledge": "\n\'game.py\' contains the game logic.\n",\n"Anything ' \ + 'UNCLEAR": "How to start the game."\n]\n\n[/CONTENT] Great! Your JSON output is properly formatted ' \ + 'and correctly includes all the required sections. Here\'s a breakdown of what each section ' \ + 'contains:\n\nRequired Python third-party packages:\n\n* pygame==2.0.4\n* pytest\n\nRequired Other ' \ + 'language third-party packages:\n\n* No third-party packages are required.\n\nFull API spec:\n\n* ' \ + 'openapi: 3.0.0\n* description: A JSON object representing the game state.\n* paths:\n + game: ' \ + 'Get the current game state.\n + moves: Make a move.\n + undo-move: Undo the last move.\n + ' \ + 'end-game: End the game.\n + start-game: Start a new game.\n + game-over: Check if the game is ' \ + 'over.\n + score: Get the current score.\n + tile: Get a specific tile.\n + tiles: Get all tiles.\n ' \ + '+ level: Get the current level.\n + level-up: Level up.\n + level-down: Level down.\n + restart: ' \ + 'Restart the game.\n + help: Get help.\n + version: Get the version of the game.\n\nLogic ' \ + 'Analysis:\n\n* game.py contains the game logic.\n\nTask list:\n\n* game.py contains the game logic ' \ + 'and should be done first.\n\nShared Knowledge:\n\n* \'game.py\' contains the game logic.\n\nAnything ' \ + 'UNCLEAR:\n\n* How to start the game.\n\nGreat job! This JSON output should provide a clear and ' \ + 'comprehensive overview of the project\'s requirements and dependencies.' + output = extract_content_from_output(output) + assert output.startswith('{\n"Required Python third-party packages') + + output = 'Sure, I would be happy to help! Here is the information you provided, formatted as a JSON object ' \ + 'inside the [CONTENT] tag:\n\n[CONTENT]\n{\n"Original Requirements": "Create a 2048 game",\n"Search ' \ + 'Information": "Search results for 2048 game",\n"Requirements": [\n"Create a game with the same rules ' \ + 'as the original 2048 game",\n"Implement a user interface that is easy to use and understand",\n"Add a ' \ + 'scoreboard to track the player progress",\n"Allow the player to undo and redo moves",\n"Implement a ' \ + 'game over screen to display the final score"\n],\n"Product Goals": [\n"Create a fun and engaging game ' \ + 'experience for the player",\n"Design a user interface that is visually appealing and easy to use",\n"' \ + 'Optimize the game for performance and responsiveness"\n],\n"User Stories": [\n"As a player, I want to ' \ + 'be able to move tiles around the board to combine numbers",\n"As a player, I want to be able to undo ' \ + 'and redo moves to correct mistakes",\n"As a player, I want to see the final score and game over screen' \ + ' when I win"\n],\n"Competitive Analysis": [\n"Competitor A: 2048 game with a simple user interface and' \ + ' basic graphics",\n"Competitor B: 2048 game with a more complex user interface and better graphics",' \ + '\n"Competitor C: 2048 game with a unique twist on the rules and a more challenging gameplay experience"' \ + '\n],\n"Competitive Quadrant Chart": "quadrantChart\\n\ttitle Reach and engagement of campaigns\\n\t\t' \ + 'x-axis Low Reach --> High Reach\\n\t\ty-axis Low Engagement --> High Engagement\\n\tquadrant-1 We ' \ + 'should expand\\n\tquadrant-2 Need to promote\\n\tquadrant-3 Re-evaluate\\n\tquadrant-4 May be ' \ + 'improved\\n\tCampaign A: [0.3, 0.6]\\n\tCampaign B: [0.45, 0.23]\\n\tCampaign C: [0.57, 0.69]\\n\t' \ + 'Campaign D: [0.78, 0.34]\\n\tCampaign E: [0.40, 0.34]\\n\tCampaign F: [0.35, 0.78]"\n],\n"Requirement ' \ + 'Analysis": "The requirements are clear and well-defined, but there may be some ambiguity around the ' \ + 'specific implementation details",\n"Requirement Pool": [\n["P0", "Implement a game with the same ' \ + 'rules as the original 2048 game"],\n["P1", "Add a scoreboard to track the player progress"],\n["P2", ' \ + '"Allow the player to undo and redo moves"]\n],\n"UI Design draft": "The UI should be simple and easy ' \ + 'to use, with a clean and visually appealing design. The game board should be the main focus of the ' \ + 'UI, with clear and concise buttons for the player to interact with.",\n"Anything UNCLEAR": ""\n}\n' \ + '[/CONTENT]\n\nI hope this helps! Let me know if you have any further questions or if there anything ' \ + 'else I can do to assist you.' + output = extract_content_from_output(output) + assert output.startswith('{\n"Original Requirements"') + + output = """ Sure, I'd be happy to help! Here's the JSON output for the given context:\n\n[CONTENT]\n{ +"Implementation approach": "We will use the open-source framework PyGame to create a 2D game engine, which will + provide us with a robust and efficient way to handle game logic and rendering. PyGame is widely used in the game + development community and has a large number of resources and tutorials available online.",\n"Python package name": + "pygame_2048",\n"File list": ["main.py", "game.py", "constants.py", "ui.py"],\n"Data structures and interface + definitions": '\nclassDiagram\n class Game{\n +int score\n +list tiles\n +function + move_tile(tile, int dx, int dy)\n +function undo_move()\n +function get_highest_score()\n }\n + class Tile{\n +int value\n +int x\n +int y\n }\n ...\n Game "1" -- "1" Food: has\n', +\n"Program call flow": '\nsequenceDiagram\n participant M as Main\n participant G as Game\n ...\n G->>M: + end game\n',\n"Anything UNCLEAR": "The requirement is clear to me."\n}\n[/CONTENT] Here's the JSON output for the + given context, wrapped inside the [CONTENT][/CONTENT] format:\n\n[CONTENT]\n{\n"Implementation approach": "We will + use the open-source framework PyGame to create a 2D game engine, which will provide us with a robust and efficient + way to handle game logic and rendering. PyGame is widely used in the game development community and has a large + number of resources and tutorials available online.",\n"Python package name": "pygame_2048",\n"File list": + ["main.py", "game.py", "constants.py", "ui.py"],\n"Data structures and interface definitions": '\nclassDiagram\n + class Game{\n +int score\n +list tiles\n +function move_tile(tile, int dx, int dy)\n ++function undo_move()\n +function get_highest_score()\n }\n class Tile{\n +int value\n +int x\n + +int y\n }\n ...\n Game "1" -- "1" Food: has\n',\n"Program call flow": '\nsequenceDiagram\n participant + M as Main\n participant G as Game\n ...\n G->>M: end game\n',\n"Anything UNCLEAR": "The requirement is + clear to me."\n}\n[/CONTENT] Great! Your JSON output is well-formatted and provides all the necessary + information for a developer to understand the design and implementation of the 2048 game. +""" + output = extract_content_from_output(output) + assert output.startswith('{\n"Implementation approach"') and "[/CONTENT]" not in output From fc4ec5a9449bd644e0e13c30f2de4867f0b9685a Mon Sep 17 00:00:00 2001 From: better629 Date: Wed, 22 Nov 2023 13:01:16 +0800 Subject: [PATCH 03/16] update retry_parse_json_text --- config/config.yaml | 5 + metagpt/actions/action.py | 17 +-- metagpt/utils/repair_llm_raw_output.py | 110 +++++++++++++----- .../utils/test_repair_llm_raw_output.py | 66 ++++++++++- 4 files changed, 158 insertions(+), 40 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index bed67083c..72d2c0b19 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -94,4 +94,9 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### browser path for pyppeteer engine, support Chrome, Chromium,MS Edge #PYPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable" +### for repair non-openai LLM's output when parse json-text if PROMPT_FORMAT=json +### due to non-openai LLM's output will not always follow the instruction, so here activate a post-process +### repair operation on the content extracted from LLM's raw output. Warning, it improves the result but not fix all cases. +# REPAIR_LLM_OUTPUT: false + PROMPT_FORMAT: json #json or markdown \ No newline at end of file diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index f9e4f926b..7433c3857 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -5,17 +5,16 @@ @Author : alexanderwu @File : action.py """ -import re + from abc import ABC from typing import Optional -from tenacity import retry, stop_after_attempt, wait_fixed +from tenacity import retry, stop_after_attempt, wait_fixed, after_log from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM from metagpt.logs import logger from metagpt.utils.common import OutputParser -from metagpt.utils.custom_decoder import CustomDecoder from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, RepairType,\ retry_parse_json_text, extract_content_from_output @@ -51,7 +50,11 @@ class Action(ABC): system_msgs.append(self.prefix) return await self.llm.aask(prompt, system_msgs) - # @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) + @retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + after=after_log(logger, logger.level("ERROR").name), + ) async def _aask_v1( self, prompt: str, @@ -65,7 +68,7 @@ class Action(ABC): system_msgs = [] system_msgs.append(self.prefix) content = await self.llm.aask(prompt, system_msgs) - logger.debug(content) + logger.debug(f"llm raw output:\n{content}") output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) output_class_fields = list(output_class.schema()["properties"].keys()) # Custom ActionOutput's fields @@ -73,8 +76,8 @@ class Action(ABC): content = repair_llm_raw_output(content, req_keys=output_class_fields + ["[/CONTENT]"]) content = extract_content_from_output(content) content = repair_llm_raw_output(content, req_keys=[None], repair_type=RepairType.JSON) # req_keys mocked - logger.info(f"extracted CONTENT from content:\n{content}") - parsed_data = retry_parse_json_text(content) + logger.info(f"extracted json CONTENT from output:\n{content}") + parsed_data = retry_parse_json_text(output=content) # should use output=content else: # using markdown parser parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index a65e4be80..c26dc838d 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -4,8 +4,9 @@ import copy from enum import Enum -from typing import Union +from typing import Union, Callable import regex as re +from tenacity import retry, stop_after_attempt, wait_fixed, after_log, RetryCallState from metagpt.logs import logger from metagpt.config import CONFIG @@ -14,8 +15,8 @@ from metagpt.utils.custom_decoder import CustomDecoder class RepairType(Enum): CS = "case sensitivity" - SCM = "special character missing" # Usually the req_key appear in pairs like `[key] xx [/key]` RKPM = "required key pair missing" # condition like `[key] xx` which lacks `[/key]` + SCM = "special character missing" # Usually the req_key appear in pairs like `[key] xx [/key]` JSON = "json format" @@ -39,9 +40,11 @@ def repair_case_sensitivity(output: str, req_key: str) -> str: return output -def repair_special_character_missing(output: str, req_key: str) -> str: +def repair_special_character_missing(output: str, req_key: str = "[/CONTENT]") -> str: """ - fix target string `[CONTENT]xxx[/CONTENT]` lacks [/CONTENT] + fix + 1. target string `[CONTENT] xx [CONTENT] xxx [CONTENT]` lacks `/` in the last `[CONTENT]` + 2. target string `xx [CONTENT] xxx [CONTENT] xxxx` lacks `/` in the last `[CONTENT]` """ sc_arr = ["/"] @@ -55,30 +58,48 @@ def repair_special_character_missing(output: str, req_key: str) -> str: # req_key with special_character usually in the tail side ridx = output.rfind(req_key_pure) output = f"{output[:ridx]}{req_key}{output[ridx + len(req_key_pure):]}" - logger.info(f"repair_special_character_missing: {req_key}") + logger.info(f"repair_special_character_missing: {sc} in {req_key_pure} as position {ridx}") return output -def repair_required_key_pair_missing(output: str, req_key: str) -> str: +def repair_required_key_pair_missing(output: str, req_key: str = "[/CONTENT]") -> str: """ implement the req_key pair in the begin or end of the content req_key format 1. `[req_key]`, and its pair `[/req_key]` 2. `[/req_key]`, and its pair `[req_key]` """ + sc = "/" # special char if req_key.startswith("[") and req_key.endswith("]"): - if "/" in req_key: - left_key = req_key.replace("/", "") # `[/req_key]` -> `[req_key]` + if sc in req_key: + left_key = req_key.replace(sc, "") # `[/req_key]` -> `[req_key]` right_key = req_key else: left_key = req_key - right_key = f"{req_key[0]}/{req_key[1:]}" # `[req_key]` -> `[/req_key]` + right_key = f"{req_key[0]}{sc}{req_key[1:]}" # `[req_key]` -> `[/req_key]` if left_key not in output: - output = left_key + output + output = left_key + "\n" + output if right_key not in output: - output = output + right_key + def judge_potential_json(routput: str, left_key: str) -> Union[str, bool]: + routput = copy.deepcopy(routput) + ridx = routput.rfind(left_key) + if ridx < 0: + return None + sub_output = routput[ridx:] + idx1 = sub_output.rfind("}") + idx2 = sub_output.rindex("]") + idx = idx1 if idx1 >= idx2 else idx2 + sub_output = sub_output[: idx] + return sub_output + + if output.strip().endswith("}") or (output.strip().endswith("]") and not output.strip().endswith(left_key)): + # # avoid [req_key]xx[req_key] case to append [/req_key] + output = output + "\n" + right_key + elif judge_potential_json(output, left_key): + sub_content = judge_potential_json(output, left_key) + output = sub_content + "\n" + right_key return output @@ -106,12 +127,12 @@ def _repair_llm_raw_output(output: str, req_key: str, repair_type: RepairType = for repair_type in repair_types: if repair_type == RepairType.CS: output = repair_case_sensitivity(output, req_key) + elif repair_type == RepairType.RKPM: + output = repair_required_key_pair_missing(output, req_key) elif repair_type == RepairType.SCM: output = repair_special_character_missing(output, req_key) elif repair_type == RepairType.JSON: output = repair_json_format(output) - elif repair_type == RepairType.RKPM: - output = repair_required_key_pair_missing(output, req_key) return output @@ -178,25 +199,58 @@ def repair_invalid_json(output: str, error: str) -> str: return output -def retry_parse_json_text(output: str, retry: int = 5) -> Union[list, dict]: +def run_after_exp_and_passon_next_retry(logger: "loguru.Logger") -> Callable[["RetryCallState"], None]: + def run_and_passon(retry_state: RetryCallState) -> None: + """ + RetryCallState example + { + "start_time":143.098322024, + "retry_object":")>", + "fn":"", + "args":"(\"tag:[/CONTENT]\",)", # function input args + "kwargs":{}, # function input kwargs + "attempt_number":1, # retry number + "outcome":"", # type(outcome.result()) = "str", type(outcome.exception()) = "class" + "outcome_timestamp":143.098416904, + "idle_for":0, + "next_action":"None" + } + """ + if retry_state.outcome.failed: + if len(retry_state.args) > 0: + # # can't used as args=retry_state.args + func_param_output = retry_state.args[0] + elif len(retry_state.kwargs) > 0: + func_param_output = retry_state.kwargs.get("output", "") + # import pdb; pdb.set_trace() + exp_str = str(retry_state.outcome.exception()) + logger.warning(f"parse json from content inside [CONTENT][/CONTENT] failed at retry " + f"{retry_state.attempt_number}, try to fix it, exp: {exp_str}") + + repaired_output = repair_invalid_json(func_param_output, exp_str) + retry_state.kwargs["output"] = repaired_output + + return run_and_passon + + +@retry( + stop=stop_after_attempt(3 if CONFIG.repair_llm_output else 0), + wait=wait_fixed(1), + after=run_after_exp_and_passon_next_retry(logger), +) +def retry_parse_json_text(output: str) -> Union[list, dict]: """ repair the json-text situation like there are extra chars like [']', '}'] + + Warning + if CONFIG.repair_llm_output is False, retry _aask_v1 {x=3} times, and the retry_parse_json_text's retry not work + if CONFIG.repair_llm_output is True, the _aask_v1 and the retry_parse_json_text will loop for {x=3*3} times. + it's a two-layer retry cycle """ - parsed_data = {} - for idx in range(retry): - raw_output = copy.deepcopy(output) + logger.debug(f"output to json decode:\n{output}") - try: - parsed_data = CustomDecoder(strict=False).decode(output) - break - except Exception as exp: - if not CONFIG.repair_llm_output: - # if repair_llm_output is False, break from the retry loop - break - - logger.warning(f"decode content into json failed, try to fix it. exp: {exp}") - error = str(exp) - output = repair_invalid_json(output, error) + # if CONFIG.repair_llm_output is True, it will try to fix output until the retry break + parsed_data = CustomDecoder(strict=False).decode(output) return parsed_data diff --git a/tests/metagpt/utils/test_repair_llm_raw_output.py b/tests/metagpt/utils/test_repair_llm_raw_output.py index 39a7343e7..dfcf60ad5 100644 --- a/tests/metagpt/utils/test_repair_llm_raw_output.py +++ b/tests/metagpt/utils/test_repair_llm_raw_output.py @@ -42,20 +42,69 @@ def test_repair_special_character_missing(): req_keys=req_keys) assert output == target_output + raw_output = """[CONTENT] tag +[CONTENT] +{ +"Anything UNCLEAR": "No unclear requirements or information." +} +[CONTENT]""" + target_output = """[CONTENT] tag +[CONTENT] +{ +"Anything UNCLEAR": "No unclear requirements or information." +} +[/CONTENT]""" + output = repair_llm_raw_output(output=raw_output, + req_keys=req_keys) + assert output == target_output + + raw_output = '[CONTENT] {"a": "b"} [CONTENT]' + target_output = '[CONTENT] {"a": "b"} [/CONTENT]' + + output = repair_llm_raw_output(output=raw_output, + req_keys=["[/CONTENT]"]) + print("output\n", output) + assert output == target_output + def test_required_key_pair_missing(): - raw_output = "[CONTENT] xxx" - target_output = "[CONTENT] xxx[/CONTENT]" + raw_output = '[CONTENT] {"a": "b"}' + target_output = '[CONTENT] {"a": "b"}\n[/CONTENT]' output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) assert output == target_output - raw_output = "xxx[/CONTENT]" - target_output = "[CONTENT]xxx[/CONTENT]" + raw_output = '''[CONTENT] +{ + "a": "b" +]''' + target_output = '''[CONTENT] +{ + "a": "b" +] +[/CONTENT]''' output = repair_llm_raw_output(output=raw_output, - req_keys=["[CONTENT]"]) + req_keys=["[/CONTENT]"]) + assert output == target_output + + raw_output = '''[CONTENT] tag +[CONTENT] +{ + "a": "b" +} +xxx +''' + target_output = '''[CONTENT] tag +[CONTENT] +{ + "a": "b" +} +[/CONTENT] +''' + output = repair_llm_raw_output(output=raw_output, + req_keys=["[/CONTENT]"]) assert output == target_output @@ -100,6 +149,13 @@ def test_retry_parse_json_text(): def test_extract_content_from_output(): + """ + cases + xxx [CONTENT] xxxx [/CONTENT] + xxx [CONTENT] xxx [CONTENT] xxxx [/CONTENT] + xxx [CONTENT] xxxx [/CONTENT] xxx [CONTENT][/CONTENT] xxx [CONTENT][/CONTENT] # target pair is the last one + """ + output = 'Sure! Here is the properly formatted JSON output based on the given context:\n\n[CONTENT]\n{\n"' \ 'Required Python third-party packages": [\n"pygame==2.0.4",\n"pytest"\n],\n"Required Other language ' \ 'third-party packages": [\n"No third-party packages are required."\n],\n"Full API spec": "\nopenapi: ' \ From fc5c01e21943edc7b84376f62e5d6c9ef5634203 Mon Sep 17 00:00:00 2001 From: better629 Date: Wed, 22 Nov 2023 13:56:49 +0800 Subject: [PATCH 04/16] fix --- metagpt/utils/repair_llm_raw_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index c26dc838d..a12a36fcc 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -82,7 +82,7 @@ def repair_required_key_pair_missing(output: str, req_key: str = "[/CONTENT]") - if left_key not in output: output = left_key + "\n" + output if right_key not in output: - def judge_potential_json(routput: str, left_key: str) -> Union[str, bool]: + def judge_potential_json(routput: str, left_key: str) -> Union[str]: routput = copy.deepcopy(routput) ridx = routput.rfind(left_key) if ridx < 0: From 642335317b6a11f67da5b39fc84deca11249d331 Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 23 Nov 2023 01:46:14 +0800 Subject: [PATCH 05/16] add independent openllm and fireworks config fields, add llm output postprecess plugin --- config/config.yaml | 9 +++ metagpt/actions/action.py | 27 ++++--- metagpt/config.py | 15 +++- metagpt/llm.py | 7 +- metagpt/provider/fireworks_api.py | 24 +++++++ metagpt/provider/open_llm_api.py | 47 ++++++++++++ .../postprecess/base_postprecess_plugin.py | 72 +++++++++++++++++++ .../postprecess/llm_output_postprecess.py | 23 ++++++ metagpt/utils/repair_llm_raw_output.py | 14 ++-- .../utils/test_repair_llm_raw_output.py | 34 ++++++--- 10 files changed, 243 insertions(+), 29 deletions(-) create mode 100644 metagpt/provider/fireworks_api.py create mode 100644 metagpt/provider/open_llm_api.py create mode 100644 metagpt/provider/postprecess/base_postprecess_plugin.py create mode 100644 metagpt/provider/postprecess/llm_output_postprecess.py diff --git a/config/config.yaml b/config/config.yaml index 72d2c0b19..080de4000 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -34,6 +34,15 @@ RPM: 10 #### if zhipuai from `https://open.bigmodel.cn`. You can set here or export API_KEY="YOUR_API_KEY" # ZHIPUAI_API_KEY: "YOUR_API_KEY" +#### if use self-host open llm model with openai-compatible interface +#OPEN_LLM_API_BASE: "http://127.0.0.1:8000/v1" +#OPEN_LLM_API_MODEL: "llama2-13b" +# +##### if use Fireworks api +#FIREWORKS_API_KEY: "YOUR_API_KEY" +#FIREWORKS_API_BASE: "https://api.fireworks.ai/inference/v1" +#FIREWORKS_API_MODEL: "YOUR_LLM_MODEL" # example, accounts/fireworks/models/llama-v2-13b-chat + #### for Search ## Supported values: serpapi/google/serper/ddg diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 7433c3857..cb5bd9ce1 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -6,17 +6,29 @@ @File : action.py """ +import typing from abc import ABC from typing import Optional -from tenacity import retry, stop_after_attempt, wait_fixed, after_log +from tenacity import retry, stop_after_attempt, wait_fixed, after_log, _utils from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM from metagpt.logs import logger from metagpt.utils.common import OutputParser -from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, RepairType,\ - retry_parse_json_text, extract_content_from_output +from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess + + +def action_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: + def log_it(retry_state: "RetryCallState") -> None: + if retry_state.fn is None: + fn_name = "" + else: + fn_name = _utils.get_callback_name(retry_state.fn) + logger.error(f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " + f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " + f"exp: {retry_state.outcome.exception()}") + return log_it class Action(ABC): @@ -53,7 +65,7 @@ class Action(ABC): @retry( stop=stop_after_attempt(3), wait=wait_fixed(1), - after=after_log(logger, logger.level("ERROR").name), + after=action_after_log(logger), ) async def _aask_v1( self, @@ -70,14 +82,9 @@ class Action(ABC): content = await self.llm.aask(prompt, system_msgs) logger.debug(f"llm raw output:\n{content}") output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) - output_class_fields = list(output_class.schema()["properties"].keys()) # Custom ActionOutput's fields if format == "json": - content = repair_llm_raw_output(content, req_keys=output_class_fields + ["[/CONTENT]"]) - content = extract_content_from_output(content) - content = repair_llm_raw_output(content, req_keys=[None], repair_type=RepairType.JSON) # req_keys mocked - logger.info(f"extracted json CONTENT from output:\n{content}") - parsed_data = retry_parse_json_text(output=content) # should use output=content + parsed_data = llm_output_postprecess(output=content, schema=output_class.schema(), req_key="[/CONTENT]") else: # using markdown parser parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) diff --git a/metagpt/config.py b/metagpt/config.py index a4c43c28a..2ce75b013 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -46,10 +46,18 @@ class Config(metaclass=Singleton): self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") self.zhipuai_api_key = self._get("ZHIPUAI_API_KEY") + + self.open_llm_api_base = self._get("OPEN_LLM_API_BASE") + self.open_llm_api_model = self._get("OPEN_LLM_API_MODEL") + + self.fireworks_api_key = self._get("FIREWORKS_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and \ (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key) and \ - (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key): - raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") + (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key) and \ + (not self.open_llm_api_base) and \ + (not self.fireworks_api_key or "YOUR_API_KEY" == self.fireworks_api_key): + raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first " + "or FIREWORKS_API_KEY or OPEN_LLM_API_BASE") self.openai_api_base = self._get("OPENAI_API_BASE") openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy if openai_proxy: @@ -69,6 +77,9 @@ class Config(metaclass=Singleton): self.domain = self._get("DOMAIN") self.spark_url = self._get("SPARK_URL") + self.fireworks_api_base = self._get("FIREWORKS_API_BASE") + self.fireworks_api_model = self._get("FIREWORKS_API_MODEL") + self.claude_api_key = self._get("Anthropic_API_KEY") self.serpapi_api_key = self._get("SERPAPI_API_KEY") self.serper_api_key = self._get("SERPER_API_KEY") diff --git a/metagpt/llm.py b/metagpt/llm.py index 4edcd7a83..1f7d1b4c9 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -6,12 +6,13 @@ @File : llm.py """ -from metagpt.logs import logger from metagpt.config import CONFIG from metagpt.provider.anthropic_api import Claude2 as Claude from metagpt.provider.openai_api import OpenAIGPTAPI from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI from metagpt.provider.spark_api import SparkAPI +from metagpt.provider.open_llm_api import OpenLLMGPTAPI +from metagpt.provider.fireworks_api import FireWorksGPTAPI from metagpt.provider.human_provider import HumanProvider @@ -26,6 +27,10 @@ def LLM() -> "BaseGPTAPI": llm = SparkAPI() elif CONFIG.zhipuai_api_key: llm = ZhiPuAIGPTAPI() + elif CONFIG.open_llm_api_base: + llm = OpenLLMGPTAPI() + elif CONFIG.fireworks_api_key: + llm = FireWorksGPTAPI() else: raise RuntimeError("You should config a LLM configuration first") diff --git a/metagpt/provider/fireworks_api.py b/metagpt/provider/fireworks_api.py new file mode 100644 index 000000000..23126af2d --- /dev/null +++ b/metagpt/provider/fireworks_api.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : fireworks.ai's api + +import openai + +from metagpt.config import CONFIG +from metagpt.provider.openai_api import OpenAIGPTAPI, CostManager, RateLimiter + + +class FireWorksGPTAPI(OpenAIGPTAPI): + + def __init__(self): + self.__init_fireworks(CONFIG) + self.llm = openai + self.model = CONFIG.fireworks_api_model + self.auto_max_tokens = False + self._cost_manager = CostManager() + RateLimiter.__init__(self, rpm=self.rpm) + + def __init_fireworks(self, config: "Config"): + openai.api_key = config.fireworks_api_key + openai.api_base = config.fireworks_api_base + self.rpm = int(config.get("RPM", 10)) diff --git a/metagpt/provider/open_llm_api.py b/metagpt/provider/open_llm_api.py new file mode 100644 index 000000000..a6820b42b --- /dev/null +++ b/metagpt/provider/open_llm_api.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : self-host open llm model with openai-compatible interface + +import openai + +from metagpt.logs import logger +from metagpt.config import CONFIG +from metagpt.provider.openai_api import OpenAIGPTAPI, CostManager, RateLimiter + + +class OpenLLMCostManager(CostManager): + """ open llm model is self-host, it's free and without cost""" + + def update_cost(self, prompt_tokens, completion_tokens, model): + """ + Update the total cost, prompt tokens, and completion tokens. + + Args: + prompt_tokens (int): The number of tokens used in the prompt. + completion_tokens (int): The number of tokens used in the completion. + model (str): The model used for the API call. + """ + self.total_prompt_tokens += prompt_tokens + self.total_completion_tokens += completion_tokens + + logger.info( + f"Max budget: ${CONFIG.max_budget:.3f} | " + f"prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}" + ) + CONFIG.total_cost = self.total_cost + + +class OpenLLMGPTAPI(OpenAIGPTAPI): + + def __init__(self): + self.__init_openllm(CONFIG) + self.llm = openai + self.model = CONFIG.open_llm_api_model + self.auto_max_tokens = False + self._cost_manager = OpenLLMCostManager() + RateLimiter.__init__(self, rpm=self.rpm) + + def __init_openllm(self, config: "Config"): + openai.api_key = "sk-xx" # self-host api doesn't need api-key, use the default value + openai.api_base = config.open_llm_api_base + self.rpm = int(config.get("RPM", 10)) diff --git a/metagpt/provider/postprecess/base_postprecess_plugin.py b/metagpt/provider/postprecess/base_postprecess_plugin.py new file mode 100644 index 000000000..702a03194 --- /dev/null +++ b/metagpt/provider/postprecess/base_postprecess_plugin.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : base llm postprocess plugin to do the operations like repair the raw llm output + +from typing import Union + +from metagpt.logs import logger +from metagpt.utils.repair_llm_raw_output import RepairType +from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, extract_content_from_output, \ + retry_parse_json_text + + +class BasePostPrecessPlugin(object): + + model = None # the plugin of the `model`, use to judge in `llm_postprecess` + + def run_repair_llm_output(self, output: str, schema: dict, req_key: str = "[/CONTENT]") -> Union[dict, list]: + """ + repair steps + 1. repair the case sensitive problem using the schema's fields + 2. extract the content from the req_key pair( xx[REQ_KEY]xxx[/REQ_KEY]xx ) + 3. repair the invalid json text in the content + 4. parse the json text and repair it according to the exception with retry loop + """ + output_class_fields = list(schema["properties"].keys()) # Custom ActionOutput's fields + + content = self.run_repair_llm_raw_output(output, req_keys=output_class_fields + [req_key]) + content = self.run_extract_content_from_output(content, right_key=req_key) + # # req_keys mocked + content = self.run_repair_llm_raw_output(content, req_keys=[None], repair_type=RepairType.JSON) + parsed_data = self.run_retry_parse_json_text(content) + + return parsed_data + + def run_repair_llm_raw_output(self, content: str, req_keys: list[str], repair_type: str = None) -> str: + """ inherited class can re-implement the function""" + return repair_llm_raw_output(content, req_keys=req_keys, repair_type=repair_type) + + def run_extract_content_from_output(self, content: str, right_key: str) -> str: + """ inherited class can re-implement the function""" + return extract_content_from_output(content, right_key=right_key) + + def run_retry_parse_json_text(self, content: str) -> Union[dict, list]: + """ inherited class can re-implement the function""" + logger.info(f"extracted json CONTENT from output:\n{content}") + parsed_data = retry_parse_json_text(output=content) # should use output=content + return parsed_data + + def run(self, output: str, schema: dict, req_key: str = "[/CONTENT]") -> Union[dict, list]: + """ + this is used for prompt with a json-format output requirement and outer pair key, like + [REQ_KEY] + { + "Key": "value" + } + [/REQ_KEY] + + Args + outer (str): llm raw output + schema: output json schema + req_key: outer pair right key, usually in `[/REQ_KEY]` format + """ + assert len(schema.get("properties")) > 0 + assert "/" in req_key + + # current, postprocess only deal the repair_llm_raw_output + new_output = self.run_repair_llm_output( + output=output, + schema=schema, + req_key=req_key + ) + return new_output diff --git a/metagpt/provider/postprecess/llm_output_postprecess.py b/metagpt/provider/postprecess/llm_output_postprecess.py new file mode 100644 index 000000000..4b5955061 --- /dev/null +++ b/metagpt/provider/postprecess/llm_output_postprecess.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : the entry of choosing which PostProcessPlugin to deal particular LLM model's output + +from typing import Union + +from metagpt.provider.postprecess.base_postprecess_plugin import BasePostPrecessPlugin + + +def llm_output_postprecess(output: str, schema: dict, req_key: str = "[/CONTENT]", + model_name: str = None) -> Union[dict, str]: + """ + default use BasePostPrecessPlugin if there is not matched plugin. + """ + # TODO choose different model's plugin according to the model_name + postprecess_plugin = BasePostPrecessPlugin() + + result = postprecess_plugin.run( + output=output, + schema=schema, + req_key=req_key + ) + return result diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index a12a36fcc..4a632b80c 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -91,13 +91,13 @@ def repair_required_key_pair_missing(output: str, req_key: str = "[/CONTENT]") - idx1 = sub_output.rfind("}") idx2 = sub_output.rindex("]") idx = idx1 if idx1 >= idx2 else idx2 - sub_output = sub_output[: idx] + sub_output = sub_output[: idx+1] return sub_output if output.strip().endswith("}") or (output.strip().endswith("]") and not output.strip().endswith(left_key)): # # avoid [req_key]xx[req_key] case to append [/req_key] output = output + "\n" + right_key - elif judge_potential_json(output, left_key): + elif judge_potential_json(output, left_key) and (not output.strip().endswith(left_key)): sub_content = judge_potential_json(output, left_key) output = sub_content + "\n" + right_key @@ -116,7 +116,7 @@ def repair_json_format(output: str) -> str: elif output.endswith("}]"): output = output[:-1] logger.info(f"repair_json_format: {'}]'}") - elif output.startswith("{") and output.startswith("]"): + elif output.startswith("{") and output.endswith("]"): output = output[:-1] + "}" return output @@ -183,9 +183,11 @@ def repair_invalid_json(output: str, error: str) -> str: if line.endswith("],"): # problem, redundant char `]` line = line.replace("]", "") - elif line.endswith("},"): + elif line.endswith("},") and not output.endswith("},"): # problem, redundant char `}` line = line.replace("}", "") + elif line.endswith("},") and output.endswith("},"): + line = line[:-1] elif '",' not in line: line = f'{line}",' elif "," not in line: @@ -218,11 +220,10 @@ def run_after_exp_and_passon_next_retry(logger: "loguru.Logger") -> Callable[["R """ if retry_state.outcome.failed: if len(retry_state.args) > 0: - # # can't used as args=retry_state.args + # # can't be used as args=retry_state.args func_param_output = retry_state.args[0] elif len(retry_state.kwargs) > 0: func_param_output = retry_state.kwargs.get("output", "") - # import pdb; pdb.set_trace() exp_str = str(retry_state.outcome.exception()) logger.warning(f"parse json from content inside [CONTENT][/CONTENT] failed at retry " f"{retry_state.attempt_number}, try to fix it, exp: {exp_str}") @@ -265,6 +266,7 @@ def extract_content_from_output(content: str, right_key: str = "[/CONTENT]"): break return cont.strip() + # TODO construct the extract pattern with the `right_key` raw_content = copy.deepcopy(content) pattern = r"\[CONTENT\]([\s\S]*)\[/CONTENT\]" new_content = re_extract_content(raw_content, pattern) diff --git a/tests/metagpt/utils/test_repair_llm_raw_output.py b/tests/metagpt/utils/test_repair_llm_raw_output.py index dfcf60ad5..8779c965c 100644 --- a/tests/metagpt/utils/test_repair_llm_raw_output.py +++ b/tests/metagpt/utils/test_repair_llm_raw_output.py @@ -77,11 +77,11 @@ def test_required_key_pair_missing(): raw_output = '''[CONTENT] { - "a": "b" + "key": "value" ]''' target_output = '''[CONTENT] { - "a": "b" + "key": "value" ] [/CONTENT]''' @@ -92,17 +92,15 @@ def test_required_key_pair_missing(): raw_output = '''[CONTENT] tag [CONTENT] { - "a": "b" + "key": "value" } xxx ''' - target_output = '''[CONTENT] tag -[CONTENT] + target_output = '''[CONTENT] { - "a": "b" + "key": "value" } -[/CONTENT] -''' +[/CONTENT]''' output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) assert output == target_output @@ -117,6 +115,22 @@ def test_repair_json_format(): repair_type=RepairType.JSON) assert output == target_output + raw_output = "[{ xxx }" + target_output = "{ xxx }" + + output = repair_llm_raw_output(output=raw_output, + req_keys=[None], + repair_type=RepairType.JSON) + assert output == target_output + + raw_output = "{ xxx ]" + target_output = "{ xxx }" + + output = repair_llm_raw_output(output=raw_output, + req_keys=[None], + repair_type=RepairType.JSON) + assert output == target_output + def test_retry_parse_json_text(): invalid_json_text = """{ @@ -130,7 +144,7 @@ def test_retry_parse_json_text(): "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", "Requirement Analysis": "The requirements are clear and well-defined" } - output = retry_parse_json_text(invalid_json_text) + output = retry_parse_json_text(output=invalid_json_text) assert output == target_json invalid_json_text = """{ @@ -144,7 +158,7 @@ def test_retry_parse_json_text(): "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", "Requirement Analysis": "The requirements are clear and well-defined" } - output = retry_parse_json_text(invalid_json_text) + output = retry_parse_json_text(output=invalid_json_text) assert output == target_json From 9a2ac792fe1100c2f86783b547bfafcdeae2c95f Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 23 Nov 2023 01:55:56 +0800 Subject: [PATCH 06/16] add __init__ --- metagpt/provider/postprecess/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 metagpt/provider/postprecess/__init__.py diff --git a/metagpt/provider/postprecess/__init__.py b/metagpt/provider/postprecess/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/metagpt/provider/postprecess/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : From 1a67878b7e8ae3c07dc9306659ff551b18fe00b0 Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 23 Nov 2023 09:29:40 +0800 Subject: [PATCH 07/16] move after_log as general one --- metagpt/actions/action.py | 16 ++-------------- metagpt/utils/utils.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 metagpt/utils/utils.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index cb5bd9ce1..0a7a1656d 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -6,7 +6,6 @@ @File : action.py """ -import typing from abc import ABC from typing import Optional @@ -16,21 +15,10 @@ from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM from metagpt.logs import logger from metagpt.utils.common import OutputParser +from metagpt.utils.utils import general_after_log from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess -def action_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: - def log_it(retry_state: "RetryCallState") -> None: - if retry_state.fn is None: - fn_name = "" - else: - fn_name = _utils.get_callback_name(retry_state.fn) - logger.error(f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " - f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " - f"exp: {retry_state.outcome.exception()}") - return log_it - - class Action(ABC): def __init__(self, name: str = "", context=None, llm: LLM = None): self.name: str = name @@ -65,7 +53,7 @@ class Action(ABC): @retry( stop=stop_after_attempt(3), wait=wait_fixed(1), - after=action_after_log(logger), + after=general_after_log(logger), ) async def _aask_v1( self, diff --git a/metagpt/utils/utils.py b/metagpt/utils/utils.py new file mode 100644 index 000000000..f479ec3b8 --- /dev/null +++ b/metagpt/utils/utils.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : + +import typing + +from tenacity import after_log, _utils + + +def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: + def log_it(retry_state: "RetryCallState") -> None: + if retry_state.fn is None: + fn_name = "" + else: + fn_name = _utils.get_callback_name(retry_state.fn) + logger.error(f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " + f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " + f"exp: {retry_state.outcome.exception()}") + return log_it From 502bb2c4498b1a672a2f890a8fda479f812165c2 Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 23 Nov 2023 11:21:25 +0800 Subject: [PATCH 08/16] fix extract_content_from_output --- metagpt/utils/repair_llm_raw_output.py | 4 +++- tests/metagpt/utils/test_repair_llm_raw_output.py | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index 4a632b80c..0b521a7b0 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -275,13 +275,15 @@ def extract_content_from_output(content: str, right_key: str = "[/CONTENT]"): # TODO find a more general pattern # # for `[CONTENT]xxx[CONTENT]xxxx[/CONTENT] situation logger.warning(f"extract_content try another pattern: {pattern}") - raw_content = copy.deepcopy(new_content + right_key) + if right_key not in new_content: + raw_content = copy.deepcopy(new_content + "\n" + right_key) # # pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]" new_content = re_extract_content(raw_content, pattern) else: if right_key in new_content: idx = new_content.find(right_key) new_content = new_content[:idx] + new_content = new_content.strip() return new_content diff --git a/tests/metagpt/utils/test_repair_llm_raw_output.py b/tests/metagpt/utils/test_repair_llm_raw_output.py index 8779c965c..553b57625 100644 --- a/tests/metagpt/utils/test_repair_llm_raw_output.py +++ b/tests/metagpt/utils/test_repair_llm_raw_output.py @@ -216,7 +216,8 @@ def test_extract_content_from_output(): 'UNCLEAR:\n\n* How to start the game.\n\nGreat job! This JSON output should provide a clear and ' \ 'comprehensive overview of the project\'s requirements and dependencies.' output = extract_content_from_output(output) - assert output.startswith('{\n"Required Python third-party packages') + assert output.startswith('{\n"Required Python third-party packages') and \ + output.endswith('UNCLEAR": "How to start the game."\n]') output = 'Sure, I would be happy to help! Here is the information you provided, formatted as a JSON object ' \ 'inside the [CONTENT] tag:\n\n[CONTENT]\n{\n"Original Requirements": "Create a 2048 game",\n"Search ' \ @@ -245,7 +246,8 @@ def test_extract_content_from_output(): '[/CONTENT]\n\nI hope this helps! Let me know if you have any further questions or if there anything ' \ 'else I can do to assist you.' output = extract_content_from_output(output) - assert output.startswith('{\n"Original Requirements"') + assert output.startswith('{\n"Original Requirements"') and \ + output.endswith('"Anything UNCLEAR": ""\n}') output = """ Sure, I'd be happy to help! Here's the JSON output for the given context:\n\n[CONTENT]\n{ "Implementation approach": "We will use the open-source framework PyGame to create a 2D game engine, which will @@ -270,4 +272,5 @@ def test_extract_content_from_output(): information for a developer to understand the design and implementation of the 2048 game. """ output = extract_content_from_output(output) - assert output.startswith('{\n"Implementation approach"') and "[/CONTENT]" not in output + assert output.startswith('{\n"Implementation approach"') and \ + output.endswith('"Anything UNCLEAR": "The requirement is clear to me."\n}') From c25d5a73d16c4977e4dac630279d650d5e34db53 Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 23 Nov 2023 11:53:11 +0800 Subject: [PATCH 09/16] add ut test_repair_invalid_json --- metagpt/utils/repair_llm_raw_output.py | 18 ++++---- .../utils/test_repair_llm_raw_output.py | 42 +++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index 0b521a7b0..f9e6d020d 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -182,19 +182,23 @@ def repair_invalid_json(output: str, error: str) -> str: # different general problems if line.endswith("],"): # problem, redundant char `]` - line = line.replace("]", "") + new_line = line.replace("]", "") elif line.endswith("},") and not output.endswith("},"): # problem, redundant char `}` - line = line.replace("}", "") + new_line = line.replace("}", "") elif line.endswith("},") and output.endswith("},"): - line = line[:-1] - elif '",' not in line: - line = f'{line}",' + new_line = line[:-1] + elif '",' not in line and ',' not in line: + new_line = f'{line}",' elif "," not in line: # problem, miss char `,` at the end. - line = f"{line}," + new_line = f"{line}," + elif "," in line and len(line) == 1: + new_line = f'"{line}' + elif '",' in line: + new_line = line[:-2] + "'," - arr[line_no] = line + arr[line_no] = new_line output = "\n".join(arr) logger.info(f"repair_invalid_json, raw error: {error}") diff --git a/tests/metagpt/utils/test_repair_llm_raw_output.py b/tests/metagpt/utils/test_repair_llm_raw_output.py index 553b57625..acacb3af3 100644 --- a/tests/metagpt/utils/test_repair_llm_raw_output.py +++ b/tests/metagpt/utils/test_repair_llm_raw_output.py @@ -132,6 +132,48 @@ def test_repair_json_format(): assert output == target_output +def test_repair_invalid_json(): + raw_output = """{ + "key": "value" + }, +}""" + target_output = """{ + "key": "value" +, +}""" + output = repair_invalid_json(raw_output, "Expecting ',' delimiter: line 3 column 1") + assert output == target_output + + raw_output = """{ + "key": " +value + }, +}""" + target_output = """{ + "key": " +value +", +}""" + output = repair_invalid_json(raw_output, "Expecting ',' delimiter: line 4 column 1") + output = repair_invalid_json(output, "Expecting ',' delimiter: line 4 column 1") + assert output == target_output + + raw_output = """{ + "key": ' +value + }, +}""" + target_output = """{ + "key": ' +value +', +}""" + output = repair_invalid_json(raw_output, "Expecting ',' delimiter: line 4 column 1") + output = repair_invalid_json(output, "Expecting ',' delimiter: line 4 column 1") + output = repair_invalid_json(output, "Expecting ',' delimiter: line 4 column 1") + assert output == target_output + + def test_retry_parse_json_text(): invalid_json_text = """{ "Original Requirements": "Create a 2048 game", From 56b75b34ddd74013187a50630b92fe7bede0a40d Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 23 Nov 2023 13:38:33 +0800 Subject: [PATCH 10/16] rm not fully ready claude_api --- metagpt/llm.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 1f7d1b4c9..7b490ec4a 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -21,8 +21,6 @@ def LLM() -> "BaseGPTAPI": # TODO a little trick, can use registry to initialize LLM instance further if CONFIG.openai_api_key: llm = OpenAIGPTAPI() - elif CONFIG.claude_api_key: - llm = Claude() elif CONFIG.spark_api_key: llm = SparkAPI() elif CONFIG.zhipuai_api_key: From f1e01c5ba8b2246e763da3e7f850f0f4f8a30675 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 4 Dec 2023 11:12:13 +0800 Subject: [PATCH 11/16] set config value not relay on key.yaml --- tests/metagpt/utils/test_repair_llm_raw_output.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/metagpt/utils/test_repair_llm_raw_output.py b/tests/metagpt/utils/test_repair_llm_raw_output.py index acacb3af3..a2dd18516 100644 --- a/tests/metagpt/utils/test_repair_llm_raw_output.py +++ b/tests/metagpt/utils/test_repair_llm_raw_output.py @@ -2,7 +2,9 @@ # -*- coding: utf-8 -*- # @Desc : unittest of repair_llm_raw_output -import pytest + +from metagpt.config import CONFIG +CONFIG.repair_llm_output = True from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, RepairType, repair_invalid_json,\ extract_content_from_output, retry_parse_json_text From fb69c107feaf866f704299b003072baa3f760ef7 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 4 Dec 2023 21:56:43 +0800 Subject: [PATCH 12/16] rm useless deepcopy --- metagpt/utils/repair_llm_raw_output.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index f9e6d020d..124bcba89 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -82,8 +82,7 @@ def repair_required_key_pair_missing(output: str, req_key: str = "[/CONTENT]") - if left_key not in output: output = left_key + "\n" + output if right_key not in output: - def judge_potential_json(routput: str, left_key: str) -> Union[str]: - routput = copy.deepcopy(routput) + def judge_potential_json(routput: str, left_key: str) -> Union[str, None]: ridx = routput.rfind(left_key) if ridx < 0: return None From 7833e5767305153b4f2b0fb602f57ab72e6cf035 Mon Sep 17 00:00:00 2001 From: zeeland Date: Tue, 5 Dec 2023 16:30:46 +0800 Subject: [PATCH 13/16] pref: optimize log --- metagpt/logs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/metagpt/logs.py b/metagpt/logs.py index b2052e9b8..471d57fe9 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -7,18 +7,24 @@ """ import sys +from datetime import datetime from loguru import logger as _logger from metagpt.const import PROJECT_ROOT + def define_log_level(print_level="INFO", logfile_level="DEBUG"): """调整日志级别到level之上 Adjust the log level to above level """ + current_date = datetime.now() + formatted_date = current_date.strftime("%Y%m%d") + _logger.remove() _logger.add(sys.stderr, level=print_level) - _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level) + _logger.add(PROJECT_ROOT / f"logs/{formatted_date}.log", level=logfile_level) return _logger + logger = define_log_level() From d6cc0165fcc0862130388301d7ba04b74f937257 Mon Sep 17 00:00:00 2001 From: mo Date: Mon, 11 Dec 2023 09:21:55 +0800 Subject: [PATCH 14/16] fix prompts --- metagpt/prompts/generate_skill.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/prompts/generate_skill.md b/metagpt/prompts/generate_skill.md index 74948cd15..e96f8181a 100644 --- a/metagpt/prompts/generate_skill.md +++ b/metagpt/prompts/generate_skill.md @@ -10,7 +10,7 @@ from typing import Optional from abc import ABC from metagpt.llm import LLM # Large language model, similar to GPT -n + class Action(ABC): def __init__(self, name='', context=None, llm: LLM = LLM()): self.name = name From 12208154ee0fc5f913fe104ec722cae074c95d42 Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 12 Dec 2023 15:06:20 +0800 Subject: [PATCH 15/16] simplify code --- metagpt/utils/repair_llm_raw_output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index 124bcba89..0a461d360 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -222,10 +222,10 @@ def run_after_exp_and_passon_next_retry(logger: "loguru.Logger") -> Callable[["R } """ if retry_state.outcome.failed: - if len(retry_state.args) > 0: + if retry_state.args: # # can't be used as args=retry_state.args func_param_output = retry_state.args[0] - elif len(retry_state.kwargs) > 0: + elif retry_state.kwargs: func_param_output = retry_state.kwargs.get("output", "") exp_str = str(retry_state.outcome.exception()) logger.warning(f"parse json from content inside [CONTENT][/CONTENT] failed at retry " From 97cd9cd98d1a53481307c6a7014b675e1c0321af Mon Sep 17 00:00:00 2001 From: 0aaryan Date: Tue, 12 Dec 2023 15:20:29 +0530 Subject: [PATCH 16/16] Fix: Spelling errors in words (quoto -> quote) #521 --- metagpt/actions/debug_error.py | 2 +- metagpt/actions/design_api.py | 4 ++-- metagpt/actions/write_code.py | 2 +- metagpt/actions/write_prd.py | 4 ++-- metagpt/actions/write_test.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index d69a22dba..101cc2025 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -22,7 +22,7 @@ The message is as follows: {context} --- Now you should start rewriting the code: -## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. +## file name of the code to rewrite: Write code with triple quote. Do your best to implement THIS IN ONLY ONE FILE. """ class DebugError(Action): def __init__(self, name="DebugError", context=None, llm=None): diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 75df8b909..fc3be602b 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -33,7 +33,7 @@ Max Output: 8192 chars or 2048 tokens. Try to use them up. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## Python package name: Provide as Python str with python triple quote, concise and clear, characters only use a combination of all lowercase and underscores ## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here @@ -86,7 +86,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## Python package name: Provide as Python str with python triple quote, concise and clear, characters only use a combination of all lowercase and underscores ## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index a5dc8e059..a89bce60f 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -18,7 +18,7 @@ NOTICE Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code: {filename} Write code with triple quoto, based on the following list and context. +## Code: {filename} Write code with triple quote, based on the following list and context. 1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. 2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets 3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index bd04ca79e..52a99dafc 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -23,7 +23,7 @@ templates = { ## Search Information {search_information} -## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME +## mermaid quadrantChart code syntax example. DONT USE QUOTE IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME ```mermaid quadrantChart title Reach and engagement of campaigns @@ -108,7 +108,7 @@ and only output the json inside this tag, nothing else ## Search Information {search_information} -## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME +## mermaid quadrantChart code syntax example. DONT USE QUOTE IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME ```mermaid quadrantChart title Reach and engagement of campaigns diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 35ff36dc2..e2352b641 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -26,7 +26,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ``` Note that the code to test is at {source_file_path}, we will put your test code at {workspace}/tests/{test_file_name}, and run your test code from {workspace}, you should correctly import the necessary classes based on these file locations! -## {test_file_name}: Write test code with triple quoto. Do your best to implement THIS ONLY ONE FILE. +## {test_file_name}: Write test code with triple quote. Do your best to implement THIS ONLY ONE FILE. """