From 7f8ae1f260f0db71a8c2bf72034fa544db7c6358 Mon Sep 17 00:00:00 2001 From: better629 Date: Fri, 26 Jan 2024 15:16:33 +0800 Subject: [PATCH] update gpt4-v --- examples/llm_hello_world.py | 6 ++---- metagpt/actions/action_node.py | 16 ++++++++++++++-- metagpt/const.py | 2 +- metagpt/provider/base_llm.py | 9 ++------- metagpt/utils/common.py | 2 +- metagpt/utils/token_counter.py | 2 +- tests/metagpt/actions/test_action_node.py | 8 ++------ 7 files changed, 23 insertions(+), 22 deletions(-) diff --git a/examples/llm_hello_world.py b/examples/llm_hello_world.py index 4baeaa01e..dfc2603aa 100644 --- a/examples/llm_hello_world.py +++ b/examples/llm_hello_world.py @@ -33,13 +33,11 @@ async def main(): invoice_path = Path(__file__).parent.joinpath("..", "tests", "data", "invoices", "invoice-2.png") img_base64 = encode_image(invoice_path) try: - res = await llm.aask(msg="if this is a invoice, just return True else return False", - images=[img_base64]) + res = await llm.aask(msg="if this is a invoice, just return True else return False", images=[img_base64]) assert "true" in res.lower() - except Exception as exp: + except Exception: pass - if __name__ == "__main__": asyncio.run(main()) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index dfe8b0aae..923d57ca6 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -395,7 +395,9 @@ class ActionNode: if schema != "raw": mapping = self.get_mapping(mode, exclude=exclude) class_name = f"{self.key}_AN" - content, scontent = await self._aask_v1(prompt, class_name, mapping, images=images, schema=schema, timeout=timeout) + content, scontent = await self._aask_v1( + prompt, class_name, mapping, images=images, schema=schema, timeout=timeout + ) self.content = content self.instruct_content = scontent else: @@ -404,7 +406,17 @@ class ActionNode: return self - async def fill(self, context, llm, schema="json", mode="auto", strgy="simple", images: Optional[Union[str, list[str]]] = None, timeout=3, exclude=[]): + async def fill( + self, + context, + llm, + schema="json", + mode="auto", + strgy="simple", + images: Optional[Union[str, list[str]]] = None, + timeout=3, + exclude=[], + ): """Fill the node(s) with mode. :param context: Everything we should know when filling node. diff --git a/metagpt/const.py b/metagpt/const.py index b21bbc282..7c73f106f 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -167,4 +167,4 @@ MC_CURRICULUM_OB = [ "failed_tasks", ] MC_CORE_INVENTORY_ITEMS = r".*_log|.*_planks|stick|crafting_table|furnace" -r"|cobblestone|dirt|coal|.*_pickaxe|.*_sword|.*_axe", # curriculum_agent: only show these items in inventory before optional_inventory_items reached in warm up \ No newline at end of file +r"|cobblestone|dirt|coal|.*_pickaxe|.*_sword|.*_axe", # curriculum_agent: only show these items in inventory before optional_inventory_items reached in warm up diff --git a/metagpt/provider/base_llm.py b/metagpt/provider/base_llm.py index f9e9cddc9..7c5892018 100644 --- a/metagpt/provider/base_llm.py +++ b/metagpt/provider/base_llm.py @@ -47,17 +47,12 @@ class BaseLLM(ABC): """ if isinstance(images, str): images = [images] - content = [ - {"type": "text", "text": msg} - ] + content = [{"type": "text", "text": msg}] for image in images: # image url or image base64 url = image if image.startswith("http") else f"data:image/jpeg;base64,{image}" # it can with multiple-image inputs - content.append({ - "type": "image_url", - "image_url": url - }) + content.append({"type": "image_url", "image_url": url}) return {"role": "user", "content": content} def _assistant_msg(self, msg: str) -> dict[str, str]: diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 1cc482852..142b88620 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -12,6 +12,7 @@ from __future__ import annotations import ast +import base64 import contextlib import csv import importlib @@ -23,7 +24,6 @@ import re import sys import traceback import typing -import base64 from pathlib import Path from typing import Any, List, Tuple, Union diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index 1f6622d28..fadf77553 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -78,7 +78,7 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): "gpt-4-32k-0613", "gpt-4-1106-preview", "gpt-4-vision-preview", - "gpt-4-1106-vision-preview" + "gpt-4-1106-vision-preview", }: tokens_per_message = 3 # # every reply is primed with <|start|>assistant<|message|> tokens_per_name = 1 diff --git a/tests/metagpt/actions/test_action_node.py b/tests/metagpt/actions/test_action_node.py index ccda665a1..8aee071d4 100644 --- a/tests/metagpt/actions/test_action_node.py +++ b/tests/metagpt/actions/test_action_node.py @@ -5,11 +5,10 @@ @Author : alexanderwu @File : test_action_node.py """ +from pathlib import Path from typing import List, Tuple import pytest -import base64 -from pathlib import Path from pydantic import ValidationError from metagpt.actions import Action @@ -247,10 +246,7 @@ def test_create_model_class_with_mapping(): @pytest.mark.asyncio async def test_action_node_with_image(): invoice = ActionNode( - key="invoice", - expected_type=bool, - instruction="if it's a invoice file, return True else False", - example="False" + key="invoice", expected_type=bool, instruction="if it's a invoice file, return True else False", example="False" ) invoice_path = Path(__file__).parent.joinpath("..", "..", "data", "invoices", "invoice-2.png")