add gpt-4v support for aask and AN.fill

tests/metagpt/actions/test_action_node.py

@@ -5,6 +5,7 @@
 @Author  : alexanderwu
 @File    : test_action_node.py
 """
+from pathlib import Path
 from typing import List, Tuple
 
 import pytest
@@ -17,6 +18,7 @@ from metagpt.llm import LLM
 from metagpt.roles import Role
 from metagpt.schema import Message
 from metagpt.team import Team
+from metagpt.utils.common import encode_image
 
 
 @pytest.mark.asyncio
@@ -241,6 +243,18 @@ def test_create_model_class_with_mapping():
     assert value == ["game.py", "app.py", "static/css/styles.css", "static/js/script.js", "templates/index.html"]
 
 
+@pytest.mark.asyncio
+async def test_action_node_with_image():
+    invoice = ActionNode(
+        key="invoice", expected_type=bool, instruction="if it's a invoice file, return True else False", example="False"
+    )
+
+    invoice_path = Path(__file__).parent.joinpath("..", "..", "data", "invoices", "invoice-2.png")
+    img_base64 = encode_image(invoice_path)
+    node = await invoice.fill(context="", llm=LLM(), images=[img_base64])
+    assert node.instruct_content.invoice
+
+
 if __name__ == "__main__":
     test_create_model_class()
     test_create_model_class_with_mapping()

tests/mock/mock_llm.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, Union
 
 from metagpt.config2 import config
 from metagpt.logs import log_llm_stream, logger
@@ -35,6 +35,7 @@ class MockLLM(OpenAILLM):
         msg: str,
         system_msgs: Optional[list[str]] = None,
         format_msgs: Optional[list[dict[str, str]]] = None,
+        images: Optional[Union[str, list[str]]] = None,
         timeout=3,
         stream=True,
     ):
@@ -47,7 +48,7 @@ class MockLLM(OpenAILLM):
             message = []
         if format_msgs:
             message.extend(format_msgs)
-        message.append(self._user_msg(msg))
+        message.append(self._user_msg(msg, images=images))
         rsp = await self.acompletion_text(message, stream=stream, timeout=timeout)
         return rsp
 
@@ -66,6 +67,7 @@ class MockLLM(OpenAILLM):
         msg: str,
         system_msgs: Optional[list[str]] = None,
         format_msgs: Optional[list[dict[str, str]]] = None,
+        images: Optional[Union[str, list[str]]] = None,
         timeout=3,
         stream=True,
     ) -> str:
@@ -73,7 +75,7 @@ class MockLLM(OpenAILLM):
         if system_msgs:
             joined_system_msg = "#MSG_SEP#".join(system_msgs) + "#SYSTEM_MSG_END#"
             msg_key = joined_system_msg + msg_key
-        rsp = await self._mock_rsp(msg_key, self.original_aask, msg, system_msgs, format_msgs, timeout, stream)
+        rsp = await self._mock_rsp(msg_key, self.original_aask, msg, system_msgs, format_msgs, images, timeout, stream)
         return rsp
 
     async def aask_batch(self, msgs: list, timeout=3) -> str: