Merge branch 'feature/omniparse_config' into 'mgx_ops'

feat: +omniparse config

See merge request pub/MetaGPT!323

metagpt/config2.py

@@ -16,6 +16,7 @@ from metagpt.configs.embedding_config import EmbeddingConfig
 from metagpt.configs.exp_pool_config import ExperiencePoolConfig
 from metagpt.configs.llm_config import LLMConfig, LLMType
 from metagpt.configs.mermaid_config import MermaidConfig
+from metagpt.configs.omniparse_config import OmniParseConfig
 from metagpt.configs.redis_config import RedisConfig
 from metagpt.configs.role_custom_config import RoleCustomConfig
 from metagpt.configs.s3_config import S3Config
@@ -88,6 +89,8 @@ class Config(CLIParams, YamlModel):
     # Role's custom configuration
     roles: Optional[List[RoleCustomConfig]] = None
 
+    omniparse: Optional[OmniParseConfig] = None
+
     @classmethod
     def from_home(cls, path):
         """Load config from ~/.metagpt/config2.yaml"""

metagpt/configs/omniparse_config.py

@@ -0,0 +1,5 @@
+from metagpt.utils.yaml_model import YamlModel
+
+
+class OmniParseConfig(YamlModel):
+    url: str = ""

metagpt/tools/libs/editor.py

@@ -7,10 +7,11 @@ from typing import List, Optional, Union
 
 from pydantic import BaseModel, ConfigDict
 
+from metagpt.config2 import Config
 from metagpt.logs import logger
 from metagpt.tools.tool_registry import register_tool
 from metagpt.utils import read_docx
-from metagpt.utils.common import aread, aread_bin, awrite_bin
+from metagpt.utils.common import aread, aread_bin, awrite_bin, check_http_endpoint
 from metagpt.utils.repo_to_markdown import is_text_file
 from metagpt.utils.report import EditorReporter
 
@@ -251,6 +252,8 @@ class Editor(BaseModel):
         from metagpt.utils.omniparse_client import OmniParseClient
 
         base_url = await get_env_default(key="base_url", app_name="OmniParse", default_value="")
+        if not base_url:
+            base_url = await Editor._read_omniparse_config()
         if not base_url:
             return None
         api_key = await get_env_default(key="api_key", app_name="OmniParse", default_value="")
@@ -261,6 +264,9 @@ class Editor(BaseModel):
             timeout = 120
 
         try:
+            if not await check_http_endpoint(url=base_url):
+                logger.warning(f"{base_url}: NOT AVAILABLE")
+                return None
             client = OmniParseClient(api_key=api_key, base_url=base_url, max_timeout=timeout)
             file_data = await aread_bin(filename=path)
             ret = await client.parse_document(file_input=file_data, bytes_filename=str(path))
@@ -279,3 +285,10 @@ class Editor(BaseModel):
             await awrite_bin(filename=filename, data=byte_data)
             result.append(f"![{i.image_name}]({str(filename)})")
         return result
+
+    @staticmethod
+    async def _read_omniparse_config() -> str:
+        config = Config.default()
+        if config.omniparse and config.omniparse.url:
+            return config.omniparse.url
+        return ""

metagpt/utils/common.py

@@ -35,6 +35,7 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
 from urllib.parse import quote, unquote
 
 import aiofiles
+import aiohttp
 import chardet
 import loguru
 import requests
@@ -1120,6 +1121,26 @@ def log_time(method):
     return timeit_wrapper_async if iscoroutinefunction(method) else timeit_wrapper
 
 
+async def check_http_endpoint(url: str, timeout: int = 3) -> bool:
+    """
+    Checks the status of an HTTP endpoint.
+
+    Args:
+        url (str): The URL of the HTTP endpoint to check.
+        timeout (int, optional): The timeout in seconds for the HTTP request. Defaults to 3.
+
+    Returns:
+        bool: True if the endpoint is online and responding with a 200 status code, False otherwise.
+    """
+    async with aiohttp.ClientSession() as session:
+        try:
+            async with session.get(url, timeout=timeout) as response:
+                return response.status == 200
+        except Exception as e:
+            print(f"Error accessing the endpoint {url}: {e}")
+            return False
+
+
 def rectify_pathname(path: Union[str, Path], default_filename: str) -> Path:
     """
     Rectifies the given path to ensure a valid output file path.