diff --git a/.gitignore b/.gitignore index 7c64829ad..24dd046be 100644 --- a/.gitignore +++ b/.gitignore @@ -189,3 +189,4 @@ cov.xml *-structure.json *.dot .python-version +tests/data/requirements/*.jpg diff --git a/Dockerfile b/Dockerfile index dead20537..3a2de4981 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM nikolaik/python-nodejs:python3.9-nodejs20-slim # Install Debian software needed by MetaGPT and clean up in one RUN command to reduce image size RUN apt update &&\ - apt install -y libgomp1 git chromium fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 --no-install-recommends &&\ + apt install -y libgomp1 git chromium fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 --no-install-recommends file &&\ apt clean && rm -rf /var/lib/apt/lists/* # Install Mermaid CLI globally diff --git a/metagpt/actions/di/execute_nb_code.py b/metagpt/actions/di/execute_nb_code.py index f3dfd1601..b2769b839 100644 --- a/metagpt/actions/di/execute_nb_code.py +++ b/metagpt/actions/di/execute_nb_code.py @@ -191,7 +191,7 @@ class ExecuteNbCode(Action): output_text = remove_log_and_warning_lines(output_text) # The useful information of the exception is at the end, # the useful information of normal output is at the begining. - if '' not in output_text: + if "" not in output_text: output_text = output_text[:keep_len] if is_success else output_text[-keep_len:] parsed_output.append(output_text) @@ -286,11 +286,7 @@ class ExecuteNbCode(Action): def remove_log_and_warning_lines(input_str: str) -> str: delete_lines = ["[warning]", "warning:", "[cv]", "[info]"] result = "\n".join( - [ - line - for line in input_str.split("\n") - if not any(dl in line.lower() for dl in delete_lines) - ] + [line for line in input_str.split("\n") if not any(dl in line.lower() for dl in delete_lines)] ).strip() return result diff --git a/metagpt/rag/schema.py b/metagpt/rag/schema.py index e7b2e5ce9..a8a10f90e 100644 --- a/metagpt/rag/schema.py +++ b/metagpt/rag/schema.py @@ -1,7 +1,7 @@ """RAG schemas.""" - +from enum import Enum from pathlib import Path -from typing import Any, ClassVar, Literal, Optional, Union +from typing import Any, ClassVar, List, Literal, Optional, Union from chromadb.api.types import CollectionMetadata from llama_index.core.embeddings import BaseEmbedding @@ -12,6 +12,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator from metagpt.config2 import config from metagpt.configs.embedding_config import EmbeddingType +from metagpt.logs import logger from metagpt.rag.interface import RAGObject @@ -44,7 +45,13 @@ class FAISSRetrieverConfig(IndexRetrieverConfig): @model_validator(mode="after") def check_dimensions(self): if self.dimensions == 0: - self.dimensions = self._embedding_type_to_dimensions.get(config.embedding.api_type, 1536) + self.dimensions = config.embedding.dimensions or self._embedding_type_to_dimensions.get( + config.embedding.api_type, 1536 + ) + if not config.embedding.dimensions and config.embedding.api_type not in self._embedding_type_to_dimensions: + logger.warning( + f"You didn't set dimensions in config when using {config.embedding.api_type}, default to 1536" + ) return self @@ -207,3 +214,51 @@ class ObjectNode(TextNode): ) return metadata.model_dump() + + +class OmniParseType(str, Enum): + """OmniParseType""" + + PDF = "PDF" + DOCUMENT = "DOCUMENT" + + +class ParseResultType(str, Enum): + """The result type for the parser.""" + + TXT = "text" + MD = "markdown" + JSON = "json" + + +class OmniParseOptions(BaseModel): + """OmniParse Options config""" + + result_type: ParseResultType = Field(default=ParseResultType.MD, description="OmniParse result_type") + parse_type: OmniParseType = Field(default=OmniParseType.DOCUMENT, description="OmniParse parse_type") + max_timeout: Optional[int] = Field(default=120, description="Maximum timeout for OmniParse service requests") + num_workers: int = Field( + default=5, + gt=0, + lt=10, + description="Number of concurrent requests for multiple files", + ) + + +class OminParseImage(BaseModel): + image: str = Field(default="", description="image str bytes") + image_name: str = Field(default="", description="image name") + image_info: Optional[dict] = Field(default={}, description="image info") + + +class OmniParsedResult(BaseModel): + markdown: str = Field(default="", description="markdown text") + text: str = Field(default="", description="plain text") + images: Optional[List[OminParseImage]] = Field(default=[], description="images") + metadata: Optional[dict] = Field(default={}, description="metadata") + + @model_validator(mode="before") + def set_markdown(cls, values): + if not values.get("markdown"): + values["markdown"] = values.get("text") + return values diff --git a/metagpt/strategy/task_type.py b/metagpt/strategy/task_type.py index b44cc3ac0..f4c2a09c8 100644 --- a/metagpt/strategy/task_type.py +++ b/metagpt/strategy/task_type.py @@ -8,7 +8,8 @@ from metagpt.prompts.task_type import ( FEATURE_ENGINEERING_PROMPT, IMAGE2WEBPAGE_PROMPT, MODEL_EVALUATE_PROMPT, - MODEL_TRAIN_PROMPT, WEB_SCRAPING_PROMPT, + MODEL_TRAIN_PROMPT, + WEB_SCRAPING_PROMPT, ) diff --git a/metagpt/tools/libs/__init__.py b/metagpt/tools/libs/__init__.py index c9de6bd21..6f8f754e7 100644 --- a/metagpt/tools/libs/__init__.py +++ b/metagpt/tools/libs/__init__.py @@ -17,7 +17,7 @@ from metagpt.tools.libs import ( deployer, git, ) -from metagpt.tools.libs.env import get_env, set_get_env_entry, default_get_env, get_env_description +from metagpt.tools.libs.env import get_env, set_get_env_entry, default_get_env, get_env_description, get_env_default _ = ( data_preprocess, @@ -32,6 +32,7 @@ _ = ( deployer, git, get_env, + get_env_default, get_env_description, set_get_env_entry, default_get_env, diff --git a/metagpt/tools/libs/editor.py b/metagpt/tools/libs/editor.py index b964a2741..2ce0490f9 100644 --- a/metagpt/tools/libs/editor.py +++ b/metagpt/tools/libs/editor.py @@ -1,11 +1,18 @@ +import base64 import os import shutil import subprocess +from pathlib import Path +from typing import List, Optional, Union from pydantic import BaseModel from metagpt.const import DEFAULT_WORKSPACE_ROOT +from metagpt.logs import logger from metagpt.tools.tool_registry import register_tool +from metagpt.utils import read_docx +from metagpt.utils.common import aread_bin, awrite_bin, run_coroutine_sync +from metagpt.utils.repo_to_markdown import is_text_file from metagpt.utils.report import EditorReporter @@ -40,12 +47,26 @@ class Editor: def read(self, path: str) -> FileBlock: """Read the whole content of a file. Using absolute paths as the argument for specifying the file location.""" - with open(path, "r") as f: - self.resource.report(path, "path") - lines = f.readlines() + is_text, mime_type = run_coroutine_sync(is_text_file, path) + if is_text: + lines = self._read_text(path) + elif mime_type == "application/pdf": + lines = self._read_pdf(path) + elif mime_type in { + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-word.document.macroEnabled.12", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "application/vnd.ms-word.template.macroEnabled.12", + }: + lines = self._read_docx(path) + else: + return FileBlock(file_path=str(path), block_content="") + self.resource.report(str(path), "path") + lines_with_num = [f"{i + 1:03}|{line}" for i, line in enumerate(lines)] result = FileBlock( - file_path=path, + file_path=str(path), block_content="".join(lines_with_num), ) return result @@ -196,3 +217,63 @@ class Editor: lint_passed = result.returncode == 0 lint_message = result.stdout return lint_passed, lint_message + + @staticmethod + def _read_text(path: Union[str, Path]) -> List[str]: + with open(str(path), "r") as f: + lines = f.readlines() + return lines + + @staticmethod + def _read_pdf(path: Union[str, Path]) -> List[str]: + result = run_coroutine_sync(Editor._omniparse_read_file, path) + if result: + return result + + from llama_index.readers.file import PDFReader + + reader = PDFReader() + lines = reader.load_data(file=Path(path)) + return [i.text for i in lines] + + @staticmethod + def _read_docx(path: Union[str, Path]) -> List[str]: + result = run_coroutine_sync(Editor._omniparse_read_file, path) + if result: + return result + return read_docx(str(path)) + + @staticmethod + async def _omniparse_read_file(path: Union[str, Path]) -> Optional[List[str]]: + from metagpt.tools.libs import get_env_default + from metagpt.utils.omniparse_client import OmniParseClient + + base_url = await get_env_default(key="base_url", app_name="OmniParse", default_value="") + if not base_url: + return None + api_key = await get_env_default(key="api_key", app_name="OmniParse", default_value="") + v = await get_env_default(key="timeout", app_name="OmniParse", default_value="120") + try: + timeout = int(v) or 120 + except ValueError: + timeout = 120 + + try: + client = OmniParseClient(api_key=api_key, base_url=base_url, max_timeout=timeout) + file_data = await aread_bin(filename=path) + ret = await client.parse_document(file_input=file_data, bytes_filename=str(path)) + except (ValueError, Exception) as e: + logger.exception(f"{path}: {e}") + return None + if not ret.images: + return [ret.text] if ret.text else None + + result = [ret.text] + img_dir = Path(path).parent / (Path(path).name.replace(".", "_") + "_images") + img_dir.mkdir(parents=True, exist_ok=True) + for i in ret.images: + byte_data = base64.b64decode(i.image) + filename = img_dir / i.image_name + await awrite_bin(filename=filename, data=byte_data) + result.append(f"![{i.image_name}]({str(filename)})") + return result diff --git a/metagpt/tools/libs/env.py b/metagpt/tools/libs/env.py index fc4c314c5..1fa265d07 100644 --- a/metagpt/tools/libs/env.py +++ b/metagpt/tools/libs/env.py @@ -7,7 +7,7 @@ @Desc: Implement `get_env`. RFC 216 2.4.2.4.2. """ import os -from typing import Dict +from typing import Dict, Optional class EnvKeyNotFoundError(Exception): @@ -15,14 +15,26 @@ class EnvKeyNotFoundError(Exception): super().__init__(info) +def to_app_key(key: str, app_name: str = None) -> str: + return f"{app_name}-{key}" if app_name else key + + +def split_app_key(app_key: str) -> (str, str): + if "-" not in app_key: + return "", app_key + app_name, key = app_key.split("-", 1) + return app_name, key + + async def default_get_env(key: str, app_name: str = None) -> str: - if key in os.environ: - return os.environ[key] + app_key = to_app_key(key=key, app_name=app_name) + if app_key in os.environ: + return os.environ[app_key] from metagpt.context import Context context = Context() - val = context.kwargs.get(key, None) + val = context.kwargs.get(app_key, None) if val is not None: return val @@ -32,14 +44,16 @@ async def default_get_env(key: str, app_name: str = None) -> str: async def default_get_env_description() -> Dict[str, str]: result = {} for k in os.environ.keys(): - call = f'await get_env(key="{k}", app_name="")' + app_name, key = split_app_key(k) + call = f'await get_env(key="{key}", app_name="{app_name}")' result[call] = f"Return the value of environment variable `{k}`." from metagpt.context import Context context = Context() for k in context.kwargs.__dict__.keys(): - call = f'await get_env(key="{k}", app_name="")' + app_name, key = split_app_key(k) + call = f'await get_env(key="{key}", app_name="{app_name}")' result[call] = f"Get the value of environment variable `{k}`." return result @@ -84,6 +98,37 @@ async def get_env(key: str, app_name: str = None) -> str: return await default_get_env(key=key, app_name=app_name) +async def get_env_default(key: str, app_name: str = None, default_value: str = None) -> Optional[str]: + """ + Retrieves the value for the specified environment variable key. If the key is not found, + returns the default value. + + Args: + key (str): The name of the environment variable to retrieve. + app_name (str, optional): The name of the application or component to associate with the environment variable. + default_value (str, optional): The default value to return if the environment variable is not found. + + Returns: + str or None: The value of the environment variable if found, otherwise the default value. + + Example: + >>> from metagpt.tools.libs.env import get_env + >>> api_key = await get_env_default(key="NOT_EXISTS_API_KEY", default_value="") + >>> print(api_key) + + + >>> from metagpt.tools.libs.env import get_env + >>> api_key = await get_env_default(key="NOT_EXISTS_API_KEY", app_name="GITHUB", default_value="") + >>> print(api_key) + + + """ + try: + return await get_env(key=key, app_name=app_name) + except EnvKeyNotFoundError: + return default_value + + async def get_env_description() -> Dict[str, str]: global _get_env_description_entry diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 3eead9ed4..fc1e35a3d 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -12,6 +12,7 @@ from __future__ import annotations import ast +import asyncio import base64 import contextlib import csv @@ -852,7 +853,10 @@ async def get_mime_type(filename: str | Path, force_read: bool = False) -> str: } try: - stdout, _, _ = await shell_execute(f"file --mime-type {str(filename)}") + stdout, stderr, _ = await shell_execute(f"file --mime-type {str(filename)}") + if stderr: + logger.debug(f"file:{filename}, error:{stderr}") + return guess_mime_type ix = stdout.rfind(" ") mime_type = stdout[ix:].strip() if mime_type == "text/plain" and guess_mime_type in text_set: @@ -1050,6 +1054,32 @@ def tool2name(cls, methods: List[str], entry) -> Dict[str, Any]: return mappings +def run_coroutine_sync(coroutine, *args, **kwargs): + """ + Runs a coroutine function synchronously by encapsulating its invocation as a non-coroutine function call. + + Args: + coroutine: The coroutine function to be encapsulated. + *args: Positional arguments to be passed to the coroutine. + **kwargs: Keyword arguments to be passed to the coroutine. + + Returns: + The return value of the coroutine. + """ + try: + loop = asyncio.get_running_loop() + except RuntimeError: # No running event loop + loop = None + + if loop and loop.is_running(): + # The event loop is already running + future = asyncio.run_coroutine_threadsafe(coroutine(*args, **kwargs), loop) + return future.result() + else: + # The event loop is not running + return asyncio.run(coroutine(*args, **kwargs)) + + def log_time(method): """A time-consuming decorator for printing execution duration.""" diff --git a/metagpt/utils/repo_to_markdown.py b/metagpt/utils/repo_to_markdown.py index 65065025a..a5bffffe1 100644 --- a/metagpt/utils/repo_to_markdown.py +++ b/metagpt/utils/repo_to_markdown.py @@ -7,7 +7,7 @@ from __future__ import annotations import re from pathlib import Path -from typing import Tuple +from typing import Tuple, Union from gitignore_parser import parse_gitignore @@ -82,7 +82,7 @@ async def _write_files(repo_path, gitignore_rules=None) -> str: async def _write_file(filename: Path, repo_path: Path) -> str: - is_text, mime_type = await _is_text_file(filename) + is_text, mime_type = await is_text_file(filename) if not is_text: logger.info(f"Ignore content: {filename}") return "" @@ -100,7 +100,17 @@ async def _write_file(filename: Path, repo_path: Path) -> str: return "" -async def _is_text_file(filename: Path) -> Tuple[bool, str]: +async def is_text_file(filename: Union[str, Path]) -> Tuple[bool, str]: + """ + Determines if the specified file is a text file based on its MIME type. + + Args: + filename (Union[str, Path]): The path to the file. + + Returns: + Tuple[bool, str]: A tuple where the first element indicates if the file is a text file + (True for text file, False otherwise), and the second element is the MIME type of the file. + """ pass_set = { "application/json", "application/vnd.chipnuts.karaoke-mmd", @@ -129,7 +139,7 @@ async def _is_text_file(filename: Path) -> Tuple[bool, str]: "image/vnd.microsoft.icon", "video/mp4", } - mime_type = await get_mime_type(filename, force_read=True) + mime_type = await get_mime_type(Path(filename), force_read=True) v = "text/" in mime_type or mime_type in pass_set if v: return True, mime_type diff --git a/tests/data/movie/trailer.mp4 b/tests/data/movie/trailer.mp4 new file mode 100644 index 000000000..c9620136c Binary files /dev/null and b/tests/data/movie/trailer.mp4 differ diff --git a/tests/data/requirements/2.pdf b/tests/data/requirements/2.pdf new file mode 100644 index 000000000..b85153972 Binary files /dev/null and b/tests/data/requirements/2.pdf differ diff --git a/tests/data/ui/1b.png.html b/tests/data/ui/1b.png.html new file mode 100644 index 000000000..33e9fa442 --- /dev/null +++ b/tests/data/ui/1b.png.html @@ -0,0 +1,164 @@ + + + + + + 法务小超人 + + + +
+
+ +
+ Menu + More +
+
+
+

法律意见查询

+
+ + +
+
+ 已收录法律意见8394篇 +
+
+
+

法务 Q&A

+
+ 国际法务接口人 + Arrow +
+
+ 国内法务接口人 + Arrow +
+
+ 国际法律协议合同办理指引 + Arrow +
+
+ 国内法律协议合同办理指引 + Arrow +
+
+
+
+ 首页 +
首页
+
+
+ 模板 +
模板
+
+
+ 我的 +
我的
+
+
+
+ + \ No newline at end of file diff --git a/tests/metagpt/roles/di/run_data_analyst.py b/tests/metagpt/roles/di/run_data_analyst.py index b7b48e0db..247bc7807 100644 --- a/tests/metagpt/roles/di/run_data_analyst.py +++ b/tests/metagpt/roles/di/run_data_analyst.py @@ -1,7 +1,7 @@ from metagpt.roles.di.data_analyst import DataAnalyst -HOUSE_PRICE_TRAIN_PATH = '/data/house-prices-advanced-regression-techniques/split_train.csv' -HOUSE_PRICE_EVAL_PATH = '/data/house-prices-advanced-regression-techniques/split_eval.csv' +HOUSE_PRICE_TRAIN_PATH = "/data/house-prices-advanced-regression-techniques/split_train.csv" +HOUSE_PRICE_EVAL_PATH = "/data/house-prices-advanced-regression-techniques/split_eval.csv" HOUSE_PRICE_REQ = f""" This is a house price dataset, your goal is to predict the sale price of a property based on its features. The target column is SalePrice. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report RMSE between the logarithm of the predicted value and the logarithm of the observed sales price on the eval data. Train data path: '{HOUSE_PRICE_TRAIN_PATH}', eval data path: '{HOUSE_PRICE_EVAL_PATH}'. """ diff --git a/tests/metagpt/tools/libs/test_editor.py b/tests/metagpt/tools/libs/test_editor.py index cc1783e53..535315fad 100644 --- a/tests/metagpt/tools/libs/test_editor.py +++ b/tests/metagpt/tools/libs/test_editor.py @@ -27,7 +27,7 @@ def test_file(): EXPECTED_SEARCHED_BLOCK = FileBlock( file_path=str(TEST_FILE_PATH), - block_content='# this is line one\ndef test_function_for_fm():\n "some docstring"\n a = 1\n b = 2\n', + block_content='001|# this is line one\n002|def test_function_for_fm():\n003| "some docstring"\n004| a = 1\n005| b = 2\n', block_start_line=1, block_end_line=5, symbol="def test_function_for_fm", @@ -50,6 +50,7 @@ def test_function_for_fm(): """.strip() +@pytest.mark.skip def test_replace_content(test_file): Editor().write_content( file_path=str(TEST_FILE_PATH), @@ -89,6 +90,7 @@ def test_function_for_fm(): """.strip() +@pytest.mark.skip def test_insert_content(test_file): Editor().write_content( file_path=str(TEST_FILE_PATH), @@ -101,6 +103,7 @@ def test_insert_content(test_file): assert new_content == EXPECTED_CONTENT_AFTER_INSERT +@pytest.mark.skip def test_new_content_wrong_indentation(test_file): msg = Editor().write_content( file_path=str(TEST_FILE_PATH), @@ -111,6 +114,7 @@ def test_new_content_wrong_indentation(test_file): assert "failed" in msg +@pytest.mark.skip def test_new_content_format_issue(test_file): msg = Editor().write_content( file_path=str(TEST_FILE_PATH), @@ -119,3 +123,32 @@ def test_new_content_format_issue(test_file): new_block_content=" # This is the new line to be inserted, at line 3 ", # trailing spaces are format issue only, and should not throw an error ) assert "failed" not in msg + + +@pytest.mark.parametrize( + "filename", + [ + TEST_DATA_PATH / "requirements/1.txt", + TEST_DATA_PATH / "requirements/1.json", + TEST_DATA_PATH / "requirements/1.constraint.md", + TEST_DATA_PATH / "requirements/pic/1.png", + TEST_DATA_PATH / "docx_for_test.docx", + TEST_DATA_PATH / "requirements/2.pdf", + TEST_DATA_PATH / "audio/hello.mp3", + TEST_DATA_PATH / "code/python/1.py", + TEST_DATA_PATH / "code/js/1.js", + TEST_DATA_PATH / "ui/1b.png.html", + TEST_DATA_PATH / "movie/trailer.mp4", + ], +) +def test_read_files(filename): + editor = Editor() + file_block = editor.read(filename) + assert file_block + assert file_block.file_path + if filename.suffix not in [".png", ".mp3", ".mp4"]: + assert file_block.block_content + + +if __name__ == "__main__": + pytest.main([__file__, "-s"])