mirror of
https://github.com/FoundationAgents/MetaGPT.git
synced 2026-05-03 12:52:37 +02:00
fix conflict
This commit is contained in:
seehi
commit
d2bd9055f3
29 changed files with 708 additions and 45 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -189,3 +189,4 @@ cov.xml
|
|||
*-structure.json
|
||||
*.dot
|
||||
.python-version
|
||||
tests/data/requirements/*.jpg
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ FROM nikolaik/python-nodejs:python3.9-nodejs20-slim
|
|||
|
||||
# Install Debian software needed by MetaGPT and clean up in one RUN command to reduce image size
|
||||
RUN apt update &&\
|
||||
apt install -y libgomp1 git chromium fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 --no-install-recommends &&\
|
||||
apt install -y libgomp1 git chromium fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 --no-install-recommends file &&\
|
||||
apt clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Mermaid CLI globally
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ class WriteDesign(Action):
|
|||
prd_filename (str, optional): The filename of the Product Requirement Document (PRD).
|
||||
legacy_design_filename (str, optional): The filename of the legacy design document.
|
||||
extra_info (str, optional): Additional information to be included in the system design.
|
||||
output_pathname (str, optional): The output path name of file that the system design should be saved to.
|
||||
output_pathname (str, optional): The output file path of the document.
|
||||
|
||||
Returns:
|
||||
str: The file path of the generated system design.
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ class ExecuteNbCode(Action):
|
|||
output_text = remove_log_and_warning_lines(output_text)
|
||||
# The useful information of the exception is at the end,
|
||||
# the useful information of normal output is at the begining.
|
||||
if '<!DOCTYPE html>' not in output_text:
|
||||
if "<!DOCTYPE html>" not in output_text:
|
||||
output_text = output_text[:keep_len] if is_success else output_text[-keep_len:]
|
||||
|
||||
parsed_output.append(output_text)
|
||||
|
|
@ -286,11 +286,7 @@ class ExecuteNbCode(Action):
|
|||
def remove_log_and_warning_lines(input_str: str) -> str:
|
||||
delete_lines = ["[warning]", "warning:", "[cv]", "[info]"]
|
||||
result = "\n".join(
|
||||
[
|
||||
line
|
||||
for line in input_str.split("\n")
|
||||
if not any(dl in line.lower() for dl in delete_lines)
|
||||
]
|
||||
[line for line in input_str.split("\n") if not any(dl in line.lower() for dl in delete_lines)]
|
||||
).strip()
|
||||
return result
|
||||
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ class WriteTasks(Action):
|
|||
|
||||
Args:
|
||||
user_requirement (str, optional): A string specifying the user's requirements. Defaults to an empty string.
|
||||
design_filename (str): The filename of the project system design file. Defaults to an empty string.
|
||||
design_filename (str): The output file path of the document. Defaults to an empty string.
|
||||
output_pathname (str, optional): The output path name of file that the project schedule should be saved to.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
|
|
@ -73,8 +73,9 @@ class WriteTasks(Action):
|
|||
# Write a project schedule with a given system design.
|
||||
>>> design_filename = "/absolute/path/to/snake_game/docs/system_design.json"
|
||||
>>> output_pathname = "/absolute/path/to/snake_game/docs/project_schedule.json"
|
||||
>>> user_requirement = "Write project schedule for a snake game following these requirements:..."
|
||||
>>> action = WriteTasks()
|
||||
>>> result = await action.run(design_filename=design_filename, output_pathname=output_pathname)
|
||||
>>> result = await action.run(user_requirement=user_requirement, design_filename=design_filename, output_pathname=output_pathname)
|
||||
>>> print(result)
|
||||
The project schedule is at /absolute/path/to/snake_game/docs/project_schedule.json
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from metagpt.actions.action_node import ActionNode
|
|||
REQUIRED_PYTHON_PACKAGES = ActionNode(
|
||||
key="Required Python packages",
|
||||
expected_type=List[str],
|
||||
instruction="Provide required Python packages in requirements.txt format.",
|
||||
instruction="Provide required Python packages in requirements.txt format. The response language should correspond to the context and requirements.",
|
||||
example=["flask==1.1.2", "bcrypt==3.2.0"],
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ class SearchEnhancedQA(Action):
|
|||
description="Action to explore the web and provide summaries of articles and webpages.",
|
||||
)
|
||||
per_page_timeout: float = Field(
|
||||
default=10, description="The maximum time for fetching a single page is in seconds. Defaults to 10s."
|
||||
default=20, description="The maximum time for fetching a single page is in seconds. Defaults to 20s."
|
||||
)
|
||||
java_script_enabled: bool = Field(
|
||||
default=False, description="Whether or not to enable JavaScript in the web browser context. Defaults to False."
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class WritePRD(Action):
|
|||
|
||||
Args:
|
||||
user_requirement (str): A string detailing the user's requirements.
|
||||
output_pathname (str, optional): The path name of file that the output document should be saved to. Defaults to "".
|
||||
output_pathname (str, optional): The output file path of the document. Defaults to "".
|
||||
legacy_prd_filename (str, optional): The file path of the legacy Product Requirement Document to use as a reference. Defaults to "".
|
||||
extra_info (str, optional): Additional information to include in the document. Defaults to "".
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
|
|
|||
|
|
@ -124,7 +124,9 @@ class MGXEnv(Environment):
|
|||
if converted_msg.role not in ["system", "user", "assistant"]:
|
||||
converted_msg.role = "assistant"
|
||||
sent_from = converted_msg.metadata[AGENT] if AGENT in converted_msg.metadata else converted_msg.sent_from
|
||||
converted_msg.content = f"from {sent_from} to {converted_msg.send_to}: {converted_msg.content}"
|
||||
converted_msg.content = (
|
||||
f"[Message] from {sent_from if sent_from else 'User'} to {converted_msg.send_to}: {converted_msg.content}"
|
||||
)
|
||||
return converted_msg
|
||||
|
||||
def __repr__(self):
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ Notice: your output JSON data section must start with **```json [**
|
|||
"""
|
||||
THOUGHT_GUIDANCE = """
|
||||
First, describe the actions you have taken recently.
|
||||
Second, describe the messages you have received recently, with a particular emphasis on messages from users.
|
||||
Second, describe the messages you have received recently, with a particular emphasis on messages from users. If necessary, develop a plan to address the new user requirements.
|
||||
Third, describe the plan status and the current task. Review the histroy, if `Current Task` has been undertaken and completed by you or anyone, you MUST use the **Plan.finish_current_task** command to finish it first before taking any action, the command will automatically move you to the next task.
|
||||
Fourth, describe any necessary human interaction. Use **RoleZero.reply_to_human** to report your progress if you complete a task or the overall requirement, pay attention to the history, DON'T repeat reporting. Use **RoleZero.ask_human** if you failed the current task, unsure of the situation encountered, need any help from human, or executing repetitive commands but receiving repetitive feedbacks without making progress.
|
||||
Fifth, describe if you should terminate, you should use **end** command to terminate if any of the following is met:
|
||||
|
|
|
|||
|
|
@ -224,6 +224,8 @@ IMPORTANT_TIPS = """
|
|||
14. If provided an issue link, you MUST go to the issue page using Browser tool to understand the issue before starting your fix.
|
||||
|
||||
15. When the edit fails, try to enlarge the starting line.
|
||||
|
||||
16. Once again, and this is critical: YOU CAN ONLY ENTER ONE COMMAND AT A TIME.
|
||||
"""
|
||||
|
||||
NEXT_STEP_TEMPLATE = f"""
|
||||
|
|
|
|||
|
|
@ -30,12 +30,13 @@ Note:
|
|||
10. Do not use escape characters in json data, particularly within file paths.
|
||||
11. Analyze the capabilities of team members and assign tasks to them based on user Requirements. If the requirements ask to ignore certain tasks, follow the requirements.
|
||||
12. Add default web technologies: HTML (*.html), CSS (*.css), and JavaScript (*.js) to your requirements.If no specific programming language is required, include these technologies in the project requirements. Using instruction to forward this information to your team members.
|
||||
13. If the the user message is a question. use 'reply to human' to respond to the question, and then end.
|
||||
"""
|
||||
TL_THOUGHT_GUIDANCE = (
|
||||
THOUGHT_GUIDANCE
|
||||
+ """
|
||||
Sixth, when planning, describe the requirements as they pertain to software development, data analysis, or other areas. If the requirements is a software development and no specific restrictions are mentioned, you must create a Product Requirements Document (PRD), write a System Design document, develop a project schedule, and then begin coding. List the steps you will undertake. Plan these steps in a single response.
|
||||
Seventh, describe the technologies you must use.
|
||||
Sixth, describe the requirements as they pertain to software development, data analysis, or other areas. If the requirements is a software development and no specific restrictions are mentioned, you must create a Product Requirements Document (PRD), write a System Design document, develop a project schedule, and then begin coding. List the steps you will undertake. Plan these steps in a single response.
|
||||
Seventh, describe the technologies you must use.
|
||||
"""
|
||||
)
|
||||
QUICK_THINK_SYSTEM_PROMPT = """
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""RAG schemas."""
|
||||
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar, Literal, Optional, Union
|
||||
from typing import Any, ClassVar, List, Literal, Optional, Union
|
||||
|
||||
from chromadb.api.types import CollectionMetadata
|
||||
from llama_index.core.embeddings import BaseEmbedding
|
||||
|
|
@ -12,6 +12,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
|
|||
|
||||
from metagpt.config2 import config
|
||||
from metagpt.configs.embedding_config import EmbeddingType
|
||||
from metagpt.logs import logger
|
||||
from metagpt.rag.interface import RAGObject
|
||||
|
||||
|
||||
|
|
@ -44,7 +45,13 @@ class FAISSRetrieverConfig(IndexRetrieverConfig):
|
|||
@model_validator(mode="after")
|
||||
def check_dimensions(self):
|
||||
if self.dimensions == 0:
|
||||
self.dimensions = self._embedding_type_to_dimensions.get(config.embedding.api_type, 1536)
|
||||
self.dimensions = config.embedding.dimensions or self._embedding_type_to_dimensions.get(
|
||||
config.embedding.api_type, 1536
|
||||
)
|
||||
if not config.embedding.dimensions and config.embedding.api_type not in self._embedding_type_to_dimensions:
|
||||
logger.warning(
|
||||
f"You didn't set dimensions in config when using {config.embedding.api_type}, default to 1536"
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
|
|
@ -207,3 +214,51 @@ class ObjectNode(TextNode):
|
|||
)
|
||||
|
||||
return metadata.model_dump()
|
||||
|
||||
|
||||
class OmniParseType(str, Enum):
|
||||
"""OmniParseType"""
|
||||
|
||||
PDF = "PDF"
|
||||
DOCUMENT = "DOCUMENT"
|
||||
|
||||
|
||||
class ParseResultType(str, Enum):
|
||||
"""The result type for the parser."""
|
||||
|
||||
TXT = "text"
|
||||
MD = "markdown"
|
||||
JSON = "json"
|
||||
|
||||
|
||||
class OmniParseOptions(BaseModel):
|
||||
"""OmniParse Options config"""
|
||||
|
||||
result_type: ParseResultType = Field(default=ParseResultType.MD, description="OmniParse result_type")
|
||||
parse_type: OmniParseType = Field(default=OmniParseType.DOCUMENT, description="OmniParse parse_type")
|
||||
max_timeout: Optional[int] = Field(default=120, description="Maximum timeout for OmniParse service requests")
|
||||
num_workers: int = Field(
|
||||
default=5,
|
||||
gt=0,
|
||||
lt=10,
|
||||
description="Number of concurrent requests for multiple files",
|
||||
)
|
||||
|
||||
|
||||
class OminParseImage(BaseModel):
|
||||
image: str = Field(default="", description="image str bytes")
|
||||
image_name: str = Field(default="", description="image name")
|
||||
image_info: Optional[dict] = Field(default={}, description="image info")
|
||||
|
||||
|
||||
class OmniParsedResult(BaseModel):
|
||||
markdown: str = Field(default="", description="markdown text")
|
||||
text: str = Field(default="", description="plain text")
|
||||
images: Optional[List[OminParseImage]] = Field(default=[], description="images")
|
||||
metadata: Optional[dict] = Field(default={}, description="metadata")
|
||||
|
||||
@model_validator(mode="before")
|
||||
def set_markdown(cls, values):
|
||||
if not values.get("markdown"):
|
||||
values["markdown"] = values.get("text")
|
||||
return values
|
||||
|
|
|
|||
|
|
@ -383,10 +383,10 @@ class RoleZero(Role):
|
|||
tool_output = await tool_obj(**cmd["args"])
|
||||
if len(tool_output) <= 10:
|
||||
command_output += (
|
||||
f"\n[command]: {cmd['args']['cmd']} \n [command output] : {tool_output} (pay attention to this.)"
|
||||
f"\n[command]: {cmd['args']['cmd']} \n[command output] : {tool_output} (pay attention to this.)"
|
||||
)
|
||||
else:
|
||||
command_output += f"\n[command]: {cmd['args']['cmd']} \n [command output] : {tool_output}"
|
||||
command_output += f"\n[command]: {cmd['args']['cmd']} \n[command output] : {tool_output}"
|
||||
return command_output
|
||||
|
||||
def _get_plan_status(self) -> Tuple[str, str]:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ from metagpt.prompts.task_type import (
|
|||
FEATURE_ENGINEERING_PROMPT,
|
||||
IMAGE2WEBPAGE_PROMPT,
|
||||
MODEL_EVALUATE_PROMPT,
|
||||
MODEL_TRAIN_PROMPT, WEB_SCRAPING_PROMPT,
|
||||
MODEL_TRAIN_PROMPT,
|
||||
WEB_SCRAPING_PROMPT,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from metagpt.tools.libs import (
|
|||
deployer,
|
||||
git,
|
||||
)
|
||||
from metagpt.tools.libs.env import get_env, set_get_env_entry, default_get_env, get_env_description
|
||||
from metagpt.tools.libs.env import get_env, set_get_env_entry, default_get_env, get_env_description, get_env_default
|
||||
|
||||
_ = (
|
||||
data_preprocess,
|
||||
|
|
@ -32,6 +32,7 @@ _ = (
|
|||
deployer,
|
||||
git,
|
||||
get_env,
|
||||
get_env_default,
|
||||
get_env_description,
|
||||
set_get_env_entry,
|
||||
default_get_env,
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ class CodeReview:
|
|||
if pre:
|
||||
patch_file_content = pre.text
|
||||
else:
|
||||
async with aiofiles.open(patch_path) as f:
|
||||
async with aiofiles.open(patch_path, encoding="utf-8") as f:
|
||||
patch_file_content = await f.read()
|
||||
await EditorReporter().async_report(patch_path)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,17 @@
|
|||
import base64
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Union
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from metagpt.logs import logger
|
||||
from metagpt.tools.tool_registry import register_tool
|
||||
from metagpt.utils import read_docx
|
||||
from metagpt.utils.common import aread_bin, awrite_bin, run_coroutine_sync
|
||||
from metagpt.utils.repo_to_markdown import is_text_file
|
||||
from metagpt.utils.report import EditorReporter
|
||||
|
||||
|
||||
|
|
@ -39,12 +46,26 @@ class Editor(BaseModel):
|
|||
|
||||
def read(self, path: str) -> FileBlock:
|
||||
"""Read the whole content of a file. Using absolute paths as the argument for specifying the file location."""
|
||||
with open(path, "r") as f:
|
||||
self.resource.report(path, "path")
|
||||
lines = f.readlines()
|
||||
is_text, mime_type = run_coroutine_sync(is_text_file, path)
|
||||
if is_text:
|
||||
lines = self._read_text(path)
|
||||
elif mime_type == "application/pdf":
|
||||
lines = self._read_pdf(path)
|
||||
elif mime_type in {
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.ms-word.document.macroEnabled.12",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.template",
|
||||
"application/vnd.ms-word.template.macroEnabled.12",
|
||||
}:
|
||||
lines = self._read_docx(path)
|
||||
else:
|
||||
return FileBlock(file_path=str(path), block_content="")
|
||||
self.resource.report(str(path), "path")
|
||||
|
||||
lines_with_num = [f"{i + 1:03}|{line}" for i, line in enumerate(lines)]
|
||||
result = FileBlock(
|
||||
file_path=path,
|
||||
file_path=str(path),
|
||||
block_content="".join(lines_with_num),
|
||||
)
|
||||
return result
|
||||
|
|
@ -195,3 +216,63 @@ class Editor(BaseModel):
|
|||
lint_passed = result.returncode == 0
|
||||
lint_message = result.stdout
|
||||
return lint_passed, lint_message
|
||||
|
||||
@staticmethod
|
||||
def _read_text(path: Union[str, Path]) -> List[str]:
|
||||
with open(str(path), "r") as f:
|
||||
lines = f.readlines()
|
||||
return lines
|
||||
|
||||
@staticmethod
|
||||
def _read_pdf(path: Union[str, Path]) -> List[str]:
|
||||
result = run_coroutine_sync(Editor._omniparse_read_file, path)
|
||||
if result:
|
||||
return result
|
||||
|
||||
from llama_index.readers.file import PDFReader
|
||||
|
||||
reader = PDFReader()
|
||||
lines = reader.load_data(file=Path(path))
|
||||
return [i.text for i in lines]
|
||||
|
||||
@staticmethod
|
||||
def _read_docx(path: Union[str, Path]) -> List[str]:
|
||||
result = run_coroutine_sync(Editor._omniparse_read_file, path)
|
||||
if result:
|
||||
return result
|
||||
return read_docx(str(path))
|
||||
|
||||
@staticmethod
|
||||
async def _omniparse_read_file(path: Union[str, Path]) -> Optional[List[str]]:
|
||||
from metagpt.tools.libs import get_env_default
|
||||
from metagpt.utils.omniparse_client import OmniParseClient
|
||||
|
||||
base_url = await get_env_default(key="base_url", app_name="OmniParse", default_value="")
|
||||
if not base_url:
|
||||
return None
|
||||
api_key = await get_env_default(key="api_key", app_name="OmniParse", default_value="")
|
||||
v = await get_env_default(key="timeout", app_name="OmniParse", default_value="120")
|
||||
try:
|
||||
timeout = int(v) or 120
|
||||
except ValueError:
|
||||
timeout = 120
|
||||
|
||||
try:
|
||||
client = OmniParseClient(api_key=api_key, base_url=base_url, max_timeout=timeout)
|
||||
file_data = await aread_bin(filename=path)
|
||||
ret = await client.parse_document(file_input=file_data, bytes_filename=str(path))
|
||||
except (ValueError, Exception) as e:
|
||||
logger.exception(f"{path}: {e}")
|
||||
return None
|
||||
if not ret.images:
|
||||
return [ret.text] if ret.text else None
|
||||
|
||||
result = [ret.text]
|
||||
img_dir = Path(path).parent / (Path(path).name.replace(".", "_") + "_images")
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
for i in ret.images:
|
||||
byte_data = base64.b64decode(i.image)
|
||||
filename = img_dir / i.image_name
|
||||
await awrite_bin(filename=filename, data=byte_data)
|
||||
result.append(f"})")
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
@Desc: Implement `get_env`. RFC 216 2.4.2.4.2.
|
||||
"""
|
||||
import os
|
||||
from typing import Dict
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
class EnvKeyNotFoundError(Exception):
|
||||
|
|
@ -15,14 +15,26 @@ class EnvKeyNotFoundError(Exception):
|
|||
super().__init__(info)
|
||||
|
||||
|
||||
def to_app_key(key: str, app_name: str = None) -> str:
|
||||
return f"{app_name}-{key}" if app_name else key
|
||||
|
||||
|
||||
def split_app_key(app_key: str) -> (str, str):
|
||||
if "-" not in app_key:
|
||||
return "", app_key
|
||||
app_name, key = app_key.split("-", 1)
|
||||
return app_name, key
|
||||
|
||||
|
||||
async def default_get_env(key: str, app_name: str = None) -> str:
|
||||
if key in os.environ:
|
||||
return os.environ[key]
|
||||
app_key = to_app_key(key=key, app_name=app_name)
|
||||
if app_key in os.environ:
|
||||
return os.environ[app_key]
|
||||
|
||||
from metagpt.context import Context
|
||||
|
||||
context = Context()
|
||||
val = context.kwargs.get(key, None)
|
||||
val = context.kwargs.get(app_key, None)
|
||||
if val is not None:
|
||||
return val
|
||||
|
||||
|
|
@ -32,14 +44,16 @@ async def default_get_env(key: str, app_name: str = None) -> str:
|
|||
async def default_get_env_description() -> Dict[str, str]:
|
||||
result = {}
|
||||
for k in os.environ.keys():
|
||||
call = f'await get_env(key="{k}", app_name="")'
|
||||
app_name, key = split_app_key(k)
|
||||
call = f'await get_env(key="{key}", app_name="{app_name}")'
|
||||
result[call] = f"Return the value of environment variable `{k}`."
|
||||
|
||||
from metagpt.context import Context
|
||||
|
||||
context = Context()
|
||||
for k in context.kwargs.__dict__.keys():
|
||||
call = f'await get_env(key="{k}", app_name="")'
|
||||
app_name, key = split_app_key(k)
|
||||
call = f'await get_env(key="{key}", app_name="{app_name}")'
|
||||
result[call] = f"Get the value of environment variable `{k}`."
|
||||
return result
|
||||
|
||||
|
|
@ -84,6 +98,37 @@ async def get_env(key: str, app_name: str = None) -> str:
|
|||
return await default_get_env(key=key, app_name=app_name)
|
||||
|
||||
|
||||
async def get_env_default(key: str, app_name: str = None, default_value: str = None) -> Optional[str]:
|
||||
"""
|
||||
Retrieves the value for the specified environment variable key. If the key is not found,
|
||||
returns the default value.
|
||||
|
||||
Args:
|
||||
key (str): The name of the environment variable to retrieve.
|
||||
app_name (str, optional): The name of the application or component to associate with the environment variable.
|
||||
default_value (str, optional): The default value to return if the environment variable is not found.
|
||||
|
||||
Returns:
|
||||
str or None: The value of the environment variable if found, otherwise the default value.
|
||||
|
||||
Example:
|
||||
>>> from metagpt.tools.libs.env import get_env
|
||||
>>> api_key = await get_env_default(key="NOT_EXISTS_API_KEY", default_value="<API_KEY>")
|
||||
>>> print(api_key)
|
||||
<API_KEY>
|
||||
|
||||
>>> from metagpt.tools.libs.env import get_env
|
||||
>>> api_key = await get_env_default(key="NOT_EXISTS_API_KEY", app_name="GITHUB", default_value="<API_KEY>")
|
||||
>>> print(api_key)
|
||||
<API_KEY>
|
||||
|
||||
"""
|
||||
try:
|
||||
return await get_env(key=key, app_name=app_name)
|
||||
except EnvKeyNotFoundError:
|
||||
return default_value
|
||||
|
||||
|
||||
async def get_env_description() -> Dict[str, str]:
|
||||
global _get_env_description_entry
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import os
|
||||
from asyncio import Queue
|
||||
from asyncio.subprocess import PIPE, STDOUT
|
||||
from typing import Optional
|
||||
|
|
@ -28,7 +29,7 @@ class Terminal:
|
|||
async def _start_process(self):
|
||||
# Start a persistent shell process
|
||||
self.process = await asyncio.create_subprocess_exec(
|
||||
*self.shell_command, stdin=PIPE, stdout=PIPE, stderr=STDOUT, executable="bash"
|
||||
*self.shell_command, stdin=PIPE, stdout=PIPE, stderr=STDOUT, executable="bash", env=os.environ.copy()
|
||||
)
|
||||
await self._check_state()
|
||||
|
||||
|
|
@ -150,6 +151,7 @@ class Bash(Terminal):
|
|||
|
||||
def __init__(self):
|
||||
"""init"""
|
||||
os.environ["SWE_CMD_WORK_DIR"] = str(DEFAULT_WORKSPACE_ROOT)
|
||||
super().__init__()
|
||||
self.start_flag = False
|
||||
|
||||
|
|
|
|||
|
|
@ -16,4 +16,4 @@ source $REPO_ROOT_DIR/metagpt/tools/swe_agent_commands/defaults.sh
|
|||
source $REPO_ROOT_DIR/metagpt/tools/swe_agent_commands/search.sh
|
||||
source $REPO_ROOT_DIR/metagpt/tools/swe_agent_commands/edit_linting.sh
|
||||
|
||||
export SWE_CMD_WORK_DIR="$REPO_ROOT_DIR/workspace/swe_agent_workdir"
|
||||
echo "SWE_CMD_WORK_DIR: $SWE_CMD_WORK_DIR"
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import csv
|
||||
|
|
@ -870,7 +871,10 @@ async def get_mime_type(filename: str | Path, force_read: bool = False) -> str:
|
|||
}
|
||||
|
||||
try:
|
||||
stdout, _, _ = await shell_execute(f"file --mime-type {str(filename)}")
|
||||
stdout, stderr, _ = await shell_execute(f"file --mime-type {str(filename)}")
|
||||
if stderr:
|
||||
logger.debug(f"file:{filename}, error:{stderr}")
|
||||
return guess_mime_type
|
||||
ix = stdout.rfind(" ")
|
||||
mime_type = stdout[ix:].strip()
|
||||
if mime_type == "text/plain" and guess_mime_type in text_set:
|
||||
|
|
@ -1068,6 +1072,32 @@ def tool2name(cls, methods: List[str], entry) -> Dict[str, Any]:
|
|||
return mappings
|
||||
|
||||
|
||||
def run_coroutine_sync(coroutine, *args, **kwargs):
|
||||
"""
|
||||
Runs a coroutine function synchronously by encapsulating its invocation as a non-coroutine function call.
|
||||
|
||||
Args:
|
||||
coroutine: The coroutine function to be encapsulated.
|
||||
*args: Positional arguments to be passed to the coroutine.
|
||||
**kwargs: Keyword arguments to be passed to the coroutine.
|
||||
|
||||
Returns:
|
||||
The return value of the coroutine.
|
||||
"""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError: # No running event loop
|
||||
loop = None
|
||||
|
||||
if loop and loop.is_running():
|
||||
# The event loop is already running
|
||||
future = asyncio.run_coroutine_threadsafe(coroutine(*args, **kwargs), loop)
|
||||
return future.result()
|
||||
else:
|
||||
# The event loop is not running
|
||||
return asyncio.run(coroutine(*args, **kwargs))
|
||||
|
||||
|
||||
def log_time(method):
|
||||
"""A time-consuming decorator for printing execution duration."""
|
||||
|
||||
|
|
|
|||
238
metagpt/utils/omniparse_client.py
Normal file
238
metagpt/utils/omniparse_client.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
import mimetypes
|
||||
from pathlib import Path
|
||||
from typing import Union
|
||||
|
||||
import httpx
|
||||
|
||||
from metagpt.rag.schema import OmniParsedResult
|
||||
from metagpt.utils.common import aread_bin
|
||||
|
||||
|
||||
class OmniParseClient:
|
||||
"""
|
||||
OmniParse Server Client
|
||||
This client interacts with the OmniParse server to parse different types of media, documents.
|
||||
|
||||
OmniParse API Documentation: https://docs.cognitivelab.in/api
|
||||
|
||||
Attributes:
|
||||
ALLOWED_DOCUMENT_EXTENSIONS (set): A set of supported document file extensions.
|
||||
ALLOWED_AUDIO_EXTENSIONS (set): A set of supported audio file extensions.
|
||||
ALLOWED_VIDEO_EXTENSIONS (set): A set of supported video file extensions.
|
||||
"""
|
||||
|
||||
ALLOWED_DOCUMENT_EXTENSIONS = {".pdf", ".ppt", ".pptx", ".doc", ".docx"}
|
||||
ALLOWED_AUDIO_EXTENSIONS = {".mp3", ".wav", ".aac"}
|
||||
ALLOWED_VIDEO_EXTENSIONS = {".mp4", ".mkv", ".avi", ".mov"}
|
||||
|
||||
def __init__(self, api_key: str = None, base_url: str = "http://localhost:8000", max_timeout: int = 120):
|
||||
"""
|
||||
Args:
|
||||
api_key: Default None, can be used for authentication later.
|
||||
base_url: Base URL for the API.
|
||||
max_timeout: Maximum request timeout in seconds.
|
||||
"""
|
||||
self.api_key = api_key
|
||||
self.base_url = base_url
|
||||
self.max_timeout = max_timeout
|
||||
|
||||
self.parse_media_endpoint = "/parse_media"
|
||||
self.parse_website_endpoint = "/parse_website"
|
||||
self.parse_document_endpoint = "/parse_document"
|
||||
|
||||
async def _request_parse(
|
||||
self,
|
||||
endpoint: str,
|
||||
method: str = "POST",
|
||||
files: dict = None,
|
||||
params: dict = None,
|
||||
data: dict = None,
|
||||
json: dict = None,
|
||||
headers: dict = None,
|
||||
**kwargs,
|
||||
) -> dict:
|
||||
"""
|
||||
Request OmniParse API to parse a document.
|
||||
|
||||
Args:
|
||||
endpoint (str): API endpoint.
|
||||
method (str, optional): HTTP method to use. Default is "POST".
|
||||
files (dict, optional): Files to include in the request.
|
||||
params (dict, optional): Query string parameters.
|
||||
data (dict, optional): Form data to include in the request body.
|
||||
json (dict, optional): JSON data to include in the request body.
|
||||
headers (dict, optional): HTTP headers to include in the request.
|
||||
**kwargs: Additional keyword arguments for httpx.AsyncClient.request()
|
||||
|
||||
Returns:
|
||||
dict: JSON response data.
|
||||
"""
|
||||
url = f"{self.base_url}{endpoint}"
|
||||
method = method.upper()
|
||||
headers = headers or {}
|
||||
_headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
|
||||
headers.update(**_headers)
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.request(
|
||||
url=url,
|
||||
method=method,
|
||||
files=files,
|
||||
params=params,
|
||||
json=json,
|
||||
data=data,
|
||||
headers=headers,
|
||||
timeout=self.max_timeout,
|
||||
**kwargs,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
async def parse_document(self, file_input: Union[str, bytes, Path], bytes_filename: str = None) -> OmniParsedResult:
|
||||
"""
|
||||
Parse document-type data (supports ".pdf", ".ppt", ".pptx", ".doc", ".docx").
|
||||
|
||||
Args:
|
||||
file_input: File path or file byte data.
|
||||
bytes_filename: Filename for byte data, useful for determining MIME type for the HTTP request.
|
||||
|
||||
Raises:
|
||||
ValueError: If the file extension is not allowed.
|
||||
|
||||
Returns:
|
||||
OmniParsedResult: The result of the document parsing.
|
||||
"""
|
||||
self.verify_file_ext(file_input, self.ALLOWED_DOCUMENT_EXTENSIONS, bytes_filename)
|
||||
file_info = await self.get_file_info(file_input, bytes_filename)
|
||||
resp = await self._request_parse(self.parse_document_endpoint, files={"file": file_info})
|
||||
data = OmniParsedResult(**resp)
|
||||
return data
|
||||
|
||||
async def parse_pdf(self, file_input: Union[str, bytes, Path]) -> OmniParsedResult:
|
||||
"""
|
||||
Parse pdf document.
|
||||
|
||||
Args:
|
||||
file_input: File path or file byte data.
|
||||
|
||||
Raises:
|
||||
ValueError: If the file extension is not allowed.
|
||||
|
||||
Returns:
|
||||
OmniParsedResult: The result of the pdf parsing.
|
||||
"""
|
||||
self.verify_file_ext(file_input, {".pdf"})
|
||||
# parse_pdf supports parsing by accepting only the byte data of the file.
|
||||
file_info = await self.get_file_info(file_input, only_bytes=True)
|
||||
endpoint = f"{self.parse_document_endpoint}/pdf"
|
||||
resp = await self._request_parse(endpoint=endpoint, files={"file": file_info})
|
||||
data = OmniParsedResult(**resp)
|
||||
return data
|
||||
|
||||
async def parse_video(self, file_input: Union[str, bytes, Path], bytes_filename: str = None) -> dict:
|
||||
"""
|
||||
Parse video-type data (supports ".mp4", ".mkv", ".avi", ".mov").
|
||||
|
||||
Args:
|
||||
file_input: File path or file byte data.
|
||||
bytes_filename: Filename for byte data, useful for determining MIME type for the HTTP request.
|
||||
|
||||
Raises:
|
||||
ValueError: If the file extension is not allowed.
|
||||
|
||||
Returns:
|
||||
dict: JSON response data.
|
||||
"""
|
||||
self.verify_file_ext(file_input, self.ALLOWED_VIDEO_EXTENSIONS, bytes_filename)
|
||||
file_info = await self.get_file_info(file_input, bytes_filename)
|
||||
return await self._request_parse(f"{self.parse_media_endpoint}/video", files={"file": file_info})
|
||||
|
||||
async def parse_audio(self, file_input: Union[str, bytes, Path], bytes_filename: str = None) -> dict:
|
||||
"""
|
||||
Parse audio-type data (supports ".mp3", ".wav", ".aac").
|
||||
|
||||
Args:
|
||||
file_input: File path or file byte data.
|
||||
bytes_filename: Filename for byte data, useful for determining MIME type for the HTTP request.
|
||||
|
||||
Raises:
|
||||
ValueError: If the file extension is not allowed.
|
||||
|
||||
Returns:
|
||||
dict: JSON response data.
|
||||
"""
|
||||
self.verify_file_ext(file_input, self.ALLOWED_AUDIO_EXTENSIONS, bytes_filename)
|
||||
file_info = await self.get_file_info(file_input, bytes_filename)
|
||||
return await self._request_parse(f"{self.parse_media_endpoint}/audio", files={"file": file_info})
|
||||
|
||||
@staticmethod
|
||||
def verify_file_ext(file_input: Union[str, bytes, Path], allowed_file_extensions: set, bytes_filename: str = None):
|
||||
"""
|
||||
Verify the file extension.
|
||||
|
||||
Args:
|
||||
file_input: File path or file byte data.
|
||||
allowed_file_extensions: Set of allowed file extensions.
|
||||
bytes_filename: Filename to use for verification when `file_input` is byte data.
|
||||
|
||||
Raises:
|
||||
ValueError: If the file extension is not allowed.
|
||||
|
||||
Returns:
|
||||
"""
|
||||
verify_file_path = None
|
||||
if isinstance(file_input, (str, Path)):
|
||||
verify_file_path = str(file_input)
|
||||
elif isinstance(file_input, bytes) and bytes_filename:
|
||||
verify_file_path = bytes_filename
|
||||
|
||||
if not verify_file_path:
|
||||
# Do not verify if only byte data is provided
|
||||
return
|
||||
|
||||
file_ext = Path(verify_file_path).suffix
|
||||
if file_ext not in allowed_file_extensions:
|
||||
raise ValueError(f"Not allowed {file_ext} File extension must be one of {allowed_file_extensions}")
|
||||
|
||||
@staticmethod
|
||||
async def get_file_info(
|
||||
file_input: Union[str, bytes, Path],
|
||||
bytes_filename: str = None,
|
||||
only_bytes: bool = False,
|
||||
) -> Union[bytes, tuple]:
|
||||
"""
|
||||
Get file information.
|
||||
|
||||
Args:
|
||||
file_input: File path or file byte data.
|
||||
bytes_filename: Filename to use when uploading byte data, useful for determining MIME type.
|
||||
only_bytes: Whether to return only byte data. Default is False, which returns a tuple.
|
||||
|
||||
Raises:
|
||||
ValueError: If bytes_filename is not provided when file_input is bytes or if file_input is not a valid type.
|
||||
|
||||
Notes:
|
||||
Since `parse_document`,`parse_video`, `parse_audio` supports parsing various file types,
|
||||
the MIME type of the file must be specified when uploading.
|
||||
|
||||
Returns: [bytes, tuple]
|
||||
Returns bytes if only_bytes is True, otherwise returns a tuple (filename, file_bytes, mime_type).
|
||||
"""
|
||||
if isinstance(file_input, (str, Path)):
|
||||
filename = Path(file_input).name
|
||||
file_bytes = await aread_bin(file_input)
|
||||
|
||||
if only_bytes:
|
||||
return file_bytes
|
||||
|
||||
mime_type = mimetypes.guess_type(file_input)[0]
|
||||
return filename, file_bytes, mime_type
|
||||
elif isinstance(file_input, bytes):
|
||||
if only_bytes:
|
||||
return file_input
|
||||
if not bytes_filename:
|
||||
raise ValueError("bytes_filename must be set when passing bytes")
|
||||
|
||||
mime_type = mimetypes.guess_type(bytes_filename)[0]
|
||||
return bytes_filename, file_input, mime_type
|
||||
else:
|
||||
raise ValueError("file_input must be a string (file path) or bytes.")
|
||||
|
|
@ -7,7 +7,7 @@ from __future__ import annotations
|
|||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
from typing import Tuple, Union
|
||||
|
||||
from gitignore_parser import parse_gitignore
|
||||
|
||||
|
|
@ -82,7 +82,7 @@ async def _write_files(repo_path, gitignore_rules=None) -> str:
|
|||
|
||||
|
||||
async def _write_file(filename: Path, repo_path: Path) -> str:
|
||||
is_text, mime_type = await _is_text_file(filename)
|
||||
is_text, mime_type = await is_text_file(filename)
|
||||
if not is_text:
|
||||
logger.info(f"Ignore content: {filename}")
|
||||
return ""
|
||||
|
|
@ -100,7 +100,17 @@ async def _write_file(filename: Path, repo_path: Path) -> str:
|
|||
return ""
|
||||
|
||||
|
||||
async def _is_text_file(filename: Path) -> Tuple[bool, str]:
|
||||
async def is_text_file(filename: Union[str, Path]) -> Tuple[bool, str]:
|
||||
"""
|
||||
Determines if the specified file is a text file based on its MIME type.
|
||||
|
||||
Args:
|
||||
filename (Union[str, Path]): The path to the file.
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: A tuple where the first element indicates if the file is a text file
|
||||
(True for text file, False otherwise), and the second element is the MIME type of the file.
|
||||
"""
|
||||
pass_set = {
|
||||
"application/json",
|
||||
"application/vnd.chipnuts.karaoke-mmd",
|
||||
|
|
@ -129,7 +139,7 @@ async def _is_text_file(filename: Path) -> Tuple[bool, str]:
|
|||
"image/vnd.microsoft.icon",
|
||||
"video/mp4",
|
||||
}
|
||||
mime_type = await get_mime_type(filename, force_read=True)
|
||||
mime_type = await get_mime_type(Path(filename), force_read=True)
|
||||
v = "text/" in mime_type or mime_type in pass_set
|
||||
if v:
|
||||
return True, mime_type
|
||||
|
|
|
|||
BIN
tests/data/movie/trailer.mp4
Normal file
BIN
tests/data/movie/trailer.mp4
Normal file
Binary file not shown.
BIN
tests/data/requirements/2.pdf
Normal file
BIN
tests/data/requirements/2.pdf
Normal file
Binary file not shown.
164
tests/data/ui/1b.png.html
Normal file
164
tests/data/ui/1b.png.html
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>法务小超人</title>
|
||||
<style>
|
||||
body {
|
||||
margin: 0;
|
||||
font-family: Arial, sans-serif;
|
||||
}
|
||||
.container {
|
||||
text-align: center;
|
||||
background-color: #f8f8f8;
|
||||
padding: 10px;
|
||||
}
|
||||
.header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 10px 20px;
|
||||
background-color: #fff;
|
||||
border-bottom: 1px solid #e6e6e6;
|
||||
}
|
||||
.header img {
|
||||
height: 24px;
|
||||
}
|
||||
.header .menu-icons {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
}
|
||||
.header .menu-icons img {
|
||||
height: 24px;
|
||||
}
|
||||
.search-section {
|
||||
background-image: url('background-image.jpg'); /* Replace with actual background image */
|
||||
background-size: cover;
|
||||
color: white;
|
||||
padding: 40px 20px;
|
||||
}
|
||||
.search-section h1 {
|
||||
margin: 0;
|
||||
font-size: 24px;
|
||||
}
|
||||
.search-input {
|
||||
margin: 20px 0;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
}
|
||||
.search-input input {
|
||||
width: 300px;
|
||||
padding: 10px;
|
||||
border-radius: 5px 0 0 5px;
|
||||
border: none;
|
||||
}
|
||||
.search-input button {
|
||||
padding: 10px 20px;
|
||||
border-radius: 0 5px 5px 0;
|
||||
border: none;
|
||||
background-color: #007BFF;
|
||||
color: white;
|
||||
cursor: pointer;
|
||||
}
|
||||
.search-result-count {
|
||||
margin: 10px 0;
|
||||
}
|
||||
.qa-section {
|
||||
background-color: #fff;
|
||||
padding: 20px;
|
||||
margin-top: -20px;
|
||||
border-top-left-radius: 20px;
|
||||
border-top-right-radius: 20px;
|
||||
box-shadow: 0 -2px 5px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
.qa-item {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 15px 0;
|
||||
border-bottom: 1px solid #e6e6e6;
|
||||
}
|
||||
.qa-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
.qa-item a {
|
||||
text-decoration: none;
|
||||
color: #333;
|
||||
}
|
||||
.qa-item img {
|
||||
height: 20px;
|
||||
}
|
||||
.footer {
|
||||
display: flex;
|
||||
justify-content: space-around;
|
||||
background-color: #fff;
|
||||
border-top: 1px solid #e6e6e6;
|
||||
padding: 10px 0;
|
||||
position: fixed;
|
||||
width: 100%;
|
||||
bottom: 0;
|
||||
}
|
||||
.footer img {
|
||||
height: 24px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<div class="logo">
|
||||
<img src="logo.png" alt="法务小超人"> <!-- Replace with actual logo -->
|
||||
</div>
|
||||
<div class="menu-icons">
|
||||
<img src="menu-icon.png" alt="Menu"> <!-- Replace with actual menu icon -->
|
||||
<img src="more-icon.png" alt="More"> <!-- Replace with actual more icon -->
|
||||
</div>
|
||||
</div>
|
||||
<div class="search-section">
|
||||
<h1>法律意见查询</h1>
|
||||
<div class="search-input">
|
||||
<input type="text" placeholder="输入国家名查询法律意见">
|
||||
<button>
|
||||
<img src="search-icon.png" alt="Search"> <!-- Replace with actual search icon -->
|
||||
</button>
|
||||
</div>
|
||||
<div class="search-result-count">
|
||||
已收录法律意见8394篇
|
||||
</div>
|
||||
</div>
|
||||
<div class="qa-section">
|
||||
<h2>法务 Q&A</h2>
|
||||
<div class="qa-item">
|
||||
<a href="#">国际法务接口人</a>
|
||||
<img src="arrow.png" alt="Arrow"> <!-- Replace with actual arrow icon -->
|
||||
</div>
|
||||
<div class="qa-item">
|
||||
<a href="#">国内法务接口人</a>
|
||||
<img src="arrow.png" alt="Arrow"> <!-- Replace with actual arrow icon -->
|
||||
</div>
|
||||
<div class="qa-item">
|
||||
<a href="#">国际法律协议合同办理指引</a>
|
||||
<img src="arrow.png" alt="Arrow"> <!-- Replace with actual arrow icon -->
|
||||
</div>
|
||||
<div class="qa-item">
|
||||
<a href="#">国内法律协议合同办理指引</a>
|
||||
<img src="arrow.png" alt="Arrow"> <!-- Replace with actual arrow icon -->
|
||||
</div>
|
||||
</div>
|
||||
<div class="footer">
|
||||
<div class="footer-item">
|
||||
<img src="home-icon.png" alt="首页"> <!-- Replace with actual home icon -->
|
||||
<div>首页</div>
|
||||
</div>
|
||||
<div class="footer-item">
|
||||
<img src="template-icon.png" alt="模板"> <!-- Replace with actual template icon -->
|
||||
<div>模板</div>
|
||||
</div>
|
||||
<div class="footer-item">
|
||||
<img src="my-icon.png" alt="我的"> <!-- Replace with actual my icon -->
|
||||
<div>我的</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
from metagpt.roles.di.data_analyst import DataAnalyst
|
||||
|
||||
HOUSE_PRICE_TRAIN_PATH = '/data/house-prices-advanced-regression-techniques/split_train.csv'
|
||||
HOUSE_PRICE_EVAL_PATH = '/data/house-prices-advanced-regression-techniques/split_eval.csv'
|
||||
HOUSE_PRICE_TRAIN_PATH = "/data/house-prices-advanced-regression-techniques/split_train.csv"
|
||||
HOUSE_PRICE_EVAL_PATH = "/data/house-prices-advanced-regression-techniques/split_eval.csv"
|
||||
HOUSE_PRICE_REQ = f"""
|
||||
This is a house price dataset, your goal is to predict the sale price of a property based on its features. The target column is SalePrice. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report RMSE between the logarithm of the predicted value and the logarithm of the observed sales price on the eval data. Train data path: '{HOUSE_PRICE_TRAIN_PATH}', eval data path: '{HOUSE_PRICE_EVAL_PATH}'.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ def test_file():
|
|||
|
||||
EXPECTED_SEARCHED_BLOCK = FileBlock(
|
||||
file_path=str(TEST_FILE_PATH),
|
||||
block_content='# this is line one\ndef test_function_for_fm():\n "some docstring"\n a = 1\n b = 2\n',
|
||||
block_content='001|# this is line one\n002|def test_function_for_fm():\n003| "some docstring"\n004| a = 1\n005| b = 2\n',
|
||||
block_start_line=1,
|
||||
block_end_line=5,
|
||||
symbol="def test_function_for_fm",
|
||||
|
|
@ -50,6 +50,7 @@ def test_function_for_fm():
|
|||
""".strip()
|
||||
|
||||
|
||||
@pytest.mark.skip
|
||||
def test_replace_content(test_file):
|
||||
Editor().write_content(
|
||||
file_path=str(TEST_FILE_PATH),
|
||||
|
|
@ -89,6 +90,7 @@ def test_function_for_fm():
|
|||
""".strip()
|
||||
|
||||
|
||||
@pytest.mark.skip
|
||||
def test_insert_content(test_file):
|
||||
Editor().write_content(
|
||||
file_path=str(TEST_FILE_PATH),
|
||||
|
|
@ -101,6 +103,7 @@ def test_insert_content(test_file):
|
|||
assert new_content == EXPECTED_CONTENT_AFTER_INSERT
|
||||
|
||||
|
||||
@pytest.mark.skip
|
||||
def test_new_content_wrong_indentation(test_file):
|
||||
msg = Editor().write_content(
|
||||
file_path=str(TEST_FILE_PATH),
|
||||
|
|
@ -111,6 +114,7 @@ def test_new_content_wrong_indentation(test_file):
|
|||
assert "failed" in msg
|
||||
|
||||
|
||||
@pytest.mark.skip
|
||||
def test_new_content_format_issue(test_file):
|
||||
msg = Editor().write_content(
|
||||
file_path=str(TEST_FILE_PATH),
|
||||
|
|
@ -119,3 +123,32 @@ def test_new_content_format_issue(test_file):
|
|||
new_block_content=" # This is the new line to be inserted, at line 3 ", # trailing spaces are format issue only, and should not throw an error
|
||||
)
|
||||
assert "failed" not in msg
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"filename",
|
||||
[
|
||||
TEST_DATA_PATH / "requirements/1.txt",
|
||||
TEST_DATA_PATH / "requirements/1.json",
|
||||
TEST_DATA_PATH / "requirements/1.constraint.md",
|
||||
TEST_DATA_PATH / "requirements/pic/1.png",
|
||||
TEST_DATA_PATH / "docx_for_test.docx",
|
||||
TEST_DATA_PATH / "requirements/2.pdf",
|
||||
TEST_DATA_PATH / "audio/hello.mp3",
|
||||
TEST_DATA_PATH / "code/python/1.py",
|
||||
TEST_DATA_PATH / "code/js/1.js",
|
||||
TEST_DATA_PATH / "ui/1b.png.html",
|
||||
TEST_DATA_PATH / "movie/trailer.mp4",
|
||||
],
|
||||
)
|
||||
def test_read_files(filename):
|
||||
editor = Editor()
|
||||
file_block = editor.read(filename)
|
||||
assert file_block
|
||||
assert file_block.file_path
|
||||
if filename.suffix not in [".png", ".mp3", ".mp4"]:
|
||||
assert file_block.block_content
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-s"])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue