apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-05-09 07:42:38 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'dev' into code_intepreter

Browse source
This commit is contained in:
yzlin 2024-01-31 00:08:09 +08:00
commit 2fcb2a1cfe
282 changed files with 6993 additions and 3210 deletions
Download patch file Download diff file Expand all files Collapse all files

14
metagpt/utils/common.py
View file

@ -29,7 +29,7 @@ from typing import Any, List, Tuple, Union
import aiofiles
import loguru
from pydantic_core import to_jsonable_python
from tenacity import RetryCallState, _utils
from tenacity import RetryCallState, RetryError, _utils
from metagpt.const import MESSAGE_ROUTE_TO_ALL
from metagpt.logs import logger
@ -407,12 +407,12 @@ def any_to_str_set(val) -> set:
return res
def is_subscribed(message: "Message", tags: set):
def is_send_to(message: "Message", addresses: set):
"""Return whether it's consumer"""
if MESSAGE_ROUTE_TO_ALL in message.send_to:
return True
for i in tags:
for i in addresses:
if i in message.send_to:
return True
return False
@ -531,7 +531,7 @@ def role_raise_decorator(func):
self.rc.memory.delete(self.latest_observed_msg)
# raise again to make it captured outside
raise Exception(format_trackback_info(limit=None))
except Exception:
except Exception as e:
if self.latest_observed_msg:
logger.warning(
"There is a exception in role's execution, in order to resume, "
@ -540,6 +540,12 @@ def role_raise_decorator(func):
# remove role newest observed msg to make it observed again
self.rc.memory.delete(self.latest_observed_msg)
# raise again to make it captured outside
if isinstance(e, RetryError):
last_error = e.last_attempt._exception
name = any_to_str(last_error)
if re.match(r"^openai\.", name) or re.match(r"^httpx\.", name):
raise last_error
raise Exception(format_trackback_info(limit=None))
return wrapper

17
metagpt/utils/cost_manager.py
View file

@ -80,3 +80,20 @@ class CostManager(BaseModel):
def get_costs(self) -> Costs:
"""Get all costs"""
return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget)
class TokenCostManager(CostManager):
"""open llm model is self-host, it's free and without cost"""
def update_cost(self, prompt_tokens, completion_tokens, model):
"""
Update the total cost, prompt tokens, and completion tokens.
Args:
prompt_tokens (int): The number of tokens used in the prompt.
completion_tokens (int): The number of tokens used in the completion.
model (str): The model used for the API call.
"""
self.total_prompt_tokens += prompt_tokens
self.total_completion_tokens += completion_tokens
logger.info(f"prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}")

20
metagpt/utils/dependency_file.py
View file

@ -9,6 +9,7 @@
from __future__ import annotations
import json
import re
from pathlib import Path
from typing import Set
@ -36,7 +37,9 @@ class DependencyFile:
"""Load dependencies from the file asynchronously."""
if not self._filename.exists():
return
self._dependencies = json.loads(await aread(self._filename))
json_data = await aread(self._filename)
json_data = re.sub(r"\\+", "/", json_data) # Compatible with windows path
self._dependencies = json.loads(json_data)
@handle_exception
async def save(self):
@ -60,17 +63,20 @@ class DependencyFile:
key = Path(filename).relative_to(root)
except ValueError:
key = filename
skey = re.sub(r"\\+", "/", str(key)) # Compatible with windows path
if dependencies:
relative_paths = []
for i in dependencies:
try:
relative_paths.append(str(Path(i).relative_to(root)))
s = str(Path(i).relative_to(root))
except ValueError:
relative_paths.append(str(i))
self._dependencies[str(key)] = relative_paths
elif str(key) in self._dependencies:
del self._dependencies[str(key)]
s = str(i)
s = re.sub(r"\\+", "/", s) # Compatible with windows path
relative_paths.append(s)
self._dependencies[skey] = relative_paths
elif skey in self._dependencies:
del self._dependencies[skey]
if persist:
await self.save()

16
metagpt/utils/embedding.py Normal file
View file

@ -0,0 +1,16 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2024/1/4 20:58
@Author : alexanderwu
@File : embedding.py
"""
from langchain_community.embeddings import OpenAIEmbeddings
from metagpt.config2 import config
def get_embedding():
llm = config.get_openai_llm()
embedding = OpenAIEmbeddings(openai_api_key=llm.api_key, openai_api_base=llm.base_url)
return embedding

110
metagpt/utils/file_repository.py
View file

@ -16,7 +16,6 @@ from typing import Dict, List, Set
import aiofiles
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.schema import Document
from metagpt.utils.common import aread
@ -46,7 +45,7 @@ class FileRepository:
# Initializing
self.workdir.mkdir(parents=True, exist_ok=True)
async def save(self, filename: Path | str, content, dependencies: List[str] = None):
async def save(self, filename: Path | str, content, dependencies: List[str] = None) -> Document:
"""Save content to a file and update its dependencies.
:param filename: The filename or path within the repository.
@ -55,6 +54,7 @@ class FileRepository:
"""
pathname = self.workdir / filename
pathname.parent.mkdir(parents=True, exist_ok=True)
content = content if content else "" # avoid `argument must be str, not None` to make it continue
async with aiofiles.open(str(pathname), mode="w") as writer:
await writer.write(content)
logger.info(f"save to: {str(pathname)}")
@ -64,6 +64,8 @@ class FileRepository:
await dependency_file.update(pathname, set(dependencies))
logger.info(f"update dependency: {str(pathname)}:{dependencies}")
return Document(root_path=str(self._relative_path), filename=str(filename), content=content)
async def get_dependency(self, filename: Path | str) -> Set[str]:
"""Get the dependencies of a file.
@ -99,21 +101,28 @@ class FileRepository:
path_name = self.workdir / filename
if not path_name.exists():
return None
if not path_name.is_file():
return None
doc.content = await aread(path_name)
return doc
async def get_all(self) -> List[Document]:
async def get_all(self, filter_ignored=True) -> List[Document]:
"""Get the content of all files in the repository.
:return: List of Document instances representing files.
"""
docs = []
for root, dirs, files in os.walk(str(self.workdir)):
for file in files:
file_path = Path(root) / file
relative_path = file_path.relative_to(self.workdir)
doc = await self.get(relative_path)
if filter_ignored:
for f in self.all_files:
doc = await self.get(f)
docs.append(doc)
else:
for root, dirs, files in os.walk(str(self.workdir)):
for file in files:
file_path = Path(root) / file
relative_path = file_path.relative_to(self.workdir)
doc = await self.get(relative_path)
docs.append(doc)
return docs
@property
@ -182,10 +191,20 @@ class FileRepository:
"""
current_time = datetime.now().strftime("%Y%m%d%H%M%S")
return current_time
# guid_suffix = str(uuid.uuid4())[:8]
# return f"{current_time}x{guid_suffix}"
async def save_doc(self, doc: Document, with_suffix: str = None, dependencies: List[str] = None):
async def save_doc(self, doc: Document, dependencies: List[str] = None):
"""Save content to a file and update its dependencies.
:param doc: The Document instance to be saved.
:type doc: Document
:param dependencies: A list of dependencies for the saved file.
:type dependencies: List[str], optional
"""
await self.save(filename=doc.filename, content=doc.content, dependencies=dependencies)
logger.debug(f"File Saved: {str(doc.filename)}")
async def save_pdf(self, doc: Document, with_suffix: str = ".md", dependencies: List[str] = None):
"""Save a Document instance as a PDF file.
This method converts the content of the Document instance to Markdown,
@ -203,70 +222,6 @@ class FileRepository:
await self.save(filename=str(filename), content=json_to_markdown(m), dependencies=dependencies)
logger.debug(f"File Saved: {str(filename)}")
@staticmethod
async def get_file(filename: Path | str, relative_path: Path | str = ".") -> Document | None:
"""Retrieve a specific file from the file repository.
:param filename: The name or path of the file to retrieve.
:type filename: Path or str
:param relative_path: The relative path within the file repository.
:type relative_path: Path or str, optional
:return: The document representing the file, or None if not found.
:rtype: Document or None
"""
file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path)
return await file_repo.get(filename=filename)
@staticmethod
async def get_all_files(relative_path: Path | str = ".") -> List[Document]:
"""Retrieve all files from the file repository.
:param relative_path: The relative path within the file repository.
:type relative_path: Path or str, optional
:return: A list of documents representing all files in the repository.
:rtype: List[Document]
"""
file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path)
return await file_repo.get_all()
@staticmethod
async def save_file(filename: Path | str, content, dependencies: List[str] = None, relative_path: Path | str = "."):
"""Save a file to the file repository.
:param filename: The name or path of the file to save.
:type filename: Path or str
:param content: The content of the file.
:param dependencies: A list of dependencies for the file.
:type dependencies: List[str], optional
:param relative_path: The relative path within the file repository.
:type relative_path: Path or str, optional
"""
file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path)
return await file_repo.save(filename=filename, content=content, dependencies=dependencies)
@staticmethod
async def save_as(
doc: Document, with_suffix: str = None, dependencies: List[str] = None, relative_path: Path | str = "."
):
"""Save a Document instance with optional modifications.
This static method creates a new FileRepository, saves the Document instance
with optional modifications (such as a suffix), and logs the saved file.
:param doc: The Document instance to be saved.
:type doc: Document
:param with_suffix: An optional suffix to append to the saved file's name.
:type with_suffix: str, optional
:param dependencies: A list of dependencies for the saved file.
:type dependencies: List[str], optional
:param relative_path: The relative path within the file repository.
:type relative_path: Path or str, optional
:return: A boolean indicating whether the save operation was successful.
:rtype: bool
"""
file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path)
return await file_repo.save_doc(doc=doc, with_suffix=with_suffix, dependencies=dependencies)
async def delete(self, filename: Path | str):
"""Delete a file from the file repository.
@ -283,8 +238,3 @@ class FileRepository:
dependency_file = await self._git_repo.get_dependency()
await dependency_file.update(filename=pathname, dependencies=None)
logger.info(f"remove dependency key: {str(pathname)}")
@staticmethod
async def delete_file(filename: Path | str, relative_path: Path | str = "."):
file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path)
await file_repo.delete(filename=filename)

17
metagpt/utils/git_repository.py
View file

@ -107,7 +107,10 @@ class GitRepository:
def delete_repository(self):
"""Delete the entire repository directory."""
if self.is_valid:
shutil.rmtree(self._repository.working_dir)
try:
shutil.rmtree(self._repository.working_dir)
except Exception as e:
logger.exception(f"Failed delete git repo:{self.workdir}, error:{e}")
@property
def changed_files(self) -> Dict[str, str]:
@ -198,11 +201,21 @@ class GitRepository:
new_path = self.workdir.parent / new_dir_name
if new_path.exists():
logger.info(f"Delete directory {str(new_path)}")
shutil.rmtree(new_path)
try:
shutil.rmtree(new_path)
except Exception as e:
logger.warning(f"rm {str(new_path)} error: {e}")
if new_path.exists(): # Recheck for windows os
logger.warning(f"Failed to delete directory {str(new_path)}")
return
try:
shutil.move(src=str(self.workdir), dst=str(new_path))
except Exception as e:
logger.warning(f"Move {str(self.workdir)} to {str(new_path)} error: {e}")
finally:
if not new_path.exists(): # Recheck for windows os
logger.warning(f"Failed to move {str(self.workdir)} to {str(new_path)}")
return
logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}")
self._repository = Repo(new_path)
self._gitignore_rules = parse_gitignore(full_path=str(new_path / ".gitignore"))

107
metagpt/utils/human_interaction.py Normal file
View file

@ -0,0 +1,107 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc : human interaction to get required type text
import json
from typing import Any, Tuple, Type
from pydantic import BaseModel
from metagpt.logs import logger
from metagpt.utils.common import import_class
class HumanInteraction(object):
stop_list = ("q", "quit", "exit")
def multilines_input(self, prompt: str = "Enter: ") -> str:
logger.warning("Enter your content, use Ctrl-D or Ctrl-Z ( windows ) to save it.")
logger.info(f"{prompt}\n")
lines = []
while True:
try:
line = input()
lines.append(line)
except EOFError:
break
return "".join(lines)
def check_input_type(self, input_str: str, req_type: Type) -> Tuple[bool, Any]:
check_ret = True
if req_type == str:
# required_type = str, just return True
return check_ret, input_str
try:
input_str = input_str.strip()
data = json.loads(input_str)
except Exception:
return False, None
actionnode_class = import_class("ActionNode", "metagpt.actions.action_node") # avoid circular import
tmp_key = "tmp"
tmp_cls = actionnode_class.create_model_class(class_name=tmp_key.upper(), mapping={tmp_key: (req_type, ...)})
try:
_ = tmp_cls(**{tmp_key: data})
except Exception:
check_ret = False
return check_ret, data
def input_until_valid(self, prompt: str, req_type: Type) -> Any:
# check the input with req_type until it's ok
while True:
input_content = self.multilines_input(prompt)
check_ret, structure_content = self.check_input_type(input_content, req_type)
if check_ret:
break
else:
logger.error(f"Input content can't meet required_type: {req_type}, please Re-Enter.")
return structure_content
def input_num_until_valid(self, num_max: int) -> int:
while True:
input_num = input("Enter the num of the interaction key: ")
input_num = input_num.strip()
if input_num in self.stop_list:
return input_num
try:
input_num = int(input_num)
if 0 <= input_num < num_max:
return input_num
except Exception:
pass
def interact_with_instruct_content(
self, instruct_content: BaseModel, mapping: dict = dict(), interact_type: str = "review"
) -> dict[str, Any]:
assert interact_type in ["review", "revise"]
assert instruct_content
instruct_content_dict = instruct_content.model_dump()
num_fields_map = dict(zip(range(0, len(instruct_content_dict)), instruct_content_dict.keys()))
logger.info(
f"\n{interact_type.upper()} interaction\n"
f"Interaction data: {num_fields_map}\n"
f"Enter the num to interact with corresponding field or `q`/`quit`/`exit` to stop interaction.\n"
f"Enter the field content until it meet field required type.\n"
)
interact_contents = {}
while True:
input_num = self.input_num_until_valid(len(instruct_content_dict))
if input_num in self.stop_list:
logger.warning("Stop human interaction")
break
field = num_fields_map.get(input_num)
logger.info(f"You choose to interact with field: {field}, and do a `{interact_type}` operation.")
if interact_type == "review":
prompt = "Enter your review comment: "
req_type = str
else:
prompt = "Enter your revise content: "
req_type = mapping.get(field)[0] # revise need input content match the required_type
field_content = self.input_until_valid(prompt=prompt, req_type=req_type)
interact_contents[field] = field_content
return interact_contents

10
metagpt/utils/make_sk_kernel.py
View file

@ -13,20 +13,20 @@ from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion impo
OpenAIChatCompletion,
)
from metagpt.config import CONFIG
from metagpt.config2 import config
def make_sk_kernel():
kernel = sk.Kernel()
if CONFIG.OPENAI_API_TYPE == "azure":
if llm := config.get_azure_llm():
kernel.add_chat_service(
"chat_completion",
AzureChatCompletion(CONFIG.DEPLOYMENT_NAME, CONFIG.OPENAI_BASE_URL, CONFIG.OPENAI_API_KEY),
AzureChatCompletion(llm.model, llm.base_url, llm.api_key),
)
else:
elif llm := config.get_openai_llm():
kernel.add_chat_service(
"chat_completion",
OpenAIChatCompletion(CONFIG.OPENAI_API_MODEL, CONFIG.OPENAI_API_KEY),
OpenAIChatCompletion(llm.model, llm.api_key),
)
return kernel

16
metagpt/utils/mermaid.py
View file

@ -4,7 +4,6 @@
@Time : 2023/7/4 10:53
@Author : alexanderwu alitrack
@File : mermaid.py
@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
"""
import asyncio
import os
@ -12,12 +11,12 @@ from pathlib import Path
import aiofiles
from metagpt.config import CONFIG
from metagpt.config2 import config
from metagpt.logs import logger
from metagpt.utils.common import check_cmd_exists
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
async def mermaid_to_file(engine, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
"""suffix: png/svg/pdf
:param mermaid_code: mermaid code
@ -35,9 +34,8 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
await f.write(mermaid_code)
# tmp.write_text(mermaid_code, encoding="utf-8")
engine = CONFIG.mermaid_engine.lower()
if engine == "nodejs":
if check_cmd_exists(CONFIG.mmdc) != 0:
if check_cmd_exists(config.mmdc) != 0:
logger.warning(
"RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc,"
"or consider changing MERMAID_ENGINE to `playwright`, `pyppeteer`, or `ink`."
@ -49,11 +47,11 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
# Call the `mmdc` command to convert the Mermaid code to a PNG
logger.info(f"Generating {output_file}..")
if CONFIG.puppeteer_config:
if config.puppeteer_config:
commands = [
CONFIG.mmdc,
config.mmdc,
"-p",
CONFIG.puppeteer_config,
config.puppeteer_config,
"-i",
str(tmp),
"-o",
@ -64,7 +62,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
str(height),
]
else:
commands = [CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]
commands = [config.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]
process = await asyncio.create_subprocess_shell(
" ".join(commands), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)

6
metagpt/utils/mmdc_pyppeteer.py
View file

@ -10,7 +10,7 @@ from urllib.parse import urljoin
from pyppeteer import launch
from metagpt.config import CONFIG
from metagpt.config2 import config
from metagpt.logs import logger
@ -30,10 +30,10 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
suffixes = ["png", "svg", "pdf"]
__dirname = os.path.dirname(os.path.abspath(__file__))
if CONFIG.pyppeteer_executable_path:
if config.pyppeteer_executable_path:
browser = await launch(
headless=True,
executablePath=CONFIG.pyppeteer_executable_path,
executablePath=config.pyppeteer_executable_path,
args=["--disable-extensions", "--no-sandbox"],
)
else:

139
metagpt/utils/project_repo.py Normal file
View file

@ -0,0 +1,139 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2024/1/8
@Author : mashenquan
@File : project_repo.py
@Desc : Wrapper for GitRepository and FileRepository of project.
Implementation of Chapter 4.6 of https://deepwisdom.feishu.cn/wiki/CUK4wImd7id9WlkQBNscIe9cnqh
"""
from __future__ import annotations
from pathlib import Path
from metagpt.const import (
CLASS_VIEW_FILE_REPO,
CODE_PLAN_AND_CHANGE_FILE_REPO,
CODE_PLAN_AND_CHANGE_PDF_FILE_REPO,
CODE_SUMMARIES_FILE_REPO,
CODE_SUMMARIES_PDF_FILE_REPO,
COMPETITIVE_ANALYSIS_FILE_REPO,
DATA_API_DESIGN_FILE_REPO,
DOCS_FILE_REPO,
GRAPH_REPO_FILE_REPO,
PRD_PDF_FILE_REPO,
PRDS_FILE_REPO,
REQUIREMENT_FILENAME,
RESOURCES_FILE_REPO,
SD_OUTPUT_FILE_REPO,
SEQ_FLOW_FILE_REPO,
SYSTEM_DESIGN_FILE_REPO,
SYSTEM_DESIGN_PDF_FILE_REPO,
TASK_FILE_REPO,
TASK_PDF_FILE_REPO,
TEST_CODES_FILE_REPO,
TEST_OUTPUTS_FILE_REPO,
)
from metagpt.utils.file_repository import FileRepository
from metagpt.utils.git_repository import GitRepository
class DocFileRepositories(FileRepository):
prd: FileRepository
system_design: FileRepository
task: FileRepository
code_summary: FileRepository
graph_repo: FileRepository
class_view: FileRepository
code_plan_and_change: FileRepository
def __init__(self, git_repo):
super().__init__(git_repo=git_repo, relative_path=DOCS_FILE_REPO)
self.prd = git_repo.new_file_repository(relative_path=PRDS_FILE_REPO)
self.system_design = git_repo.new_file_repository(relative_path=SYSTEM_DESIGN_FILE_REPO)
self.task = git_repo.new_file_repository(relative_path=TASK_FILE_REPO)
self.code_summary = git_repo.new_file_repository(relative_path=CODE_SUMMARIES_FILE_REPO)
self.graph_repo = git_repo.new_file_repository(relative_path=GRAPH_REPO_FILE_REPO)
self.class_view = git_repo.new_file_repository(relative_path=CLASS_VIEW_FILE_REPO)
self.code_plan_and_change = git_repo.new_file_repository(relative_path=CODE_PLAN_AND_CHANGE_FILE_REPO)
class ResourceFileRepositories(FileRepository):
competitive_analysis: FileRepository
data_api_design: FileRepository
seq_flow: FileRepository
system_design: FileRepository
prd: FileRepository
api_spec_and_task: FileRepository
code_summary: FileRepository
sd_output: FileRepository
code_plan_and_change: FileRepository
def __init__(self, git_repo):
super().__init__(git_repo=git_repo, relative_path=RESOURCES_FILE_REPO)
self.competitive_analysis = git_repo.new_file_repository(relative_path=COMPETITIVE_ANALYSIS_FILE_REPO)
self.data_api_design = git_repo.new_file_repository(relative_path=DATA_API_DESIGN_FILE_REPO)
self.seq_flow = git_repo.new_file_repository(relative_path=SEQ_FLOW_FILE_REPO)
self.system_design = git_repo.new_file_repository(relative_path=SYSTEM_DESIGN_PDF_FILE_REPO)
self.prd = git_repo.new_file_repository(relative_path=PRD_PDF_FILE_REPO)
self.api_spec_and_task = git_repo.new_file_repository(relative_path=TASK_PDF_FILE_REPO)
self.code_summary = git_repo.new_file_repository(relative_path=CODE_SUMMARIES_PDF_FILE_REPO)
self.sd_output = git_repo.new_file_repository(relative_path=SD_OUTPUT_FILE_REPO)
self.code_plan_and_change = git_repo.new_file_repository(relative_path=CODE_PLAN_AND_CHANGE_PDF_FILE_REPO)
class ProjectRepo(FileRepository):
def __init__(self, root: str | Path | GitRepository):
if isinstance(root, str) or isinstance(root, Path):
git_repo_ = GitRepository(local_path=Path(root))
elif isinstance(root, GitRepository):
git_repo_ = root
else:
raise ValueError("Invalid root")
super().__init__(git_repo=git_repo_, relative_path=Path("."))
self._git_repo = git_repo_
self.docs = DocFileRepositories(self._git_repo)
self.resources = ResourceFileRepositories(self._git_repo)
self.tests = self._git_repo.new_file_repository(relative_path=TEST_CODES_FILE_REPO)
self.test_outputs = self._git_repo.new_file_repository(relative_path=TEST_OUTPUTS_FILE_REPO)
self._srcs_path = None
@property
async def requirement(self):
return await self.docs.get(filename=REQUIREMENT_FILENAME)
@property
def git_repo(self) -> GitRepository:
return self._git_repo
@property
def workdir(self) -> Path:
return Path(self.git_repo.workdir)
@property
def srcs(self) -> FileRepository:
if not self._srcs_path:
raise ValueError("Call with_srcs first.")
return self._git_repo.new_file_repository(self._srcs_path)
def code_files_exists(self) -> bool:
git_workdir = self.git_repo.workdir
src_workdir = git_workdir / git_workdir.name
if not src_workdir.exists():
return False
code_files = self.with_src_path(path=git_workdir / git_workdir.name).srcs.all_files
if not code_files:
return False
def with_src_path(self, path: str | Path) -> ProjectRepo:
try:
self._srcs_path = Path(path).relative_to(self.workdir)
except ValueError:
self._srcs_path = Path(path)
return self
@property
def src_relative_path(self) -> Path | None:
return self._srcs_path

30
metagpt/utils/redis.py
View file

@ -12,26 +12,25 @@ from datetime import timedelta
import aioredis # https://aioredis.readthedocs.io/en/latest/getting-started/
from metagpt.config import CONFIG
from metagpt.configs.redis_config import RedisConfig
from metagpt.logs import logger
class Redis:
def __init__(self):
def __init__(self, config: RedisConfig = None):
self.config = config
self._client = None
async def _connect(self, force=False):
if self._client and not force:
return True
if not self.is_configured:
return False
try:
self._client = await aioredis.from_url(
f"redis://{CONFIG.REDIS_HOST}:{CONFIG.REDIS_PORT}",
username=CONFIG.REDIS_USER,
password=CONFIG.REDIS_PASSWORD,
db=CONFIG.REDIS_DB,
self.config.to_url(),
username=self.config.username,
password=self.config.password,
db=self.config.db,
)
return True
except Exception as e:
@ -62,18 +61,3 @@ class Redis:
return
await self._client.close()
self._client = None
@property
def is_valid(self) -> bool:
return self._client is not None
@property
def is_configured(self) -> bool:
return bool(
CONFIG.REDIS_HOST
and CONFIG.REDIS_HOST != "YOUR_REDIS_HOST"
and CONFIG.REDIS_PORT
and CONFIG.REDIS_PORT != "YOUR_REDIS_PORT"
and CONFIG.REDIS_DB is not None
and CONFIG.REDIS_PASSWORD is not None
)

30
metagpt/utils/repair_llm_raw_output.py
View file

@ -9,7 +9,7 @@ from typing import Callable, Union
import regex as re
from tenacity import RetryCallState, retry, stop_after_attempt, wait_fixed
from metagpt.config import CONFIG
from metagpt.config2 import config
from metagpt.logs import logger
from metagpt.utils.custom_decoder import CustomDecoder
@ -120,6 +120,15 @@ def repair_json_format(output: str) -> str:
elif output.startswith("{") and output.endswith("]"):
output = output[:-1] + "}"
# remove `#` in output json str, usually appeared in `glm-4`
arr = output.split("\n")
new_arr = []
for line in arr:
idx = line.find("#")
if idx >= 0:
line = line[:idx]
new_arr.append(line)
output = "\n".join(new_arr)
return output
@ -152,7 +161,7 @@ def repair_llm_raw_output(output: str, req_keys: list[str], repair_type: RepairT
target: { xxx }
output: { xxx }]
"""
if not CONFIG.repair_llm_output:
if not config.repair_llm_output:
return output
# do the repairation usually for non-openai models
@ -168,15 +177,17 @@ def repair_invalid_json(output: str, error: str) -> str:
example 1. json.decoder.JSONDecodeError: Expecting ',' delimiter: line 154 column 1 (char 2765)
example 2. xxx.JSONDecodeError: Expecting property name enclosed in double quotes: line 14 column 1 (char 266)
"""
pattern = r"line ([0-9]+)"
pattern = r"line ([0-9]+) column ([0-9]+)"
matches = re.findall(pattern, error, re.DOTALL)
if len(matches) > 0:
line_no = int(matches[0]) - 1
line_no = int(matches[0][0]) - 1
col_no = int(matches[0][1]) - 1
# due to CustomDecoder can handle `"": ''` or `'': ""`, so convert `"""` -> `"`, `'''` -> `'`
output = output.replace('"""', '"').replace("'''", '"')
arr = output.split("\n")
rline = arr[line_no] # raw line
line = arr[line_no].strip()
# different general problems
if line.endswith("],"):
@ -187,9 +198,12 @@ def repair_invalid_json(output: str, error: str) -> str:
new_line = line.replace("}", "")
elif line.endswith("},") and output.endswith("},"):
new_line = line[:-1]
elif '",' not in line and "," not in line:
elif (rline[col_no] in ["'", '"']) and (line.startswith('"') or line.startswith("'")) and "," not in line:
# problem, `"""` or `'''` without `,`
new_line = f",{line}"
elif '",' not in line and "," not in line and '"' not in line:
new_line = f'{line}",'
elif "," not in line:
elif not line.endswith(","):
# problem, miss char `,` at the end.
new_line = f"{line},"
elif "," in line and len(line) == 1:
@ -231,7 +245,7 @@ def run_after_exp_and_passon_next_retry(logger: "loguru.Logger") -> Callable[["R
func_param_output = retry_state.kwargs.get("output", "")
exp_str = str(retry_state.outcome.exception())
fix_str = "try to fix it, " if CONFIG.repair_llm_output else ""
fix_str = "try to fix it, " if config.repair_llm_output else ""
logger.warning(
f"parse json from content inside [CONTENT][/CONTENT] failed at retry "
f"{retry_state.attempt_number}, {fix_str}exp: {exp_str}"
@ -244,7 +258,7 @@ def run_after_exp_and_passon_next_retry(logger: "loguru.Logger") -> Callable[["R
@retry(
stop=stop_after_attempt(3 if CONFIG.repair_llm_output else 0),
stop=stop_after_attempt(3 if config.repair_llm_output else 0),
wait=wait_fixed(1),
after=run_after_exp_and_passon_next_retry(logger),
)

32
metagpt/utils/s3.py
View file

@ -8,7 +8,7 @@ from typing import Optional
import aioboto3
import aiofiles
from metagpt.config import CONFIG
from metagpt.config2 import S3Config
from metagpt.const import BASE64_FORMAT
from metagpt.logs import logger
@ -16,13 +16,14 @@ from metagpt.logs import logger
class S3:
"""A class for interacting with Amazon S3 storage."""
def __init__(self):
def __init__(self, config: S3Config):
self.session = aioboto3.Session()
self.config = config
self.auth_config = {
"service_name": "s3",
"aws_access_key_id": CONFIG.S3_ACCESS_KEY,
"aws_secret_access_key": CONFIG.S3_SECRET_KEY,
"endpoint_url": CONFIG.S3_ENDPOINT_URL,
"aws_access_key_id": config.access_key,
"aws_secret_access_key": config.secret_key,
"endpoint_url": config.endpoint,
}
async def upload_file(
@ -139,8 +140,8 @@ class S3:
data = base64.b64decode(data) if format == BASE64_FORMAT else data.encode(encoding="utf-8")
await file.write(data)
bucket = CONFIG.S3_BUCKET
object_pathname = CONFIG.S3_BUCKET or "system"
bucket = self.config.bucket
object_pathname = self.config.bucket or "system"
object_pathname += f"/{object_name}"
object_pathname = os.path.normpath(object_pathname)
await self.upload_file(bucket=bucket, local_path=str(pathname), object_name=object_pathname)
@ -151,20 +152,3 @@ class S3:
logger.exception(f"{e}, stack:{traceback.format_exc()}")
pathname.unlink(missing_ok=True)
return None
@property
def is_valid(self):
return self.is_configured
@property
def is_configured(self) -> bool:
return bool(
CONFIG.S3_ACCESS_KEY
and CONFIG.S3_ACCESS_KEY != "YOUR_S3_ACCESS_KEY"
and CONFIG.S3_SECRET_KEY
and CONFIG.S3_SECRET_KEY != "YOUR_S3_SECRET_KEY"
and CONFIG.S3_ENDPOINT_URL
and CONFIG.S3_ENDPOINT_URL != "YOUR_S3_ENDPOINT_URL"
and CONFIG.S3_BUCKET
and CONFIG.S3_BUCKET != "YOUR_S3_BUCKET"
)

21
metagpt/utils/token_counter.py
View file

@ -4,10 +4,11 @@
@Time : 2023/5/18 00:40
@Author : alexanderwu
@File : token_counter.py
ref1: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
ref2: https://github.com/Significant-Gravitas/Auto-GPT/blob/master/autogpt/llm/token_counter.py
ref3: https://github.com/hwchase17/langchain/blob/master/langchain/chat_models/openai.py
ref4: https://ai.google.dev/models/gemini
ref1: https://openai.com/pricing
ref2: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
ref3: https://github.com/Significant-Gravitas/Auto-GPT/blob/master/autogpt/llm/token_counter.py
ref4: https://github.com/hwchase17/langchain/blob/master/langchain/chat_models/openai.py
ref5: https://ai.google.dev/models/gemini
"""
import tiktoken
@ -25,9 +26,13 @@ TOKEN_COSTS = {
"gpt-4-32k": {"prompt": 0.06, "completion": 0.12},
"gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12},
"gpt-4-0613": {"prompt": 0.06, "completion": 0.12},
"gpt-4-turbo-preview": {"prompt": 0.01, "completion": 0.03},
"gpt-4-0125-preview": {"prompt": 0.01, "completion": 0.03},
"gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03},
"gpt-4-1106-vision-preview": {"prompt": 0.01, "completion": 0.03},
"text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0},
"chatglm_turbo": {"prompt": 0.0, "completion": 0.00069}, # 32k version, prompt + completion tokens=0.005￥/k-tokens
"glm-3-turbo": {"prompt": 0.0, "completion": 0.0007}, # 128k version, prompt + completion tokens=0.005￥/k-tokens
"glm-4": {"prompt": 0.0, "completion": 0.014}, # 128k version, prompt + completion tokens=0.1￥/k-tokens
"gemini-pro": {"prompt": 0.00025, "completion": 0.0005},
}
@ -46,7 +51,10 @@ TOKEN_MAX = {
"gpt-4-32k": 32768,
"gpt-4-32k-0314": 32768,
"gpt-4-0613": 8192,
"gpt-4-turbo-preview": 128000,
"gpt-4-0125-preview": 128000,
"gpt-4-1106-preview": 128000,
"gpt-4-1106-vision-preview": 128000,
"text-embedding-ada-002": 8192,
"chatglm_turbo": 32768,
"gemini-pro": 32768,
@ -71,7 +79,10 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"):
"gpt-4-32k-0314",
"gpt-4-0613",
"gpt-4-32k-0613",
"gpt-4-turbo-preview",
"gpt-4-0125-preview",
"gpt-4-1106-preview",
"gpt-4-1106-vision-preview",
}:
tokens_per_message = 3 # # every reply is primed with <|start|>assistant<|message|>
tokens_per_name = 1

48
metagpt/utils/yaml_model.py Normal file
View file

@ -0,0 +1,48 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2024/1/4 10:18
@Author : alexanderwu
@File : YamlModel.py
"""
from pathlib import Path
from typing import Dict, Optional
import yaml
from pydantic import BaseModel, model_validator
class YamlModel(BaseModel):
"""Base class for yaml model"""
extra_fields: Optional[Dict[str, str]] = None
@classmethod
def read_yaml(cls, file_path: Path, encoding: str = "utf-8") -> Dict:
"""Read yaml file and return a dict"""
if not file_path.exists():
return {}
with open(file_path, "r", encoding=encoding) as file:
return yaml.safe_load(file)
@classmethod
def from_yaml_file(cls, file_path: Path) -> "YamlModel":
"""Read yaml file and return a YamlModel instance"""
return cls(**cls.read_yaml(file_path))
def to_yaml_file(self, file_path: Path, encoding: str = "utf-8") -> None:
"""Dump YamlModel instance to yaml file"""
with open(file_path, "w", encoding=encoding) as file:
yaml.dump(self.model_dump(), file)
class YamlModelWithoutDefault(YamlModel):
"""YamlModel without default values"""
@model_validator(mode="before")
@classmethod
def check_not_default_config(cls, values):
"""Check if there is any default config in config.yaml"""
if any(["YOUR" in v for v in values]):
raise ValueError("Please set your config in config.yaml")
return values