apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-04-28 02:23:52 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: merge geekan/dev

Browse source
This commit is contained in:
莘权 马 2024-01-27 13:38:14 +08:00
commit aa0909525e
62 changed files with 1733 additions and 154 deletions
Download patch file Download diff file Expand all files Collapse all files

20
metagpt/utils/dependency_file.py
View file

@ -9,6 +9,7 @@
from __future__ import annotations
import json
import re
from pathlib import Path
from typing import Set
@ -36,7 +37,9 @@ class DependencyFile:
"""Load dependencies from the file asynchronously."""
if not self._filename.exists():
return
self._dependencies = json.loads(await aread(self._filename))
json_data = await aread(self._filename)
json_data = re.sub(r"\\+", "/", json_data) # Compatible with windows path
self._dependencies = json.loads(json_data)
@handle_exception
async def save(self):
@ -60,17 +63,20 @@ class DependencyFile:
key = Path(filename).relative_to(root)
except ValueError:
key = filename
skey = re.sub(r"\\+", "/", str(key)) # Compatible with windows path
if dependencies:
relative_paths = []
for i in dependencies:
try:
relative_paths.append(str(Path(i).relative_to(root)))
s = str(Path(i).relative_to(root))
except ValueError:
relative_paths.append(str(i))
self._dependencies[str(key)] = relative_paths
elif str(key) in self._dependencies:
del self._dependencies[str(key)]
s = str(i)
s = re.sub(r"\\+", "/", s) # Compatible with windows path
relative_paths.append(s)
self._dependencies[skey] = relative_paths
elif skey in self._dependencies:
del self._dependencies[skey]
if persist:
await self.save()

19
metagpt/utils/file_repository.py
View file

@ -101,21 +101,28 @@ class FileRepository:
path_name = self.workdir / filename
if not path_name.exists():
return None
if not path_name.is_file():
return None
doc.content = await aread(path_name)
return doc
async def get_all(self) -> List[Document]:
async def get_all(self, filter_ignored=True) -> List[Document]:
"""Get the content of all files in the repository.
:return: List of Document instances representing files.
"""
docs = []
for root, dirs, files in os.walk(str(self.workdir)):
for file in files:
file_path = Path(root) / file
relative_path = file_path.relative_to(self.workdir)
doc = await self.get(relative_path)
if filter_ignored:
for f in self.all_files:
doc = await self.get(f)
docs.append(doc)
else:
for root, dirs, files in os.walk(str(self.workdir)):
for file in files:
file_path = Path(root) / file
relative_path = file_path.relative_to(self.workdir)
doc = await self.get(relative_path)
docs.append(doc)
return docs
@property

6
metagpt/utils/project_repo.py
View file

@ -13,6 +13,8 @@ from pathlib import Path
from metagpt.const import (
CLASS_VIEW_FILE_REPO,
CODE_PLAN_AND_CHANGE_FILE_REPO,
CODE_PLAN_AND_CHANGE_PDF_FILE_REPO,
CODE_SUMMARIES_FILE_REPO,
CODE_SUMMARIES_PDF_FILE_REPO,
COMPETITIVE_ANALYSIS_FILE_REPO,
@ -43,6 +45,7 @@ class DocFileRepositories(FileRepository):
code_summary: FileRepository
graph_repo: FileRepository
class_view: FileRepository
code_plan_and_change: FileRepository
def __init__(self, git_repo):
super().__init__(git_repo=git_repo, relative_path=DOCS_FILE_REPO)
@ -53,6 +56,7 @@ class DocFileRepositories(FileRepository):
self.code_summary = git_repo.new_file_repository(relative_path=CODE_SUMMARIES_FILE_REPO)
self.graph_repo = git_repo.new_file_repository(relative_path=GRAPH_REPO_FILE_REPO)
self.class_view = git_repo.new_file_repository(relative_path=CLASS_VIEW_FILE_REPO)
self.code_plan_and_change = git_repo.new_file_repository(relative_path=CODE_PLAN_AND_CHANGE_FILE_REPO)
class ResourceFileRepositories(FileRepository):
@ -64,6 +68,7 @@ class ResourceFileRepositories(FileRepository):
api_spec_and_task: FileRepository
code_summary: FileRepository
sd_output: FileRepository
code_plan_and_change: FileRepository
def __init__(self, git_repo):
super().__init__(git_repo=git_repo, relative_path=RESOURCES_FILE_REPO)
@ -76,6 +81,7 @@ class ResourceFileRepositories(FileRepository):
self.api_spec_and_task = git_repo.new_file_repository(relative_path=TASK_PDF_FILE_REPO)
self.code_summary = git_repo.new_file_repository(relative_path=CODE_SUMMARIES_PDF_FILE_REPO)
self.sd_output = git_repo.new_file_repository(relative_path=SD_OUTPUT_FILE_REPO)
self.code_plan_and_change = git_repo.new_file_repository(relative_path=CODE_PLAN_AND_CHANGE_PDF_FILE_REPO)
class ProjectRepo(FileRepository):

18
metagpt/utils/token_counter.py
View file

@ -4,10 +4,11 @@
@Time : 2023/5/18 00:40
@Author : alexanderwu
@File : token_counter.py
ref1: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
ref2: https://github.com/Significant-Gravitas/Auto-GPT/blob/master/autogpt/llm/token_counter.py
ref3: https://github.com/hwchase17/langchain/blob/master/langchain/chat_models/openai.py
ref4: https://ai.google.dev/models/gemini
ref1: https://openai.com/pricing
ref2: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
ref3: https://github.com/Significant-Gravitas/Auto-GPT/blob/master/autogpt/llm/token_counter.py
ref4: https://github.com/hwchase17/langchain/blob/master/langchain/chat_models/openai.py
ref5: https://ai.google.dev/models/gemini
"""
import tiktoken
@ -25,7 +26,10 @@ TOKEN_COSTS = {
"gpt-4-32k": {"prompt": 0.06, "completion": 0.12},
"gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12},
"gpt-4-0613": {"prompt": 0.06, "completion": 0.12},
"gpt-4-turbo-preview": {"prompt": 0.01, "completion": 0.03},
"gpt-4-0125-preview": {"prompt": 0.01, "completion": 0.03},
"gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03},
"gpt-4-1106-vision-preview": {"prompt": 0.01, "completion": 0.03},
"text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0},
"glm-3-turbo": {"prompt": 0.0, "completion": 0.0007}, # 128k version, prompt + completion tokens=0.005￥/k-tokens
"glm-4": {"prompt": 0.0, "completion": 0.014}, # 128k version, prompt + completion tokens=0.1￥/k-tokens
@ -47,7 +51,10 @@ TOKEN_MAX = {
"gpt-4-32k": 32768,
"gpt-4-32k-0314": 32768,
"gpt-4-0613": 8192,
"gpt-4-turbo-preview": 128000,
"gpt-4-0125-preview": 128000,
"gpt-4-1106-preview": 128000,
"gpt-4-1106-vision-preview": 128000,
"text-embedding-ada-002": 8192,
"chatglm_turbo": 32768,
"gemini-pro": 32768,
@ -72,7 +79,10 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"):
"gpt-4-32k-0314",
"gpt-4-0613",
"gpt-4-32k-0613",
"gpt-4-turbo-preview",
"gpt-4-0125-preview",
"gpt-4-1106-preview",
"gpt-4-1106-vision-preview",
}:
tokens_per_message = 3 # # every reply is primed with <|start|>assistant<|message|>
tokens_per_name = 1