apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-06-11 15:15:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: ProjectRepo + srcs

Browse source 
feat: ProjectRepo + git_repo

feat: Replace FileRepository with ProjectRepo
This commit is contained in:
莘权 马 2024-01-11 11:25:00 +08:00
parent 60969b6aed
commit bf6fc25f57
15 changed files with 178 additions and 238 deletions
Download patch file Download diff file Expand all files Collapse all files

14
metagpt/actions/action.py
View file

@ -21,7 +21,7 @@ from metagpt.schema import (
SerializationMixin,
TestingContext,
)
from metagpt.utils.file_repository import FileRepository
from metagpt.utils.project_repo import ProjectRepo
class Action(SerializationMixin, ContextMixin, BaseModel):
@ -34,16 +34,8 @@ class Action(SerializationMixin, ContextMixin, BaseModel):
node: ActionNode = Field(default=None, exclude=True)
@property
def git_repo(self):
return self.context.git_repo
@property
def file_repo(self):
return FileRepository(self.context.git_repo)
@property
def src_workspace(self):
return self.context.src_workspace
def project_repo(self):
return ProjectRepo(git_repo=self.context.git_repo)
@property
def prompt_schema(self):

13
metagpt/actions/debug_error.py
View file

@ -13,7 +13,6 @@ import re
from pydantic import Field
from metagpt.actions.action import Action
from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO
from metagpt.logs import logger
from metagpt.schema import RunCodeContext, RunCodeResult
from metagpt.utils.common import CodeParser
@ -50,9 +49,7 @@ class DebugError(Action):
i_context: RunCodeContext = Field(default_factory=RunCodeContext)
async def run(self, *args, **kwargs) -> str:
output_doc = await self.file_repo.get_file(
filename=self.i_context.output_filename, relative_path=TEST_OUTPUTS_FILE_REPO
)
output_doc = await self.project_repo.test_outputs.get(filename=self.i_context.output_filename)
if not output_doc:
return ""
output_detail = RunCodeResult.loads(output_doc.content)
@ -62,14 +59,12 @@ class DebugError(Action):
return ""
logger.info(f"Debug and rewrite {self.i_context.test_filename}")
code_doc = await self.file_repo.get_file(
filename=self.i_context.code_filename, relative_path=self.context.src_workspace
code_doc = await self.project_repo.with_src_path(self.context.src_workspace).srcs.get(
filename=self.i_context.code_filename
)
if not code_doc:
return ""
test_doc = await self.file_repo.get_file(
filename=self.i_context.test_filename, relative_path=TEST_CODES_FILE_REPO
)
test_doc = await self.project_repo.tests.get(filename=self.i_context.test_filename)
if not test_doc:
return ""
prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr)

45
metagpt/actions/design_api.py
View file

@ -15,13 +15,7 @@ from typing import Optional
from metagpt.actions import Action, ActionOutput
from metagpt.actions.design_api_an import DESIGN_API_NODE
from metagpt.const import (
DATA_API_DESIGN_FILE_REPO,
PRDS_FILE_REPO,
SEQ_FLOW_FILE_REPO,
SYSTEM_DESIGN_FILE_REPO,
SYSTEM_DESIGN_PDF_FILE_REPO,
)
from metagpt.const import DATA_API_DESIGN_FILE_REPO, SEQ_FLOW_FILE_REPO
from metagpt.logs import logger
from metagpt.schema import Document, Documents, Message
from metagpt.utils.mermaid import mermaid_to_file
@ -46,27 +40,21 @@ class WriteDesign(Action):
async def run(self, with_messages: Message, schema: str = None):
# Use `git status` to identify which PRD documents have been modified in the `docs/prds` directory.
prds_file_repo = self.git_repo.new_file_repository(PRDS_FILE_REPO)
changed_prds = prds_file_repo.changed_files
changed_prds = self.project_repo.docs.prd.changed_files
# Use `git status` to identify which design documents in the `docs/system_designs` directory have undergone
# changes.
system_design_file_repo = self.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO)
changed_system_designs = system_design_file_repo.changed_files
changed_system_designs = self.project_repo.docs.system_design.changed_files
# For those PRDs and design documents that have undergone changes, regenerate the design content.
changed_files = Documents()
for filename in changed_prds.keys():
doc = await self._update_system_design(
filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo
)
doc = await self._update_system_design(filename=filename)
changed_files.docs[filename] = doc
for filename in changed_system_designs.keys():
if filename in changed_files.docs:
continue
doc = await self._update_system_design(
filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo
)
doc = await self._update_system_design(filename=filename)
changed_files.docs[filename] = doc
if not changed_files.docs:
logger.info("Nothing has changed.")
@ -84,24 +72,22 @@ class WriteDesign(Action):
system_design_doc.content = node.instruct_content.model_dump_json()
return system_design_doc
async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document:
prd = await prds_file_repo.get(filename)
old_system_design_doc = await system_design_file_repo.get(filename)
async def _update_system_design(self, filename) -> Document:
prd = await self.project_repo.docs.prd.get(filename)
old_system_design_doc = await self.project_repo.docs.system_design.get(filename)
if not old_system_design_doc:
system_design = await self._new_system_design(context=prd.content)
doc = Document(
root_path=SYSTEM_DESIGN_FILE_REPO,
doc = await self.project_repo.docs.system_design.save(
filename=filename,
content=system_design.instruct_content.model_dump_json(),
dependencies={prd.root_relative_path},
)
else:
doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc)
await system_design_file_repo.save(
filename=filename, content=doc.content, dependencies={prd.root_relative_path}
)
await self.project_repo.docs.system_design.save_doc(doc=doc, dependencies={prd.root_relative_path})
await self._save_data_api_design(doc)
await self._save_seq_flow(doc)
await self._save_pdf(doc)
await self.project_repo.resources.system_design.save_pdf(doc=doc)
return doc
async def _save_data_api_design(self, design_doc):
@ -109,7 +95,7 @@ class WriteDesign(Action):
data_api_design = m.get("Data structures and interfaces")
if not data_api_design:
return
pathname = self.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("")
pathname = self.project_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("")
await self._save_mermaid_file(data_api_design, pathname)
logger.info(f"Save class view to {str(pathname)}")
@ -118,13 +104,10 @@ class WriteDesign(Action):
seq_flow = m.get("Program call flow")
if not seq_flow:
return
pathname = self.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("")
pathname = self.project_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("")
await self._save_mermaid_file(seq_flow, pathname)
logger.info(f"Saving sequence flow to {str(pathname)}")
async def _save_pdf(self, design_doc):
await self.file_repo.save_as(doc=design_doc, with_suffix=".md", relative_path=SYSTEM_DESIGN_PDF_FILE_REPO)
async def _save_mermaid_file(self, data: str, pathname: Path):
pathname.parent.mkdir(parents=True, exist_ok=True)
await mermaid_to_file(self.config.mermaid_engine, data, pathname)

8
metagpt/actions/prepare_documents.py
View file

@ -12,8 +12,7 @@ from pathlib import Path
from typing import Optional
from metagpt.actions import Action, ActionOutput
from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME
from metagpt.schema import Document
from metagpt.const import REQUIREMENT_FILENAME
from metagpt.utils.file_repository import FileRepository
from metagpt.utils.git_repository import GitRepository
@ -38,7 +37,6 @@ class PrepareDocuments(Action):
if path.exists() and not self.config.inc:
shutil.rmtree(path)
self.config.project_path = path
self.config.project_name = path.name
self.context.git_repo = GitRepository(local_path=path, auto_init=True)
async def run(self, with_messages, **kwargs):
@ -46,9 +44,7 @@ class PrepareDocuments(Action):
self._init_repo()
# Write the newly added requirements from the main parameter idea to `docs/requirement.txt`.
doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
await self.file_repo.save_file(filename=REQUIREMENT_FILENAME, content=doc.content, relative_path=DOCS_FILE_REPO)
doc = await self.project_repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
# Send a Message notification to the WritePRD action, instructing it to process requirements using
# `docs/requirement.txt` and `docs/prds/`.
return ActionOutput(content=doc.content, instruct_content=doc)

49
metagpt/actions/project_management.py
View file

@ -16,12 +16,7 @@ from typing import Optional
from metagpt.actions import ActionOutput
from metagpt.actions.action import Action
from metagpt.actions.project_management_an import PM_NODE
from metagpt.const import (
PACKAGE_REQUIREMENTS_FILENAME,
SYSTEM_DESIGN_FILE_REPO,
TASK_FILE_REPO,
TASK_PDF_FILE_REPO,
)
from metagpt.const import PACKAGE_REQUIREMENTS_FILENAME
from metagpt.logs import logger
from metagpt.schema import Document, Documents
@ -39,27 +34,20 @@ class WriteTasks(Action):
i_context: Optional[str] = None
async def run(self, with_messages):
system_design_file_repo = self.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO)
changed_system_designs = system_design_file_repo.changed_files
tasks_file_repo = self.git_repo.new_file_repository(TASK_FILE_REPO)
changed_tasks = tasks_file_repo.changed_files
changed_system_designs = self.project_repo.docs.system_design.changed_files
changed_tasks = self.project_repo.docs.task.changed_files
change_files = Documents()
# Rewrite the system designs that have undergone changes based on the git head diff under
# `docs/system_designs/`.
for filename in changed_system_designs:
task_doc = await self._update_tasks(
filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo
)
task_doc = await self._update_tasks(filename=filename)
change_files.docs[filename] = task_doc
# Rewrite the task files that have undergone changes based on the git head diff under `docs/tasks/`.
for filename in changed_tasks:
if filename in change_files.docs:
continue
task_doc = await self._update_tasks(
filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo
)
task_doc = await self._update_tasks(filename=filename)
change_files.docs[filename] = task_doc
if not change_files.docs:
@ -68,21 +56,22 @@ class WriteTasks(Action):
# global optimization in subsequent steps.
return ActionOutput(content=change_files.model_dump_json(), instruct_content=change_files)
async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo):
system_design_doc = await system_design_file_repo.get(filename)
task_doc = await tasks_file_repo.get(filename)
async def _update_tasks(self, filename):
system_design_doc = await self.project_repo.docs.system_design.get(filename)
task_doc = await self.project_repo.docs.task.get(filename)
if task_doc:
task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc)
await self.project_repo.docs.task.save_doc(
doc=task_doc, dependencies={system_design_doc.root_relative_path}
)
else:
rsp = await self._run_new_tasks(context=system_design_doc.content)
task_doc = Document(
root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.model_dump_json()
task_doc = await self.project_repo.docs.task.save(
filename=filename,
content=rsp.instruct_content.model_dump_json(),
dependencies={system_design_doc.root_relative_path},
)
await tasks_file_repo.save(
filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path}
)
await self._update_requirements(task_doc)
await self._save_pdf(task_doc=task_doc)
return task_doc
async def _run_new_tasks(self, context):
@ -98,8 +87,7 @@ class WriteTasks(Action):
async def _update_requirements(self, doc):
m = json.loads(doc.content)
packages = set(m.get("Required Python third-party packages", set()))
file_repo = self.git_repo.new_file_repository()
requirement_doc = await file_repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME)
requirement_doc = await self.project_repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME)
if not requirement_doc:
requirement_doc = Document(filename=PACKAGE_REQUIREMENTS_FILENAME, root_path=".", content="")
lines = requirement_doc.content.splitlines()
@ -107,7 +95,4 @@ class WriteTasks(Action):
if pkg == "":
continue
packages.add(pkg)
await file_repo.save(PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages))
async def _save_pdf(self, task_doc):
await self.file_repo.save_as(doc=task_doc, with_suffix=".md", relative_path=TASK_PDF_FILE_REPO)
await self.project_repo.save(filename=PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages))

8
metagpt/actions/summarize_code.py
View file

@ -11,7 +11,6 @@ from pydantic import Field
from tenacity import retry, stop_after_attempt, wait_random_exponential
from metagpt.actions.action import Action
from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO
from metagpt.logs import logger
from metagpt.schema import CodeSummarizeContext
@ -99,11 +98,10 @@ class SummarizeCode(Action):
async def run(self):
design_pathname = Path(self.i_context.design_filename)
repo = self.file_repo
design_doc = await repo.get_file(filename=design_pathname.name, relative_path=SYSTEM_DESIGN_FILE_REPO)
design_doc = await self.project_repo.docs.system_design.get(filename=design_pathname.name)
task_pathname = Path(self.i_context.task_filename)
task_doc = await repo.get_file(filename=task_pathname.name, relative_path=TASK_FILE_REPO)
src_file_repo = self.git_repo.new_file_repository(relative_path=self.context.src_workspace)
task_doc = await self.project_repo.docs.task.get(filename=task_pathname.name)
src_file_repo = self.project_repo.with_src_path(self.context.src_workspace).srcs
code_blocks = []
for filename in self.i_context.codes_filenames:
code_doc = await src_file_repo.get(filename)

28
metagpt/actions/write_code.py
View file

@ -21,13 +21,7 @@ from pydantic import Field
from tenacity import retry, stop_after_attempt, wait_random_exponential
from metagpt.actions.action import Action
from metagpt.const import (
BUGFIX_FILENAME,
CODE_SUMMARIES_FILE_REPO,
DOCS_FILE_REPO,
TASK_FILE_REPO,
TEST_OUTPUTS_FILE_REPO,
)
from metagpt.const import BUGFIX_FILENAME
from metagpt.logs import logger
from metagpt.schema import CodingContext, Document, RunCodeResult
from metagpt.utils.common import CodeParser
@ -94,16 +88,12 @@ class WriteCode(Action):
return code
async def run(self, *args, **kwargs) -> CodingContext:
bug_feedback = await self.file_repo.get_file(filename=BUGFIX_FILENAME, relative_path=DOCS_FILE_REPO)
bug_feedback = await self.project_repo.docs.get(filename=BUGFIX_FILENAME)
coding_context = CodingContext.loads(self.i_context.content)
test_doc = await self.file_repo.get_file(
filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO
)
test_doc = await self.project_repo.test_outputs.get(filename="test_" + coding_context.filename + ".json")
summary_doc = None
if coding_context.design_doc and coding_context.design_doc.filename:
summary_doc = await self.file_repo.get_file(
filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO
)
summary_doc = await self.project_repo.docs.code_summary.get(filename=coding_context.design_doc.filename)
logs = ""
if test_doc:
test_detail = RunCodeResult.loads(test_doc.content)
@ -115,8 +105,7 @@ class WriteCode(Action):
code_context = await self.get_codes(
coding_context.task_doc,
exclude=self.i_context.filename,
git_repo=self.git_repo,
src_workspace=self.context.src_workspace,
project_repo=self.project_repo.with_src_path(self.context.src_workspace),
)
prompt = PROMPT_TEMPLATE.format(
@ -138,16 +127,15 @@ class WriteCode(Action):
return coding_context
@staticmethod
async def get_codes(task_doc, exclude, git_repo, src_workspace) -> str:
async def get_codes(task_doc, exclude, project_repo) -> str:
if not task_doc:
return ""
if not task_doc.content:
file_repo = git_repo.new_file_repository()
task_doc.content = file_repo.get_file(filename=task_doc.filename, relative_path=TASK_FILE_REPO)
task_doc = project_repo.docs.task.get(filename=task_doc.filename)
m = json.loads(task_doc.content)
code_filenames = m.get("Task list", [])
codes = []
src_file_repo = git_repo.new_file_repository(relative_path=src_workspace)
src_file_repo = project_repo.srcs
for filename in code_filenames:
if filename == exclude:
continue

3
metagpt/actions/write_code_review.py
View file

@ -143,8 +143,7 @@ class WriteCodeReview(Action):
code_context = await WriteCode.get_codes(
self.i_context.task_doc,
exclude=self.i_context.filename,
git_repo=self.context.git_repo,
src_workspace=self.src_workspace,
project_repo=self.project_repo.with_src_path(self.context.src_workspace),
)
context = "\n".join(
[

58
metagpt/actions/write_prd.py
View file

@ -29,9 +29,6 @@ from metagpt.actions.write_prd_an import (
from metagpt.const import (
BUGFIX_FILENAME,
COMPETITIVE_ANALYSIS_FILE_REPO,
DOCS_FILE_REPO,
PRD_PDF_FILE_REPO,
PRDS_FILE_REPO,
REQUIREMENT_FILENAME,
)
from metagpt.logs import logger
@ -67,11 +64,10 @@ class WritePRD(Action):
async def run(self, with_messages, *args, **kwargs) -> ActionOutput | Message:
# Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are
# related to the PRD. If they are related, rewrite the PRD.
docs_file_repo = self.git_repo.new_file_repository(relative_path=DOCS_FILE_REPO)
requirement_doc = await docs_file_repo.get(filename=REQUIREMENT_FILENAME)
requirement_doc = await self.project_repo.docs.get(filename=REQUIREMENT_FILENAME)
if requirement_doc and await self._is_bugfix(requirement_doc.content):
await docs_file_repo.save(filename=BUGFIX_FILENAME, content=requirement_doc.content)
await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="")
await self.project_repo.docs.save(filename=BUGFIX_FILENAME, content=requirement_doc.content)
await self.project_repo.docs.save(filename=REQUIREMENT_FILENAME, content="")
bug_fix = BugFixContext(filename=BUGFIX_FILENAME)
return Message(
content=bug_fix.model_dump_json(),
@ -82,24 +78,19 @@ class WritePRD(Action):
send_to="Alex", # the name of Engineer
)
else:
await docs_file_repo.delete(filename=BUGFIX_FILENAME)
await self.project_repo.docs.delete(filename=BUGFIX_FILENAME)
prds_file_repo = self.git_repo.new_file_repository(PRDS_FILE_REPO)
prd_docs = await prds_file_repo.get_all()
prd_docs = await self.project_repo.docs.prd.get_all()
change_files = Documents()
for prd_doc in prd_docs:
prd_doc = await self._update_prd(
requirement_doc=requirement_doc, prd_doc=prd_doc, prds_file_repo=prds_file_repo, *args, **kwargs
)
prd_doc = await self._update_prd(requirement_doc=requirement_doc, prd_doc=prd_doc, *args, **kwargs)
if not prd_doc:
continue
change_files.docs[prd_doc.filename] = prd_doc
logger.info(f"rewrite prd: {prd_doc.filename}")
# If there is no existing PRD, generate one using 'docs/requirement.txt'.
if not change_files.docs:
prd_doc = await self._update_prd(
requirement_doc=requirement_doc, prd_doc=None, prds_file_repo=prds_file_repo, *args, **kwargs
)
prd_doc = await self._update_prd(requirement_doc=requirement_doc, *args, **kwargs)
if prd_doc:
change_files.docs[prd_doc.filename] = prd_doc
logger.debug(f"new prd: {prd_doc.filename}")
@ -109,13 +100,6 @@ class WritePRD(Action):
return ActionOutput(content=change_files.model_dump_json(), instruct_content=change_files)
async def _run_new_requirement(self, requirements) -> ActionOutput:
# sas = SearchAndSummarize()
# # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US)
# rsp = ""
# info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}"
# if sas.result:
# logger.info(sas.result)
# logger.info(rsp)
project_name = self.project_name
context = CONTEXT_TEMPLATE.format(requirements=requirements, project_name=project_name)
exclude = [PROJECT_NAME.key] if project_name else []
@ -137,23 +121,21 @@ class WritePRD(Action):
await self._rename_workspace(node)
return prd_doc
async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None:
async def _update_prd(self, requirement_doc, prd_doc=None, *args, **kwargs) -> Document | None:
if not prd_doc:
prd = await self._run_new_requirement(
requirements=[requirement_doc.content if requirement_doc else ""], *args, **kwargs
)
new_prd_doc = Document(
root_path=PRDS_FILE_REPO,
filename=FileRepository.new_filename() + ".json",
content=prd.instruct_content.model_dump_json(),
new_prd_doc = await self.project_repo.docs.prd.save(
filename=FileRepository.new_filename() + ".json", content=prd.instruct_content.model_dump_json()
)
elif await self._is_relative(requirement_doc, prd_doc):
new_prd_doc = await self._merge(requirement_doc, prd_doc)
self.project_repo.docs.prd.save_doc(doc=new_prd_doc)
else:
return None
await prds_file_repo.save(filename=new_prd_doc.filename, content=new_prd_doc.content)
await self._save_competitive_analysis(new_prd_doc)
await self._save_pdf(new_prd_doc)
await self.project_repo.resources.prd.save_pdf(doc=new_prd_doc)
return new_prd_doc
async def _save_competitive_analysis(self, prd_doc):
@ -161,14 +143,13 @@ class WritePRD(Action):
quadrant_chart = m.get("Competitive Quadrant Chart")
if not quadrant_chart:
return
pathname = self.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("")
pathname = (
self.project_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("")
)
if not pathname.parent.exists():
pathname.parent.mkdir(parents=True, exist_ok=True)
await mermaid_to_file(self.config.mermaid_engine, quadrant_chart, pathname)
async def _save_pdf(self, prd_doc):
await self.file_repo.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO)
async def _rename_workspace(self, prd):
if not self.project_name:
if isinstance(prd, (ActionOutput, ActionNode)):
@ -177,11 +158,14 @@ class WritePRD(Action):
ws_name = CodeParser.parse_str(block="Project Name", text=prd)
if ws_name:
self.project_name = ws_name
self.git_repo.rename_root(self.project_name)
self.project_repo.git_repo.rename_root(self.project_name)
async def _is_bugfix(self, context) -> bool:
src_workspace_path = self.git_repo.workdir / self.git_repo.workdir.name
code_files = self.git_repo.get_files(relative_path=src_workspace_path)
git_workdir = self.project_repo.git_repo.workdir
src_workdir = git_workdir / git_workdir.name
if not src_workdir.exists():
return False
code_files = self.project_repo.with_src_path(path=git_workdir / git_workdir.name).srcs.all_files
if not code_files:
return False
node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm)

82
metagpt/roles/engineer.py
View file

@ -27,12 +27,7 @@ from typing import Set
from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks
from metagpt.actions.fix_bug import FixBug
from metagpt.actions.summarize_code import SummarizeCode
from metagpt.const import (
CODE_SUMMARIES_FILE_REPO,
CODE_SUMMARIES_PDF_FILE_REPO,
SYSTEM_DESIGN_FILE_REPO,
TASK_FILE_REPO,
)
from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import (
@ -97,7 +92,6 @@ class Engineer(Role):
async def _act_sp_with_cr(self, review=False) -> Set[str]:
changed_files = set()
src_file_repo = self.git_repo.new_file_repository(self.src_workspace)
for todo in self.code_todos:
"""
# Select essential information from the historical data to reduce the length of the prompt (summarized from human experience):
@ -112,8 +106,8 @@ class Engineer(Role):
action = WriteCodeReview(i_context=coding_context, context=self.context, llm=self.llm)
self._init_action_system_message(action)
coding_context = await action.run()
await src_file_repo.save(
coding_context.filename,
await self.project_repo.srcs.save(
filename=coding_context.filename,
dependencies={coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path},
content=coding_context.code_doc.content,
)
@ -153,31 +147,28 @@ class Engineer(Role):
)
async def _act_summarize(self):
code_summaries_file_repo = self.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO)
code_summaries_pdf_file_repo = self.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO)
tasks = []
src_relative_path = self.src_workspace.relative_to(self.git_repo.workdir)
for todo in self.summarize_todos:
summary = await todo.run()
summary_filename = Path(todo.i_context.design_filename).with_suffix(".md").name
dependencies = {todo.i_context.design_filename, todo.i_context.task_filename}
for filename in todo.i_context.codes_filenames:
rpath = src_relative_path / filename
rpath = self.project_repo.src_relative_path / filename
dependencies.add(str(rpath))
await code_summaries_pdf_file_repo.save(
await self.project_repo.resources.code_summary.save(
filename=summary_filename, content=summary, dependencies=dependencies
)
is_pass, reason = await self._is_pass(summary)
if not is_pass:
todo.i_context.reason = reason
tasks.append(todo.i_context.dict())
await code_summaries_file_repo.save(
await self.project_repo.docs.code_summary.save(
filename=Path(todo.i_context.design_filename).name,
content=todo.i_context.model_dump_json(),
dependencies=dependencies,
)
else:
await code_summaries_file_repo.delete(filename=Path(todo.i_context.design_filename).name)
await self.project_repo.docs.code_summary.delete(filename=Path(todo.i_context.design_filename).name)
logger.info(f"--max-auto-summarize-code={self.config.max_auto_summarize_code}")
if not tasks or self.config.max_auto_summarize_code == 0:
@ -220,60 +211,54 @@ class Engineer(Role):
return self.rc.todo
return None
@staticmethod
async def _new_coding_context(
filename, src_file_repo, task_file_repo, design_file_repo, dependency
) -> CodingContext:
old_code_doc = await src_file_repo.get(filename)
async def _new_coding_context(self, filename, dependency) -> CodingContext:
old_code_doc = await self.project_repo.srcs.get(filename)
if not old_code_doc:
old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content="")
old_code_doc = Document(root_path=str(self.project_repo.src_relative_path), filename=filename, content="")
dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)}
task_doc = None
design_doc = None
for i in dependencies:
if str(i.parent) == TASK_FILE_REPO:
task_doc = await task_file_repo.get(i.name)
task_doc = await self.project_repo.docs.task.get(i.name)
elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO:
design_doc = await design_file_repo.get(i.name)
design_doc = await self.project_repo.docs.system_design.get(i.name)
if not task_doc or not design_doc:
logger.error(f'Detected source code "{filename}" from an unknown origin.')
raise ValueError(f'Detected source code "{filename}" from an unknown origin.')
context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc)
return context
@staticmethod
async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_repo, dependency):
context = await Engineer._new_coding_context(
filename, src_file_repo, task_file_repo, design_file_repo, dependency
)
async def _new_coding_doc(self, filename, dependency):
context = await self._new_coding_context(filename, dependency)
coding_doc = Document(
root_path=str(src_file_repo.root_path), filename=filename, content=context.model_dump_json()
root_path=str(self.project_repo.src_relative_path), filename=filename, content=context.model_dump_json()
)
return coding_doc
async def _new_code_actions(self, bug_fix=False):
# Prepare file repos
src_file_repo = self.git_repo.new_file_repository(self.src_workspace)
changed_src_files = src_file_repo.all_files if bug_fix else src_file_repo.changed_files
task_file_repo = self.git_repo.new_file_repository(TASK_FILE_REPO)
changed_task_files = task_file_repo.changed_files
design_file_repo = self.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO)
changed_src_files = self.project_repo.srcs.all_files if bug_fix else self.project_repo.srcs.changed_files
changed_task_files = self.project_repo.docs.task.changed_files
changed_files = Documents()
# Recode caused by upstream changes.
for filename in changed_task_files:
design_doc = await design_file_repo.get(filename)
task_doc = await task_file_repo.get(filename)
design_doc = await self.project_repo.docs.system_design.get(filename)
task_doc = await self.project_repo.docs.task.get(filename)
task_list = self._parse_tasks(task_doc)
for task_filename in task_list:
old_code_doc = await src_file_repo.get(task_filename)
old_code_doc = await self.project_repo.srcs.get(task_filename)
if not old_code_doc:
old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=task_filename, content="")
old_code_doc = Document(
root_path=str(self.project_repo.src_relative_path), filename=task_filename, content=""
)
context = CodingContext(
filename=task_filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc
)
coding_doc = Document(
root_path=str(src_file_repo.root_path), filename=task_filename, content=context.model_dump_json()
root_path=str(self.project_repo.src_relative_path),
filename=task_filename,
content=context.model_dump_json(),
)
if task_filename in changed_files.docs:
logger.warning(
@ -289,13 +274,7 @@ class Engineer(Role):
for filename in changed_src_files:
if filename in changed_files.docs:
continue
coding_doc = await self._new_coding_doc(
filename=filename,
src_file_repo=src_file_repo,
task_file_repo=task_file_repo,
design_file_repo=design_file_repo,
dependency=dependency,
)
coding_doc = await self._new_coding_doc(filename=filename, dependency=dependency)
changed_files.docs[filename] = coding_doc
self.code_todos.append(WriteCode(i_context=coding_doc, context=self.context, llm=self.llm))
@ -303,13 +282,12 @@ class Engineer(Role):
self.set_todo(self.code_todos[0])
async def _new_summarize_actions(self):
src_file_repo = self.git_repo.new_file_repository(self.src_workspace)
src_files = src_file_repo.all_files
src_files = self.project_repo.srcs.all_files
# Generate a SummarizeCode action for each pair of (system_design_doc, task_doc).
summarizations = defaultdict(list)
for filename in src_files:
dependencies = await src_file_repo.get_dependency(filename=filename)
ctx = CodeSummarizeContext.loads(filenames=dependencies)
dependencies = await self.project_repo.srcs.get_dependency(filename=filename)
ctx = CodeSummarizeContext.loads(filenames=list(dependencies))
summarizations[ctx].append(filename)
for ctx, filenames in summarizations.items():
ctx.codes_filenames = filenames

33
metagpt/roles/qa_engineer.py
View file

@ -17,11 +17,7 @@
from metagpt.actions import DebugError, RunCode, WriteTest
from metagpt.actions.summarize_code import SummarizeCode
from metagpt.const import (
MESSAGE_ROUTE_TO_NONE,
TEST_CODES_FILE_REPO,
TEST_OUTPUTS_FILE_REPO,
)
from metagpt.const import MESSAGE_ROUTE_TO_NONE, TEST_CODES_FILE_REPO
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import Document, Message, RunCodeContext, TestingContext
@ -48,37 +44,32 @@ class QaEngineer(Role):
self.test_round = 0
async def _write_test(self, message: Message) -> None:
src_file_repo = self.context.git_repo.new_file_repository(self.context.src_workspace)
src_file_repo = self.project_repo.with_src_path(self.context.src_workspace).srcs
changed_files = set(src_file_repo.changed_files.keys())
# Unit tests only.
if self.config.reqa_file and self.config.reqa_file not in changed_files:
changed_files.add(self.config.reqa_file)
tests_file_repo = self.context.git_repo.new_file_repository(TEST_CODES_FILE_REPO)
for filename in changed_files:
# write tests
if not filename or "test" in filename:
continue
code_doc = await src_file_repo.get(filename)
test_doc = await tests_file_repo.get("test_" + code_doc.filename)
test_doc = await self.project_repo.tests.get("test_" + code_doc.filename)
if not test_doc:
test_doc = Document(
root_path=str(tests_file_repo.root_path), filename="test_" + code_doc.filename, content=""
root_path=str(self.project_repo.tests.root_path), filename="test_" + code_doc.filename, content=""
)
logger.info(f"Writing {test_doc.filename}..")
context = TestingContext(filename=test_doc.filename, test_doc=test_doc, code_doc=code_doc)
context = await WriteTest(i_context=context, context=self.context, llm=self.llm).run()
await tests_file_repo.save(
filename=context.test_doc.filename,
content=context.test_doc.content,
dependencies={context.code_doc.root_relative_path},
)
await self.project_repo.tests.save_doc(doc=test_doc, dependencies={context.code_doc.root_relative_path})
# prepare context for run tests in next round
run_code_context = RunCodeContext(
command=["python", context.test_doc.root_relative_path],
code_filename=context.code_doc.filename,
test_filename=context.test_doc.filename,
working_directory=str(self.context.git_repo.workdir),
working_directory=str(self.project_repo.workdir),
additional_python_paths=[str(self.context.src_workspace)],
)
self.publish_message(
@ -91,25 +82,23 @@ class QaEngineer(Role):
)
)
logger.info(f"Done {str(tests_file_repo.workdir)} generating.")
logger.info(f"Done {str(self.project_repo.tests.workdir)} generating.")
async def _run_code(self, msg):
run_code_context = RunCodeContext.loads(msg.content)
src_doc = await self.context.git_repo.new_file_repository(self.context.src_workspace).get(
src_doc = await self.project_repo.with_src_path(self.context.src_workspace).srcs.get(
run_code_context.code_filename
)
if not src_doc:
return
test_doc = await self.context.git_repo.new_file_repository(TEST_CODES_FILE_REPO).get(
run_code_context.test_filename
)
test_doc = await self.project_repo.tests.get(run_code_context.test_filename)
if not test_doc:
return
run_code_context.code = src_doc.content
run_code_context.test_code = test_doc.content
result = await RunCode(i_context=run_code_context, context=self.context, llm=self.llm).run()
run_code_context.output_filename = run_code_context.test_filename + ".json"
await self.context.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).save(
await self.project_repo.test_outputs.save(
filename=run_code_context.output_filename,
content=result.model_dump_json(),
dependencies={src_doc.root_relative_path, test_doc.root_relative_path},
@ -132,7 +121,7 @@ class QaEngineer(Role):
async def _debug_error(self, msg):
run_code_context = RunCodeContext.loads(msg.content)
code = await DebugError(i_context=run_code_context, context=self.context, llm=self.llm).run()
await self.context.file_repo.save_file(
await self.project_repo.tests.save(
filename=run_code_context.test_filename, content=code, relative_path=TEST_CODES_FILE_REPO
)
run_code_context.output = None

8
metagpt/roles/role.py
View file

@ -36,6 +36,7 @@ from metagpt.memory import Memory
from metagpt.provider import HumanProvider
from metagpt.schema import Message, MessageQueue, SerializationMixin
from metagpt.utils.common import any_to_name, any_to_str, role_raise_decorator
from metagpt.utils.project_repo import ProjectRepo
from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output
PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}. """
@ -188,6 +189,11 @@ class Role(SerializationMixin, ContextMixin, BaseModel):
def src_workspace(self, value):
self.context.src_workspace = value
@property
def project_repo(self) -> ProjectRepo:
project_repo = ProjectRepo(git_repo=self.context.git_repo)
return project_repo.with_src_path(self.context.src_workspace) if self.context.src_workspace else project_repo
@property
def prompt_schema(self):
"""Prompt schema: json/markdown"""
@ -427,7 +433,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel):
break
# act
logger.debug(f"{self._setting}: {self.rc.state=}, will do {self.rc.todo}")
rsp = await self._act() # 这个rsp是否需要publish_message？
rsp = await self._act()
actions_taken += 1
return rsp # return output from the last action

16
metagpt/utils/file_repository.py
View file

@ -183,10 +183,20 @@ class FileRepository:
"""
current_time = datetime.now().strftime("%Y%m%d%H%M%S")
return current_time
# guid_suffix = str(uuid.uuid4())[:8]
# return f"{current_time}x{guid_suffix}"
async def save_doc(self, doc: Document, with_suffix: str = None, dependencies: List[str] = None):
async def save_doc(self, doc: Document, dependencies: List[str] = None):
"""Save content to a file and update its dependencies.
:param doc: The Document instance to be saved.
:type doc: Document
:param dependencies: A list of dependencies for the saved file.
:type dependencies: List[str], optional
"""
await self.save(filename=doc.filename, content=doc.content, dependencies=dependencies)
logger.debug(f"File Saved: {str(doc.filename)}")
async def save_pdf(self, doc: Document, with_suffix: str = ".md", dependencies: List[str] = None):
"""Save a Document instance as a PDF file.
This method converts the content of the Document instance to Markdown,

7
metagpt/utils/git_repository.py
View file

@ -199,10 +199,17 @@ class GitRepository:
if new_path.exists():
logger.info(f"Delete directory {str(new_path)}")
shutil.rmtree(new_path)
if new_path.exists(): # Recheck for windows os
logger.warning(f"Failed to delete directory {str(new_path)}")
return
try:
shutil.move(src=str(self.workdir), dst=str(new_path))
except Exception as e:
logger.warning(f"Move {str(self.workdir)} to {str(new_path)} error: {e}")
finally:
if not new_path.exists(): # Recheck for windows os
logger.warning(f"Failed to move {str(self.workdir)} to {str(new_path)}")
return
logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}")
self._repository = Repo(new_path)
self._gitignore_rules = parse_gitignore(full_path=str(new_path / ".gitignore"))

44
metagpt/utils/project_repo.py
View file

@ -17,9 +17,11 @@ from metagpt.const import (
CODE_SUMMARIES_PDF_FILE_REPO,
COMPETITIVE_ANALYSIS_FILE_REPO,
DATA_API_DESIGN_FILE_REPO,
DOCS_FILE_REPO,
GRAPH_REPO_FILE_REPO,
PRD_PDF_FILE_REPO,
PRDS_FILE_REPO,
RESOURCES_FILE_REPO,
SD_OUTPUT_FILE_REPO,
SEQ_FLOW_FILE_REPO,
SYSTEM_DESIGN_FILE_REPO,
@ -33,7 +35,7 @@ from metagpt.utils.file_repository import FileRepository
from metagpt.utils.git_repository import GitRepository
class DocFileRepositories:
class DocFileRepositories(FileRepository):
prd: FileRepository
system_design: FileRepository
task: FileRepository
@ -42,6 +44,8 @@ class DocFileRepositories:
class_view: FileRepository
def __init__(self, git_repo):
super().__init__(git_repo=git_repo, relative_path=DOCS_FILE_REPO)
self.prd = git_repo.new_file_repository(relative_path=PRDS_FILE_REPO)
self.system_design = git_repo.new_file_repository(relative_path=SYSTEM_DESIGN_FILE_REPO)
self.task = git_repo.new_file_repository(relative_path=TASK_FILE_REPO)
@ -50,7 +54,7 @@ class DocFileRepositories:
self.class_view = git_repo.new_file_repository(relative_path=CLASS_VIEW_FILE_REPO)
class ResourceFileRepositories:
class ResourceFileRepositories(FileRepository):
competitive_analysis: FileRepository
data_api_design: FileRepository
seq_flow: FileRepository
@ -61,6 +65,8 @@ class ResourceFileRepositories:
sd_output: FileRepository
def __init__(self, git_repo):
super().__init__(git_repo=git_repo, relative_path=RESOURCES_FILE_REPO)
self.competitive_analysis = git_repo.new_file_repository(relative_path=COMPETITIVE_ANALYSIS_FILE_REPO)
self.data_api_design = git_repo.new_file_repository(relative_path=DATA_API_DESIGN_FILE_REPO)
self.seq_flow = git_repo.new_file_repository(relative_path=SEQ_FLOW_FILE_REPO)
@ -72,16 +78,40 @@ class ResourceFileRepositories:
class ProjectRepo(FileRepository):
def __init__(self, root: str | Path):
git_repo = GitRepository(local_path=Path(root))
super().__init__(git_repo=git_repo, relative_path=Path("."))
def __init__(self, root: str | Path = None, git_repo: GitRepository = None):
if not root and not git_repo:
raise ValueError("Invalid root and git_repo")
git_repo_ = git_repo or GitRepository(local_path=Path(root))
super().__init__(git_repo=git_repo_, relative_path=Path("."))
self._git_repo = git_repo
self._git_repo = git_repo_
self.docs = DocFileRepositories(self._git_repo)
self.resources = ResourceFileRepositories(self._git_repo)
self.tests = self._git_repo.new_file_repository(relative_path=TEST_CODES_FILE_REPO)
self.test_outputs = self._git_repo.new_file_repository(relative_path=TEST_OUTPUTS_FILE_REPO)
self._srcs_path = None
@property
def git_repo(self):
def git_repo(self) -> GitRepository:
return self._git_repo
@property
def workdir(self) -> Path:
return Path(self.git_repo.workdir)
@property
def srcs(self) -> FileRepository:
if not self._srcs_path:
raise ValueError("Call with_srcs first.")
return self._git_repo.new_file_repository(self._srcs_path)
def with_src_path(self, path: str | Path) -> ProjectRepo:
try:
self._srcs_path = Path(path).relative_to(self.workdir)
except ValueError:
self._srcs_path = Path(path)
return self
@property
def src_relative_path(self) -> Path | None:
return self._srcs_path