fix conflict

This commit is contained in:
seehi 2024-05-06 11:24:35 +08:00
commit 14d88cc712
47 changed files with 1269 additions and 587 deletions

View file

@ -16,15 +16,7 @@ from metagpt.tools.libs import (
browser,
deployer,
)
from metagpt.tools.libs.software_development import (
write_prd,
write_design,
write_project_plan,
write_codes,
run_qa_test,
fix_bug,
git_archive,
)
from metagpt.tools.libs.env import get_env, set_get_env_entry, default_get_env, get_env_description
_ = (
data_preprocess,
@ -33,15 +25,12 @@ _ = (
gpt_v_generator,
web_scraping,
email_login,
write_prd,
write_design,
write_project_plan,
write_codes,
run_qa_test,
fix_bug,
git_archive,
terminal,
file_manager,
browser,
deployer,
get_env,
get_env_description,
set_get_env_entry,
default_get_env,
) # Avoid pre-commit error

109
metagpt/tools/libs/env.py Normal file
View file

@ -0,0 +1,109 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2024/4/25
@Author : mashenquan
@File : env.py
@Desc: Implement `get_env`. RFC 216 2.4.2.4.2.
"""
import os
from typing import Dict
class EnvKeyNotFoundError(Exception):
def __init__(self, info):
super().__init__(info)
async def default_get_env(key: str, app_name: str = None) -> str:
if key in os.environ:
return os.environ[key]
from metagpt.context import Context
context = Context()
val = context.kwargs.get(key, None)
if val is not None:
return val
raise EnvKeyNotFoundError(f"EnvKeyNotFoundError: {key}, app_name:{app_name or ''}")
async def default_get_env_description() -> Dict[str, str]:
result = {}
for k in os.environ.keys():
call = f'await get_env(key="{k}", app_name="")'
result[call] = f"Return the value of environment variable `{k}`."
from metagpt.context import Context
context = Context()
for k in context.kwargs.__dict__.keys():
call = f'await get_env(key="{k}", app_name="")'
result[call] = f"Get the value of environment variable `{k}`."
return result
_get_env_entry = default_get_env
_get_env_description_entry = default_get_env_description
async def get_env(key: str, app_name: str = None) -> str:
"""
Retrieve the value of the environment variable for the specified key.
Args:
key (str): The key of the environment variable.
app_name (str, optional): The name of the application. Defaults to None.
Returns:
str: The value corresponding to the given key in the environment variables.
If no value is found for the given key, an empty string is returned.
Example:
This function can be used to retrieve environment variables asynchronously.
It should be called using `await`.
>>> from metagpt.tools.libs.env import get_env
>>> api_key = await get_env("API_KEY")
>>> print(api_key)
<API_KEY>
>>> from metagpt.tools.libs.env import get_env
>>> api_key = await get_env(key="API_KEY", app_name="GITHUB")
>>> print(api_key)
<API_KEY>
Note:
This is an asynchronous function and must be called using `await`.
"""
global _get_env_entry
if _get_env_entry:
return await _get_env_entry(key=key, app_name=app_name)
return await default_get_env(key=key, app_name=app_name)
async def get_env_description() -> Dict[str, str]:
global _get_env_description_entry
if _get_env_description_entry:
return await _get_env_description_entry()
return await default_get_env_description()
def set_get_env_entry(value, description):
"""Modify `get_env` entry and `get_description` entry.
Args:
value (function): New function entry.
description (str): Description of the function.
This function modifies the `get_env` entry by updating the function
to the provided `value` and its description to the provided `description`.
"""
global _get_env_entry
global _get_env_description_entry
_get_env_entry = value
_get_env_description_entry = description

View file

@ -3,6 +3,10 @@
from __future__ import annotations
from pathlib import Path
from typing import Optional
from github.Issue import Issue
from github.PullRequest import PullRequest
from metagpt.tools.tool_registry import register_tool
from metagpt.utils.git_repository import GitRepository
@ -63,3 +67,147 @@ async def git_checkout(repo_dir: str | Path, commit_id: str):
if not repo.is_valid:
ValueError(f"Invalid git root: {repo_dir}")
await repo.checkout(commit_id)
@register_tool(tags=["git"])
async def create_pull_request(
access_token: str,
base: str,
head: str,
base_repo_name: str,
head_repo_name: Optional[str] = None,
title: Optional[str] = None,
body: Optional[str] = None,
) -> PullRequest:
"""
Creates a pull request in a Git repository.
Args:
access_token (str): The access token for authentication.
base (str): The name of the base branch of the pull request (e.g., 'main', 'master').
head (str): The name of the head branch of the pull request (e.g., 'feature-branch').
base_repo_name (str): The full repository name (user/repo) where the pull request will be created.
head_repo_name (Optional[str], optional): The full repository name (user/repo) where the pull request will merge from. Defaults to None.
title (Optional[str]): The title of the pull request.
body (Optional[str]): The body of the pull request.
Returns:
PullRequest: The created pull request object.
Raises:
ValueError: If `access_token` is invalid. Visit: "https://github.com/settings/tokens"
Any exceptions that might occur during the pull request creation process.
Note:
This function is intended to be used in an asynchronous context (with `await`).
Example:
>>> # Merge Request
>>> repo_name = "user/repo" # "user/repo" for example: "https://github.com/user/repo.git"
>>> base = "master" # branch that merge to
>>> head = "feature/new_feature" # branch that merge from
>>> title = "Implement new feature"
>>> body = "This pull request adds functionality X, Y, and Z."
>>> pull_request = await create_pull_request(
repo_name=repo_name,
base=base,
head=head,
title=title,
body=body,
access_token=get_env("git_access_token")
)
>>> print(pull_request)
PullRequest(title="Implement new feature", number=26)
>>> # Pull Request
>>> base_repo_name = "user1/repo1" # for example: "user1/repo1" from "https://github.com/user1/repo1.git"
>>> head_repo_name = "user2/repo2" # for example: "user2/repo2" from "https://github.com/user2/repo2.git"
>>> base = "master" # branch that merge to
>>> head = "feature/new_feature" # branch that merge from
>>> title = "Implement new feature"
>>> body = "This pull request adds functionality X, Y, and Z."
>>> pull_request = await create_pull_request(
base_repo_name=base_repo_name,
head_repo_name=head_repo_name,
base=base,
head=head,
title=title,
body=body,
access_token=get_env("git_access_token")
)
>>> print(pull_request)
PullRequest(title="Implement new feature", number=26)
"""
return await GitRepository.create_pull(
base_repo_name=base_repo_name,
head_repo_name=head_repo_name,
base=base,
head=head,
title=title,
body=body,
access_token=access_token,
)
@register_tool(tags=["git"])
async def create_issue(
access_token: str,
repo_name: str,
title: str,
body: Optional[str] = None,
assignee: Optional[str] = None,
labels: Optional[list[str]] = None,
) -> Issue:
"""
Creates an issue in the specified repository.
Args:
access_token (str): The access token for authentication.
Visit `https://github.com/settings/tokens` to obtain a personal access token.
For more authentication options, visit: `https://pygithub.readthedocs.io/en/latest/examples/Authentication.html`
repo_name (str): The full repository name (user/repo) where the issue will be created.
title (str): The title of the issue.
body (Optional[str], optional): The body of the issue. Defaults to None.
assignee (Optional[str], optional): The username of the assignee for the issue. Defaults to None.
labels (Optional[list[str]], optional): A list of label names to associate with the issue. Defaults to None.
Returns:
Issue: The created issue object.
Example:
>>> # Create an issue with title and body
>>> repo_name = "username/repository"
>>> title = "Bug Report"
>>> body = "I found a bug in the application."
>>> issue = await create_issue(
repo_name=repo_name,
title=title,
body=body,
access_token=get_env("git_access_token")
)
>>> print(issue)
Issue(title="Bug Report", number=26)
>>> # Create an issue with title, body, assignee, and labels
>>> repo_name = "username/repository"
>>> title = "Bug Report"
>>> body = "I found a bug in the application."
>>> assignee = "john_doe"
>>> labels = ["enhancement", "help wanted"]
>>> issue = await create_issue(
repo_name=repo_name,
title=title,
body=body,
assignee=assigee,
labels=labels,
access_token=get_env("git_access_token")
)
>>> print(issue)
Issue(title="Bug Report", number=26)
"""
return await GitRepository.create_issue(
repo_name=repo_name, title=title, body=body, assignee=assignee, labels=labels, access_token=access_token
)

View file

@ -89,7 +89,7 @@ class GPTvGenerator:
webpages_path.mkdir(parents=True, exist_ok=True)
index_path = webpages_path / "index.html"
index_path.write_text(CodeParser.parse_code(block=None, text=webpages, lang="html"))
index_path.write_text(CodeParser.parse_code(text=webpages, lang="html"))
extract_and_save_code(folder=webpages_path, text=webpages, pattern="styles?.css", language="css")
@ -102,5 +102,5 @@ def extract_and_save_code(folder, text, pattern, language):
word = re.search(pattern, text)
if word:
path = folder / word.group(0)
code = CodeParser.parse_code(block=None, text=text, lang=language)
code = CodeParser.parse_code(text=text, lang=language)
path.write_text(code, encoding="utf-8")

View file

@ -3,384 +3,10 @@
from __future__ import annotations
from pathlib import Path
from typing import Optional
from metagpt.const import ASSISTANT_ALIAS, BUGFIX_FILENAME, REQUIREMENT_FILENAME
from metagpt.const import ASSISTANT_ALIAS
from metagpt.logs import ToolLogItem, log_tool_output
from metagpt.schema import BugFixContext, Message
from metagpt.tools.tool_registry import register_tool
from metagpt.utils.common import any_to_str
from metagpt.utils.project_repo import ProjectRepo
@register_tool(tags=["software development", "ProductManager"])
async def write_prd(idea: str, project_path: Optional[str | Path] = None) -> Path:
"""Writes a PRD based on user requirements.
Args:
idea (str): The idea or concept for the PRD.
project_path (Optional[str|Path], optional): The path to an existing project directory.
If it's None, a new project path will be created. Defaults to None.
Returns:
Path: The path to the PRD files under the project directory
Example:
>>> # Create a new project:
>>> from metagpt.tools.libs.software_development import write_prd
>>> prd_path = await write_prd("Create a new feature for the application")
>>> print(prd_path)
'/path/to/project_path/docs/prd/'
>>> # Add user requirements to the exists project:
>>> from metagpt.tools.libs.software_development import write_prd
>>> project_path = '/path/to/exists_project_path'
>>> prd_path = await write_prd("Create a new feature for the application", project_path=project_path)
>>> print(prd_path = )
'/path/to/project_path/docs/prd/'
"""
from metagpt.actions import UserRequirement
from metagpt.context import Context
from metagpt.roles import ProductManager
log_tool_output(output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_prd.__name__)], tool_name=write_prd.__name__)
ctx = Context()
if project_path and Path(project_path).exists():
ctx.config.project_path = Path(project_path)
ctx.config.inc = True
role = ProductManager(context=ctx)
msg = await role.run(with_message=Message(content=idea, cause_by=UserRequirement))
await role.run(with_message=msg)
outputs = [
ToolLogItem(name="Intermedia PRD File", value=str(ctx.repo.docs.prd.workdir / i))
for i in ctx.repo.docs.prd.changed_files.keys()
]
outputs.extend(
[
ToolLogItem(name="PRD File", value=str(ctx.repo.resources.prd.workdir / i))
for i in ctx.repo.resources.prd.changed_files.keys()
]
)
outputs.extend(
[
ToolLogItem(name="Competitive Analysis", value=str(ctx.repo.resources.competitive_analysis.workdir / i))
for i in ctx.repo.resources.competitive_analysis.changed_files.keys()
]
)
log_tool_output(output=outputs, tool_name=write_prd.__name__)
return ctx.repo.docs.prd.workdir
@register_tool(tags=["Design", "software development", "Architect"])
async def write_design(prd_path: str | Path) -> Path:
"""Writes a system design to the project repository, based on the PRD of the project.
Args:
prd_path (str|Path): The path to the PRD files under the project directory.
Returns:
Path: The path to the system design files under the project directory.
Example:
>>> from metagpt.tools.libs.software_development import write_design
>>> prd_path = '/path/to/project_path/docs/prd' # Returned by `write_prd`
>>> system_design_path = await write_desgin(prd_path)
>>> print(system_design_path)
'/path/to/project_path/docs/system_design/'
"""
from metagpt.actions import WritePRD
from metagpt.context import Context
from metagpt.roles import Architect
log_tool_output(
output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_design.__name__)], tool_name=write_design.__name__
)
ctx = Context()
prd_path = Path(prd_path)
project_path = (Path(prd_path) if not prd_path.is_file() else prd_path.parent) / "../.."
ctx.set_repo_dir(project_path)
role = Architect(context=ctx)
await role.run(with_message=Message(content="", cause_by=WritePRD))
outputs = [
ToolLogItem(name="Intermedia Design File", value=str(ctx.repo.docs.system_design.workdir / i))
for i in ctx.repo.docs.system_design.changed_files.keys()
]
for i in ctx.repo.resources.system_design.changed_files.keys():
outputs.append(ToolLogItem(name="Design File", value=str(ctx.repo.resources.system_design.workdir / i)))
for i in ctx.repo.resources.data_api_design.changed_files.keys():
outputs.append(
ToolLogItem(name="Class Diagram File", value=str(ctx.repo.resources.data_api_design.workdir / i))
)
for i in ctx.repo.resources.seq_flow.changed_files.keys():
outputs.append(ToolLogItem(name="Sequence Diagram File", value=str(ctx.repo.resources.seq_flow.workdir / i)))
log_tool_output(output=outputs, tool_name=write_design.__name__)
return ctx.repo.docs.system_design.workdir
@register_tool(tags=["software development", "Architect"])
async def write_project_plan(system_design_path: str | Path) -> Path:
"""Writes a project plan to the project repository, based on the design of the project.
Args:
system_design_path (str|Path): The path to the system design files under the project directory.
Returns:
Path: The path to task files under the project directory.
Example:
>>> from metagpt.tools.libs.software_development import write_project_plan
>>> system_design_path = '/path/to/project_path/docs/system_design/' # Returned by `write_design`
>>> task_path = await write_project_plan(system_design_path)
>>> print(task_path)
'/path/to/project_path/docs/task'
"""
from metagpt.actions import WriteDesign
from metagpt.context import Context
from metagpt.roles import ProjectManager
log_tool_output(
output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_project_plan.__name__)],
tool_name=write_project_plan.__name__,
)
ctx = Context()
system_design_path = Path(system_design_path)
project_path = (system_design_path if not system_design_path.is_file() else system_design_path.parent) / "../.."
ctx.set_repo_dir(project_path)
role = ProjectManager(context=ctx)
await role.run(with_message=Message(content="", cause_by=WriteDesign))
outputs = [
ToolLogItem(name="Intermedia Project Plan", value=str(ctx.repo.docs.task.workdir / i))
for i in ctx.repo.docs.task.changed_files.keys()
]
outputs.extend(
[
ToolLogItem(name="Project Plan", value=str(ctx.repo.resources.api_spec_and_task.workdir / i))
for i in ctx.repo.resources.api_spec_and_task.changed_files.keys()
]
)
log_tool_output(output=outputs, tool_name=write_project_plan.__name__)
return ctx.repo.docs.task.workdir
@register_tool(tags=["software development", "Engineer"])
async def write_codes(task_path: str | Path, inc: bool = False) -> Path:
"""Writes code to implement designed features according to the project plan and adds them to the project repository.
In code writing tasks, prioritize calling this tool against writing code from scratch directly.
Args:
task_path (str|Path): The path to task files under the project directory.
inc (bool, optional): Whether to write incremental codes. Defaults to False.
Returns:
Path: The path to the source code files under the project directory.
Example:
# Write codes to a new project
>>> from metagpt.tools.libs.software_development import write_codes
>>> task_path = '/path/to/project_path/docs/task' # Returned by `write_project_plan`
>>> src_path = await write_codes(task_path)
>>> print(src_path)
'/path/to/project_path/src/'
# Write increment codes to the exists project
>>> from metagpt.tools.libs.software_development import write_codes
>>> task_path = '/path/to/project_path/docs/task' # Returned by `write_prd`
>>> src_path = await write_codes(task_path, inc=True)
>>> print(src_path)
'/path/to/project_path/src/'
"""
from metagpt.actions import WriteTasks
from metagpt.context import Context
from metagpt.roles import Engineer
log_tool_output(
output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_codes.__name__)], tool_name=write_codes.__name__
)
ctx = Context()
ctx.config.inc = inc
task_path = Path(task_path)
project_path = (task_path if not task_path.is_file() else task_path.parent) / "../.."
ctx.set_repo_dir(project_path)
role = Engineer(context=ctx)
msg = Message(content="", cause_by=WriteTasks, send_to=role)
me = {any_to_str(role), role.name}
while me.intersection(msg.send_to):
msg = await role.run(with_message=msg)
outputs = [
ToolLogItem(name="Source File", value=str(ctx.repo.srcs.workdir / i))
for i in ctx.repo.srcs.changed_files.keys()
]
log_tool_output(output=outputs, tool_name=write_codes.__name__)
return ctx.repo.srcs.workdir
@register_tool(tags=["software development", "QaEngineer"])
async def run_qa_test(src_path: str | Path) -> Path:
"""Run QA test on the project repository.
Args:
src_path (str|Path): The path to the source code files under the project directory.
Returns:
Path: The path to the unit tests under the project directory
Example:
>>> from metagpt.tools.libs.software_development import run_qa_test
>>> src_path = '/path/to/project_path/src/' # Returned by `write_codes`
>>> test_path = await run_qa_test(src_path)
>>> print(test_path)
'/path/to/project_path/tests'
"""
from metagpt.actions.summarize_code import SummarizeCode
from metagpt.context import Context
from metagpt.environment import Environment
from metagpt.roles import QaEngineer
log_tool_output(
output=[ToolLogItem(name=ASSISTANT_ALIAS, value=run_qa_test.__name__)], tool_name=run_qa_test.__name__
)
ctx = Context()
src_path = Path(src_path)
project_path = (src_path if not src_path.is_file() else src_path.parent) / ".."
ctx.set_repo_dir(project_path)
ctx.src_workspace = ctx.git_repo.workdir / ctx.git_repo.workdir.name
env = Environment(context=ctx)
role = QaEngineer(context=ctx)
env.add_role(role)
msg = Message(content="", cause_by=SummarizeCode, send_to=role)
env.publish_message(msg)
while not env.is_idle:
await env.run()
outputs = [
ToolLogItem(name="Unit Test File", value=str(ctx.repo.tests.workdir / i))
for i in ctx.repo.tests.changed_files.keys()
]
log_tool_output(output=outputs, tool_name=run_qa_test.__name__)
return ctx.repo.tests.workdir
@register_tool(tags=["software development", "Engineer"])
async def fix_bug(project_path: str | Path, issue: str) -> Path:
"""Fix bugs in the project repository.
Args:
project_path (str|Path): The path to the project repository.
issue (str): Description of the bug or issue.
Returns:
Path: The path to the project directory
Example:
>>> from metagpt.tools.libs.software_development import fix_bug
>>> project_path = '/path/to/project_path' # Returned by `write_codes`
>>> issue = 'Exception: exception about ...; Bug: bug about ...; Issue: issue about ...'
>>> project_path = await fix_bug(project_path=project_path, issue=issue)
>>> print(project_path)
'/path/to/project_path'
"""
from metagpt.actions.fix_bug import FixBug
from metagpt.context import Context
from metagpt.roles import Engineer
log_tool_output(output=[ToolLogItem(name=ASSISTANT_ALIAS, value=fix_bug.__name__)], tool_name=fix_bug.__name__)
ctx = Context()
ctx.set_repo_dir(project_path)
ctx.src_workspace = ctx.git_repo.workdir / ctx.git_repo.workdir.name
await ctx.repo.docs.save(filename=BUGFIX_FILENAME, content=issue)
await ctx.repo.docs.save(filename=REQUIREMENT_FILENAME, content="")
role = Engineer(context=ctx)
bug_fix = BugFixContext(filename=BUGFIX_FILENAME)
msg = Message(
content=bug_fix.model_dump_json(),
instruct_content=bug_fix,
role="",
cause_by=FixBug,
sent_from=role,
send_to=role,
)
me = {any_to_str(role), role.name}
while me.intersection(msg.send_to):
msg = await role.run(with_message=msg)
outputs = [
ToolLogItem(name="Changed File", value=str(ctx.repo.srcs.workdir / i))
for i in ctx.repo.srcs.changed_files.keys()
]
log_tool_output(output=outputs, tool_name=fix_bug.__name__)
return project_path
@register_tool(tags=["software development", "git"])
async def git_archive(project_path: str | Path) -> str:
"""Stage and commit changes for the project repository using Git.
Args:
project_path (str|Path): The path to the project repository.
Returns:
git log
Example:
>>> from metagpt.tools.libs.software_development import git_archive
>>> project_path = '/path/to/project_path' # Returned by `write_prd`
>>> git_log = await git_archive(project_path=project_path)
>>> print(git_log)
commit a221d1c418c07f2b4fc07001e486285ead1a520a (HEAD -> feature/toollib/software_company, geekan/main)
Merge: e01afd09 4a72f398
Author: Sirui Hong <x@xx.github.com>
Date: Tue Mar 19 15:16:03 2024 +0800
Merge pull request #1037 from iorisa/fixbug/issues/1018
fixbug: #1018
"""
from metagpt.context import Context
log_tool_output(
output=[ToolLogItem(name=ASSISTANT_ALIAS, value=git_archive.__name__)], tool_name=git_archive.__name__
)
ctx = Context()
project_dir = ProjectRepo.search_project_path(project_path)
if not project_dir:
ValueError(f"{project_path} is not a valid git repository.")
ctx.set_repo_dir(project_dir)
files = " ".join(ctx.git_repo.changed_files.keys())
outputs = [ToolLogItem(name="cmd", value=f"git add {files}")]
log_tool_output(output=outputs, tool_name=git_archive.__name__)
ctx.git_repo.archive()
outputs = [ToolLogItem(name="cmd", value="git commit -m 'Archive'")]
log_tool_output(output=outputs, tool_name=git_archive.__name__)
return ctx.git_repo.log()
@register_tool(tags=["software development", "import git repo"])

View file

@ -132,7 +132,7 @@ class ToolRecommender(BaseModel):
topk=topk,
)
rsp = await LLM().aask(prompt, stream=False)
rsp = CodeParser.parse_code(block=None, text=rsp)
rsp = CodeParser.parse_code(text=rsp)
ranked_tools = json.loads(rsp)
valid_tools = validate_tool_names(ranked_tools)