feat: merge send18

This commit is contained in:
莘权 马 2023-12-14 22:59:41 +08:00
parent 7effe7f74c
commit ea21217a69
54 changed files with 366 additions and 930 deletions

View file

@ -16,19 +16,13 @@
@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results
of SummarizeCode.
"""
<<<<<<< HEAD
from __future__ import annotations
import json
from collections import defaultdict
=======
import asyncio
from collections import OrderedDict
>>>>>>> send18/dev
from pathlib import Path
from typing import Set
<<<<<<< HEAD
from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks
from metagpt.actions.fix_bug import FixBug
from metagpt.actions.summarize_code import SummarizeCode
@ -49,18 +43,6 @@ from metagpt.schema import (
Message,
)
from metagpt.utils.common import any_to_name, any_to_str, any_to_str_set
=======
import aiofiles
from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import Message
from metagpt.utils.common import CodeParser
from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP
>>>>>>> send18/dev
IS_PASS_PROMPT = """
{context}
@ -85,7 +67,6 @@ class Engineer(Role):
use_code_review (bool): Whether to use code review.
"""
<<<<<<< HEAD
def __init__(
self,
name: str = "Alex",
@ -96,18 +77,6 @@ class Engineer(Role):
use_code_review: bool = False,
) -> None:
"""Initializes the Engineer role with given attributes."""
=======
class Engineer(Role):
def __init__(
self,
name="Alex",
profile="Engineer",
goal="Write elegant, readable, extensible, efficient code",
constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain",
n_borg=1,
use_code_review=False,
):
>>>>>>> send18/dev
super().__init__(name, profile, goal, constraints)
self.use_code_review = use_code_review
self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview, FixBug])
@ -121,7 +90,6 @@ class Engineer(Role):
m = json.loads(task_msg.content)
return m.get("Task list")
<<<<<<< HEAD
async def _act_sp_with_cr(self, review=False) -> Set[str]:
changed_files = set()
src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace)
@ -145,83 +113,8 @@ class Engineer(Role):
msg = Message(
content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode
)
=======
@classmethod
def parse_tasks(self, task_msg: Message) -> list[str]:
if task_msg.instruct_content:
return task_msg.instruct_content.dict().get("Task list")
return CodeParser.parse_file_list(block="Task list", text=task_msg.content)
@classmethod
def parse_code(self, code_text: str) -> str:
return CodeParser.parse_code(block="", text=code_text)
@classmethod
def parse_workspace(cls, system_design_msg: Message) -> str:
if system_design_msg.instruct_content:
return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"')
return CodeParser.parse_str(block="Python package name", text=system_design_msg.content)
def get_workspace(self) -> Path:
msg = self._rc.memory.get_by_action(WriteDesign)[-1]
if not msg:
return CONFIG.workspace / "src"
workspace = self.parse_workspace(msg)
# Codes are written in workspace/{package_name}/{package_name}
return CONFIG.workspace / workspace
async def write_file(self, filename: str, code: str):
workspace = self.get_workspace()
filename = filename.replace('"', "").replace("\n", "")
file = workspace / filename
file.parent.mkdir(parents=True, exist_ok=True)
async with aiofiles.open(file, "w") as f:
await f.write(code)
return file
def recv(self, message: Message) -> None:
self._rc.memory.add(message)
if message in self._rc.important_memory:
self.todos = self.parse_tasks(message)
async def _act_mp(self) -> Message:
# self.recreate_workspace()
todo_coros = []
for todo in self.todos:
todo_coro = WriteCode().run(
context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo
)
todo_coros.append(todo_coro)
rsps = await gather_ordered_k(todo_coros, self.n_borg)
for todo, code_rsp in zip(self.todos, rsps):
_ = self.parse_code(code_rsp)
logger.info(todo)
logger.info(code_rsp)
# self.write_file(todo, code)
msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo))
self._rc.memory.add(msg)
del self.todos[0]
logger.info(f"Done {self.get_workspace()} generating.")
msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo))
return msg
async def _act_sp(self) -> Message:
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
instruct_content = {}
for todo in self.todos:
code = await WriteCode().run(context=self._rc.history, filename=todo)
# logger.info(todo)
# logger.info(code_rsp)
# code = self.parse_code(code_rsp)
file_path = await self.write_file(todo, code)
msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo))
>>>>>>> send18/dev
self._rc.memory.add(msg)
instruct_content[todo] = code
<<<<<<< HEAD
changed_files.add(coding_context.code_doc.filename)
if not changed_files:
logger.info("Nothing has changed.")
@ -247,22 +140,8 @@ class Engineer(Role):
cause_by=WriteCodeReview if self.use_code_review else WriteCode,
send_to=self,
sent_from=self,
=======
# code_msg = todo + FILENAME_CODE_SEP + str(file_path)
code_msg = (todo, file_path)
code_msg_all.append(code_msg)
logger.info(f"Done {self.get_workspace()} generating.")
msg = Message(
content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all),
instruct_content=instruct_content,
role=self.profile,
cause_by=type(self._rc.todo),
send_to="QaEngineer",
>>>>>>> send18/dev
)
<<<<<<< HEAD
async def _act_summarize(self):
code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO)
code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO)
@ -353,49 +232,6 @@ class Engineer(Role):
async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_repo, dependency):
context = await Engineer._new_coding_context(
filename, src_file_repo, task_file_repo, design_file_repo, dependency
=======
async def _act_sp_precision(self) -> Message:
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
instruct_content = {}
for todo in self.todos:
"""
# 从历史信息中挑选必须的信息以减少prompt长度人工经验总结
1. Architect全部
2. ProjectManager全部
3. 是否需要其他代码暂时需要
TODO:目标是不需要在任务拆分清楚后根据设计思路不需要其他代码也能够写清楚单个文件如果不能则表示还需要在定义的更清晰这个是代码能够写长的关键
"""
context = []
msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode])
for m in msg:
context.append(m.content)
context_str = "\n".join(context)
# 编写code
code = await WriteCode().run(context=context_str, filename=todo)
# code review
if self.use_code_review:
try:
rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo)
code = rewrite_code
except Exception as e:
logger.error("code review failed!", e)
pass
file_path = await self.write_file(todo, code)
msg = Message(content=code, role=self.profile, cause_by=WriteCode)
self._rc.memory.add(msg)
instruct_content[todo] = code
code_msg = (todo, file_path)
code_msg_all.append(code_msg)
logger.info(f"Done {self.get_workspace()} generating.")
msg = Message(
content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all),
instruct_content=instruct_content,
role=self.profile,
cause_by=type(self._rc.todo),
send_to="QaEngineer",
>>>>>>> send18/dev
)
coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json())
return coding_doc

View file

@ -14,10 +14,7 @@
@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results
of SummarizeCode.
"""
<<<<<<< HEAD
from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest
# from metagpt.const import WORKSPACE_ROOT
from metagpt.actions import DebugError, RunCode, WriteTest
from metagpt.actions.summarize_code import SummarizeCode
from metagpt.config import CONFIG
from metagpt.const import (
@ -25,13 +22,6 @@ from metagpt.const import (
TEST_CODES_FILE_REPO,
TEST_OUTPUTS_FILE_REPO,
)
=======
import os
from pathlib import Path
from metagpt.actions import DebugError, RunCode, WriteCode, WriteDesign, WriteTest
from metagpt.config import CONFIG
>>>>>>> send18/dev
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import Document, Message, RunCodeContext, TestingContext
@ -55,32 +45,6 @@ class QaEngineer(Role):
self.test_round = 0
self.test_round_allowed = test_round_allowed
<<<<<<< HEAD
=======
@classmethod
def parse_workspace(cls, system_design_msg: Message) -> str:
if not system_design_msg.instruct_content:
return system_design_msg.instruct_content.dict().get("Python package name")
return CodeParser.parse_str(block="Python package name", text=system_design_msg.content)
def get_workspace(self, return_proj_dir=True) -> Path:
msg = self._rc.memory.get_by_action(WriteDesign)[-1]
if not msg:
return CONFIG.workspace / "src"
workspace = self.parse_workspace(msg)
# project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc.
if return_proj_dir:
return CONFIG.workspace / workspace
# development codes directory: workspace/{package_name}/{package_name}
return CONFIG.workspace / workspace / workspace
def write_file(self, filename: str, code: str):
workspace = self.get_workspace() / "tests"
file = workspace / filename
file.parent.mkdir(parents=True, exist_ok=True)
file.write_text(code)
>>>>>>> send18/dev
async def _write_test(self, message: Message) -> None:
src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace)
changed_files = set(src_file_repo.changed_files.keys())

View file

@ -1,16 +1,10 @@
#!/usr/bin/env python
"""
<<<<<<< HEAD
@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue.
@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of
the `cause_by` value in the `Message` to a string to support the new message distribution feature.
"""
=======
@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue.
"""
>>>>>>> send18/dev
import asyncio
from pydantic import BaseModel
@ -47,8 +41,6 @@ class Researcher(Role):
if language not in ("en-us", "zh-cn"):
logger.warning(f"The language `{language}` has not been tested, it may not work.")
<<<<<<< HEAD
=======
async def _think(self) -> bool:
if self._rc.todo is None:
self._set_state(0)
@ -60,7 +52,6 @@ class Researcher(Role):
self._rc.todo = None
return False
>>>>>>> send18/dev
async def _act(self) -> Message:
logger.info(f"{self._setting}: ready to {self._rc.todo}")
todo = self._rc.todo

View file

@ -4,7 +4,7 @@
@Time : 2023/5/11 14:42
@Author : alexanderwu
@File : role.py
<<<<<<< HEAD
@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue.
@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116:
1. Merge the `recv` functionality into the `_observe` function. Future message reading operations will be
consolidated within the `_observe` function.
@ -18,10 +18,6 @@
only. In the normal workflow, you should use `publish_message` or `put_message` to transmit messages.
@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing
functionality is to be consolidated into the `Environment` class.
=======
@Modified By: mashenquan, 2023-8-7, Support template-style variables, such as '{teaching_language} Teacher'.
@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue.
>>>>>>> send18/dev
"""
from __future__ import annotations
@ -31,20 +27,11 @@ from typing import Iterable, Set, Type
from pydantic import BaseModel, Field
from metagpt.actions import Action, ActionOutput
from metagpt.config import CONFIG
<<<<<<< HEAD
from metagpt.llm import LLM, HumanProvider
from metagpt.logs import logger
from metagpt.memory import Memory
from metagpt.schema import Message, MessageQueue
from metagpt.utils.common import any_to_name, any_to_str
=======
from metagpt.const import OPTIONS
from metagpt.llm import LLMFactory
from metagpt.logs import logger
from metagpt.memory import LongTermMemory, Memory
from metagpt.schema import Message, MessageTag
>>>>>>> send18/dev
PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """
@ -87,11 +74,7 @@ class RoleReactMode(str, Enum):
class RoleSetting(BaseModel):
<<<<<<< HEAD
"""Role Settings"""
=======
"""Role properties"""
>>>>>>> send18/dev
name: str
profile: str
@ -108,16 +91,10 @@ class RoleSetting(BaseModel):
class RoleContext(BaseModel):
<<<<<<< HEAD
"""Role Runtime Context"""
env: "Environment" = Field(default=None)
msg_buffer: MessageQueue = Field(default_factory=MessageQueue) # Message Buffer with Asynchronous Updates
=======
"""Runtime role context"""
env: "Environment" = Field(default=None)
>>>>>>> send18/dev
memory: Memory = Field(default_factory=Memory)
# long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory)
state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None
@ -133,34 +110,22 @@ class RoleContext(BaseModel):
arbitrary_types_allowed = True
def check(self, role_id: str):
if CONFIG.long_term_memory:
self.long_term_memory.recover_memory(role_id, self)
self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation
# if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory:
# self.long_term_memory.recover_memory(role_id, self)
# self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation
pass
@property
def important_memory(self) -> list[Message]:
<<<<<<< HEAD
"""Get the information corresponding to the watched actions"""
=======
"""Retrieve information corresponding to the attention action."""
>>>>>>> send18/dev
return self.memory.get_by_actions(self.watch)
@property
def history(self) -> list[Message]:
return self.memory.get()
@property
def prerequisite(self):
"""Retrieve information with `prerequisite` tag"""
if self.memory and hasattr(self.memory, "get_by_tags"):
vv = self.memory.get_by_tags([MessageTag.Prerequisite.value])
return vv[-1:] if len(vv) > 1 else vv
return []
class Role:
<<<<<<< HEAD
"""Role/Agent"""
def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False):
@ -168,20 +133,6 @@ class Role:
self._setting = RoleSetting(
name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, is_human=is_human
)
=======
"""Role/Proxy"""
def __init__(self, name="", profile="", goal="", constraints="", desc="", *args, **kwargs):
# Replace template-style variables, such as '{teaching_language} Teacher'.
name = Role.format_value(name)
profile = Role.format_value(profile)
goal = Role.format_value(goal)
constraints = Role.format_value(constraints)
desc = Role.format_value(desc)
self._llm = LLMFactory.new_llm()
self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc)
>>>>>>> send18/dev
self._states = []
self._actions = []
self._role_id = str(self._setting)
@ -258,12 +209,8 @@ class Role:
self._rc.todo = self._actions[self._rc.state] if state >= 0 else None
def set_env(self, env: "Environment"):
<<<<<<< HEAD
"""Set the environment in which the role works. The role can talk to the environment and can also receive
messages by observing."""
=======
"""设置角色工作所处的环境,角色可以向环境说话,也可以通过观察接受环境消息"""
>>>>>>> send18/dev
self._rc.env = env
if env:
env.set_subscription(self, self._subscription)
@ -275,7 +222,6 @@ class Role:
@property
def name(self):
<<<<<<< HEAD
"""Get virtual user name"""
return self._setting.name
@ -283,9 +229,6 @@ class Role:
def subscription(self) -> Set:
"""The labels for messages to be consumed by the Role object."""
return self._subscription
=======
"""Return role `name`, read only"""
return self._setting.name
@property
def desc(self):
@ -306,7 +249,6 @@ class Role:
def action_count(self):
"""Return number of action"""
return len(self._actions)
>>>>>>> send18/dev
def _get_prefix(self):
"""Get the role prefix"""
@ -314,20 +256,14 @@ class Role:
return self._setting.desc
return PREFIX_TEMPLATE.format(**self._setting.dict())
<<<<<<< HEAD
async def _think(self) -> None:
"""Think about what to do and decide on the next action"""
=======
async def _think(self) -> bool:
"""Consider what to do and decide on the next course of action. Return false if nothing can be done."""
>>>>>>> send18/dev
if len(self._actions) == 1:
# If there is only one action, then only this one can be performed
self._set_state(0)
return True
prompt = self._get_prefix()
prompt += STATE_TEMPLATE.format(
<<<<<<< HEAD
history=self._rc.history,
states="\n".join(self._states),
n_states=len(self._states) - 1,
@ -344,49 +280,27 @@ class Role:
if next_state == -1:
logger.info(f"End actions with {next_state=}")
self._set_state(next_state)
=======
history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1
)
next_state = await self._llm.aask(prompt)
logger.debug(f"{prompt=}")
if not next_state.isdigit() or int(next_state) not in range(len(self._states)):
logger.warning(f"Invalid answer of state, {next_state=}")
next_state = "0"
self._set_state(int(next_state))
return True
>>>>>>> send18/dev
async def _act(self) -> Message:
logger.info(f"{self._setting}: ready to {self._rc.todo}")
<<<<<<< HEAD
response = await self._rc.todo.run(self._rc.important_memory)
=======
requirement = self._rc.important_memory or self._rc.prerequisite
response = await self._rc.todo.run(requirement)
# logger.info(response)
>>>>>>> send18/dev
if isinstance(response, ActionOutput):
msg = Message(
content=response.content,
instruct_content=response.instruct_content,
role=self.profile,
<<<<<<< HEAD
cause_by=self._rc.todo,
sent_from=self,
)
elif isinstance(response, Message):
msg = response
=======
cause_by=type(self._rc.todo),
)
>>>>>>> send18/dev
else:
msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self)
self._rc.memory.add(msg)
return msg
<<<<<<< HEAD
async def _observe(self, ignore_memory=False) -> int:
"""Prepare new messages for processing from the message buffer and other sources."""
# Read unprocessed messages from the msg buffer.
@ -400,21 +314,6 @@ class Role:
# Design Rules:
# If you need to further categorize Message objects, you can do so using the Message.set_meta function.
# msg_buffer is a receiving buffer, avoid adding message data and operations to msg_buffer.
=======
async def _observe(self) -> int:
"""从环境中观察,获得重要信息,并加入记忆"""
if not self._rc.env:
return 0
env_msgs = self._rc.env.memory.get()
observed = self._rc.env.memory.get_by_actions(self._rc.watch)
self._rc.news = self._rc.memory.remember(observed) # remember recent exact or similar memories
for i in env_msgs:
self.recv(i)
>>>>>>> send18/dev
news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news]
if news_text:
logger.debug(f"{self._setting} observed: {news_text}")
@ -505,36 +404,10 @@ class Role:
self.publish_message(rsp)
return rsp
<<<<<<< HEAD
@property
def is_idle(self) -> bool:
"""If true, all actions have been executed."""
return not self._rc.news and not self._rc.todo and self._rc.msg_buffer.empty()
=======
@staticmethod
def format_value(value):
"""Fill parameters inside `value` with `options`."""
if not isinstance(value, str):
return value
if "{" not in value:
return value
merged_opts = OPTIONS.get() or {}
try:
return value.format(**merged_opts)
except KeyError as e:
logger.warning(f"Parameter is missing:{e}")
for k, v in merged_opts.items():
value = value.replace("{" + f"{k}" + "}", str(v))
return value
def add_action(self, act):
self._actions.append(act)
def add_to_do(self, act):
self._rc.todo = act
>>>>>>> send18/dev
async def think(self) -> Action:
"""The exported `think` function"""
@ -547,16 +420,7 @@ class Role:
return ActionOutput(content=msg.content, instruct_content=msg.instruct_content)
@property
<<<<<<< HEAD
def todo(self) -> str:
if self._actions:
return any_to_name(self._actions[0])
return ""
=======
def todo_description(self):
if not self._rc or not self._rc.todo:
return ""
if self._rc.todo.desc:
return self._rc.todo.desc
return f"{type(self._rc.todo).__name__}"
>>>>>>> send18/dev