feat: merge send18

This commit is contained in:
莘权 马 2023-12-14 22:59:41 +08:00
parent 7effe7f74c
commit ea21217a69
54 changed files with 366 additions and 930 deletions

View file

@ -8,19 +8,21 @@
@Modified By: mashenquan, 2023/9/8. Replace LLM with LLMFactory
"""
import re
from __future__ import annotations
import re
from abc import ABC
from typing import Optional
from tenacity import retry, stop_after_attempt, wait_random_exponential
from metagpt.actions.action_output import ActionOutput
from metagpt.llm import LLM
from metagpt.logs import logger
from metagpt.utils.common import OutputParser
from metagpt.utils.custom_decoder import CustomDecoder
from metagpt.logs import logger
from metagpt.provider.base_gpt_api import BaseGPTAPI
from metagpt.utils.common import OutputParser
from metagpt.utils.custom_decoder import CustomDecoder
class Action(ABC):
def __init__(self, name: str = "", context=None, llm: BaseGPTAPI = None):

View file

@ -7,7 +7,7 @@
@Modified By: mashenquan, 2023/8/20. Allow 'instruct_content' to be blank.
"""
from typing import Dict, Type, Optional
from typing import Dict, Optional, Type
from pydantic import BaseModel, create_model, root_validator, validator
@ -16,7 +16,7 @@ class ActionOutput:
content: str
instruct_content: Optional[BaseModel] = None
def __init__(self, content: str, instruct_content: BaseModel=None):
def __init__(self, content: str, instruct_content: BaseModel = None):
self.content = content
self.instruct_content = instruct_content

View file

@ -4,7 +4,6 @@
@Time : 2023/5/11 19:26
@Author : alexanderwu
@File : design_api.py
<<<<<<< HEAD
@Modified By: mashenquan, 2023/11/27.
1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality.
@ -23,16 +22,6 @@ from metagpt.const import (
SYSTEM_DESIGN_FILE_REPO,
SYSTEM_DESIGN_PDF_FILE_REPO,
)
=======
@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class.
"""
from typing import List
import aiofiles
from metagpt.actions import Action
from metagpt.config import CONFIG
>>>>>>> send18/dev
from metagpt.logs import logger
from metagpt.schema import Document, Documents
from metagpt.utils.file_repository import FileRepository
@ -208,7 +197,6 @@ class WriteDesign(Action):
"clearly and in detail."
)
<<<<<<< HEAD
async def run(self, with_messages, format=CONFIG.prompt_format):
# Use `git diff` to identify which PRD documents have been modified in the `docs/prds` directory.
prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO)
@ -244,30 +232,6 @@ class WriteDesign(Action):
format_example = format_example.format(project_name=CONFIG.project_name)
prompt = prompt_template.format(context=context, format_example=format_example)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format)
=======
async def _save_system_design(self, docs_path, resources_path, content):
data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content)
seq_flow = CodeParser.parse_code(block="Program call flow", text=content)
await mermaid_to_file(data_api_design, resources_path / "data_api_design")
await mermaid_to_file(seq_flow, resources_path / "seq_flow")
system_design_file = docs_path / "system_design.md"
logger.info(f"Saving System Designs to {system_design_file}")
async with aiofiles.open(system_design_file, "w") as f:
await f.write(content)
async def _save(self, system_design: str):
workspace = CONFIG.workspace
docs_path = workspace / "docs"
resources_path = workspace / "resources"
docs_path.mkdir(parents=True, exist_ok=True)
resources_path.mkdir(parents=True, exist_ok=True)
await self._save_system_design(docs_path, resources_path, system_design)
async def run(self, context, **kwargs):
prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING)
await self._save(system_design.content)
>>>>>>> send18/dev
return system_design
async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format):

View file

@ -4,19 +4,14 @@
@Time : 2023/5/11 19:12
@Author : alexanderwu
@File : project_management.py
<<<<<<< HEAD
@Modified By: mashenquan, 2023/11/27.
1. Divide the context into three components: legacy code, unit test code, and console log.
2. Move the document storage operations related to WritePRD from the save operation of WriteDesign.
3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality.
=======
@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class.
>>>>>>> send18/dev
"""
import json
from typing import List
<<<<<<< HEAD
from metagpt.actions import ActionOutput
from metagpt.actions.action import Action
from metagpt.config import CONFIG
@ -91,14 +86,6 @@ and only output the json inside this tag, nothing else
},
"markdown": {
"PROMPT_TEMPLATE": """
=======
import aiofiles
from metagpt.actions.action import Action
from metagpt.config import CONFIG
PROMPT_TEMPLATE = """
>>>>>>> send18/dev
# Context
{context}
@ -121,11 +108,7 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first.
<<<<<<< HEAD
## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs.
=======
"""
>>>>>>> send18/dev
""",
"FORMAT_EXAMPLE": '''
@ -197,7 +180,6 @@ MERGE_PROMPT = """
# Context
{context}
<<<<<<< HEAD
## Old Tasks
{old_tasks}
-----
@ -228,13 +210,10 @@ and only output the json inside this tag, nothing else
"""
=======
>>>>>>> send18/dev
class WriteTasks(Action):
def __init__(self, name="CreateTasks", context=None, llm=None):
super().__init__(name, context, llm)
<<<<<<< HEAD
async def run(self, with_messages, format=CONFIG.prompt_format):
system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO)
changed_system_designs = system_design_file_repo.changed_files
@ -286,29 +265,13 @@ class WriteTasks(Action):
prompt_template, format_example = get_template(templates, format)
prompt = prompt_template.format(context=context, format_example=format_example)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format)
=======
async def _save(self, rsp):
file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md"
async with aiofiles.open(file_path, "w") as f:
await f.write(rsp.content)
# Write requirements.txt
requirements_path = CONFIG.workspace / "requirements.txt"
async with aiofiles.open(requirements_path, "w") as f:
await f.write(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n'))
async def run(self, context, **kwargs):
prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING)
await self._save(rsp)
>>>>>>> send18/dev
return rsp
async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document:
_, format_example = get_template(templates, format)
prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content,
format_example=format_example)
prompt = MERGE_PROMPT.format(
context=system_design_doc.content, old_tasks=task_doc.content, format_example=format_example
)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format)
task_doc.content = rsp.instruct_content.json(ensure_ascii=False)
return task_doc

View file

@ -14,27 +14,23 @@
3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into
RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError.
"""
<<<<<<< HEAD
import json
from tenacity import retry, stop_after_attempt, wait_random_exponential
from metagpt.actions.action import Action
from metagpt.config import CONFIG
from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO, TASK_FILE_REPO, BUGFIX_FILENAME, \
DOCS_FILE_REPO
=======
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action import Action
>>>>>>> send18/dev
from metagpt.const import (
BUGFIX_FILENAME,
CODE_SUMMARIES_FILE_REPO,
DOCS_FILE_REPO,
TASK_FILE_REPO,
TEST_OUTPUTS_FILE_REPO,
)
from metagpt.logs import logger
from metagpt.schema import CodingContext, Document, RunCodeResult
from metagpt.utils.common import CodeParser
<<<<<<< HEAD
from metagpt.utils.file_repository import FileRepository
=======
>>>>>>> send18/dev
PROMPT_TEMPLATE = """
NOTICE
@ -98,21 +94,12 @@ class WriteCode(Action):
def __init__(self, name="WriteCode", context=None, llm=None):
super().__init__(name, context, llm)
<<<<<<< HEAD
@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
async def write_code(self, prompt) -> str:
=======
def _is_invalid(self, filename):
return any(i in filename for i in ["mp3", "wav"])
@retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
async def write_code(self, prompt):
>>>>>>> send18/dev
code_rsp = await self._aask(prompt)
code = CodeParser.parse_code(block="", text=code_rsp)
return code
<<<<<<< HEAD
async def run(self, *args, **kwargs) -> CodingContext:
bug_feedback = await FileRepository.get_file(filename=BUGFIX_FILENAME, relative_path=DOCS_FILE_REPO)
coding_context = CodingContext.loads(self.context.content)
@ -139,11 +126,6 @@ class WriteCode(Action):
summary_log=summary_doc.content if summary_doc else "",
)
logger.info(f"Writing {coding_context.filename}..")
=======
async def run(self, context, filename):
prompt = PROMPT_TEMPLATE.format(context=context, filename=filename)
logger.info(f"Writing {filename}..")
>>>>>>> send18/dev
code = await self.write_code(prompt)
if not coding_context.code_doc:
coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace)
@ -166,4 +148,3 @@ class WriteCode(Action):
continue
codes.append(doc.content)
return "\n----------\n".join(codes)

View file

@ -16,22 +16,20 @@ import json
from pathlib import Path
from typing import List
import aiofiles
from metagpt.actions import Action, ActionOutput
from metagpt.actions.fix_bug import FixBug
from metagpt.actions.search_and_summarize import SearchAndSummarize
from metagpt.config import CONFIG
<<<<<<< HEAD
from metagpt.const import (
BUGFIX_FILENAME,
COMPETITIVE_ANALYSIS_FILE_REPO,
DOCS_FILE_REPO,
PRD_PDF_FILE_REPO,
PRDS_FILE_REPO,
REQUIREMENT_FILENAME, BUGFIX_FILENAME,
REQUIREMENT_FILENAME,
)
from metagpt.logs import logger
from metagpt.schema import Document, Documents, Message, BugFixContext
from metagpt.schema import BugFixContext, Document, Documents, Message
from metagpt.utils.common import CodeParser
from metagpt.utils.file_repository import FileRepository
from metagpt.utils.get_template import get_template
@ -55,11 +53,6 @@ Requirements: According to the context, fill in the following missing informatio
ATTENTION: Output carefully referenced "Format example" in format.
## YOU NEED TO FULFILL THE BELOW JSON DOC
=======
from metagpt.logs import logger
from metagpt.utils.common import CodeParser
from metagpt.utils.mermaid import mermaid_to_file
>>>>>>> send18/dev
{{
"Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc.
@ -245,11 +238,7 @@ OUTPUT_MAPPING = {
"Competitive Analysis": (List[str], ...),
"Competitive Quadrant Chart": (str, ...),
"Requirement Analysis": (str, ...),
<<<<<<< HEAD
"Requirement Pool": (List[List[str]], ...),
=======
"Requirement Pool": (List[Tuple[str, str]], ...),
>>>>>>> send18/dev
"UI Design draft": (str, ...),
"Anything UNCLEAR": (str, ...),
}
@ -346,12 +335,14 @@ class WritePRD(Action):
await docs_file_repo.save(filename=BUGFIX_FILENAME, content=requirement_doc.content)
await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="")
bug_fix = BugFixContext(filename=BUGFIX_FILENAME)
return Message(content=bug_fix.json(), instruct_content=bug_fix,
role=self.profile,
cause_by=FixBug,
sent_from=self,
send_to="Alex", # the name of Engineer
)
return Message(
content=bug_fix.json(),
instruct_content=bug_fix,
role=self.profile,
cause_by=FixBug,
sent_from=self,
send_to="Alex", # the name of Engineer
)
else:
await docs_file_repo.delete(filename=BUGFIX_FILENAME)
@ -388,7 +379,6 @@ class WritePRD(Action):
logger.info(sas.result)
logger.info(rsp)
<<<<<<< HEAD
# logger.info(format)
prompt_template, format_example = get_template(templates, format)
project_name = CONFIG.project_name if CONFIG.project_name else ""
@ -447,7 +437,7 @@ class WritePRD(Action):
if not quadrant_chart:
return
pathname = (
CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("")
CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("")
)
if not pathname.parent.exists():
pathname.parent.mkdir(parents=True, exist_ok=True)
@ -480,33 +470,3 @@ class WritePRD(Action):
if "YES" in res:
return True
return False
=======
prompt = PROMPT_TEMPLATE.format(
requirements=requirements, search_information=info, format_example=FORMAT_EXAMPLE
)
logger.debug(prompt)
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING)
await self._save(prd.content)
return prd
async def _save_prd(self, docs_path, resources_path, prd):
prd_file = docs_path / "prd.md"
quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd)
await mermaid_to_file(
mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / "competitive_analysis"
)
async with aiofiles.open(prd_file, "w") as f:
await f.write(prd)
logger.info(f"Saving PRD to {prd_file}")
async def _save(self, prd):
workspace = CONFIG.workspace
workspace.mkdir(parents=True, exist_ok=True)
docs_path = workspace / "docs"
resources_path = workspace / "resources"
docs_path.mkdir(parents=True, exist_ok=True)
resources_path.mkdir(parents=True, exist_ok=True)
await self._save_prd(docs_path, resources_path, prd)
>>>>>>> send18/dev

View file

@ -5,9 +5,10 @@
@Author : mashenquan
@File : write_teaching_plan.py
"""
from metagpt.logs import logger
from metagpt.actions import Action
from metagpt.logs import logger
from metagpt.schema import Message
from metagpt.utils.common import format_value
class TeachingPlanRequirement(Action):
@ -40,17 +41,18 @@ class WriteTeachingPlanPart(Action):
statement_patterns = self.TOPIC_STATEMENTS.get(self.topic, [])
statements = []
from metagpt.roles import Role
for p in statement_patterns:
s = Role.format_value(p)
s = format_value(p)
statements.append(s)
formatter = self.PROMPT_TITLE_TEMPLATE if self.topic == self.COURSE_TITLE else self.PROMPT_TEMPLATE
prompt = formatter.format(formation=self.FORMATION,
role=self.prefix,
statements="\n".join(statements),
lesson=messages[0].content,
topic=self.topic,
language=self.language)
prompt = formatter.format(
formation=self.FORMATION,
role=self.prefix,
statements="\n".join(statements),
lesson=messages[0].content,
topic=self.topic,
language=self.language,
)
logger.debug(prompt)
rsp = await self._aask(prompt=prompt)
@ -61,14 +63,14 @@ class WriteTeachingPlanPart(Action):
def _set_result(self, rsp):
if self.DATA_BEGIN_TAG in rsp:
ix = rsp.index(self.DATA_BEGIN_TAG)
rsp = rsp[ix + len(self.DATA_BEGIN_TAG):]
rsp = rsp[ix + len(self.DATA_BEGIN_TAG) :]
if self.DATA_END_TAG in rsp:
ix = rsp.index(self.DATA_END_TAG)
rsp = rsp[0:ix]
self.rsp = rsp.strip()
if self.topic != self.COURSE_TITLE:
return
if '#' not in self.rsp or self.rsp.index('#') != 0:
if "#" not in self.rsp or self.rsp.index("#") != 0:
self.rsp = "# " + self.rsp
def __str__(self):
@ -79,81 +81,102 @@ class WriteTeachingPlanPart(Action):
"""Show `topic` value when debug"""
return self.topic
FORMATION = "\"Capacity and role\" defines the role you are currently playing;\n" \
"\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n" \
"\t\"Statement\" defines the work detail you need to complete at this stage;\n" \
"\t\"Answer options\" defines the format requirements for your responses;\n" \
"\t\"Constraint\" defines the conditions that your responses must comply with."
FORMATION = (
'"Capacity and role" defines the role you are currently playing;\n'
'\t"[LESSON_BEGIN]" and "[LESSON_END]" tags enclose the content of textbook;\n'
'\t"Statement" defines the work detail you need to complete at this stage;\n'
'\t"Answer options" defines the format requirements for your responses;\n'
'\t"Constraint" defines the conditions that your responses must comply with.'
)
COURSE_TITLE = "Title"
TOPICS = [
COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content",
"Teaching Methods and Strategies", "Learning Activities",
"Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement",
"Vocabulary Cloze", "Choice Questions", "Grammar Questions", "Translation Questions"
COURSE_TITLE,
"Teaching Hours",
"Teaching Objectives",
"Teaching Content",
"Teaching Methods and Strategies",
"Learning Activities",
"Teaching Time Allocation",
"Assessment and Feedback",
"Teaching Summary and Improvement",
"Vocabulary Cloze",
"Choice Questions",
"Grammar Questions",
"Translation Questions",
]
TOPIC_STATEMENTS = {
COURSE_TITLE: ["Statement: Find and return the title of the lesson only in markdown first-level header format, "
"without anything else."],
COURSE_TITLE: [
"Statement: Find and return the title of the lesson only in markdown first-level header format, "
"without anything else."
],
"Teaching Content": [
"Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar "
'Statement: "Teaching Content" must include vocabulary, analysis, and examples of various grammar '
"structures that appear in the textbook, as well as the listening materials and key points.",
"Statement: \"Teaching Content\" must include more examples."],
'Statement: "Teaching Content" must include more examples.',
],
"Teaching Time Allocation": [
"Statement: \"Teaching Time Allocation\" must include how much time is allocated to each "
"part of the textbook content."],
'Statement: "Teaching Time Allocation" must include how much time is allocated to each '
"part of the textbook content."
],
"Teaching Methods and Strategies": [
"Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, "
'Statement: "Teaching Methods and Strategies" must include teaching focus, difficulties, materials, '
"procedures, in detail."
],
"Vocabulary Cloze": [
"Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", "
'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", '
"create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} "
"answers, and it should also include 10 {teaching_language} questions with {language} answers. "
"The key-related vocabulary and phrases in the textbook content must all be included in the exercises.",
],
"Grammar Questions": [
"Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", "
"create grammar questions. 10 questions."],
'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", '
"create grammar questions. 10 questions."
],
"Choice Questions": [
"Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", "
"create choice questions. 10 questions."],
'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", '
"create choice questions. 10 questions."
],
"Translation Questions": [
"Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", "
'Statement: Based on the content of the textbook enclosed by "[LESSON_BEGIN]" and "[LESSON_END]", '
"create translation questions. The translation should include 10 {language} questions with "
"{teaching_language} answers, and it should also include 10 {teaching_language} questions with "
"{language} answers."
]
],
}
# Teaching plan title
PROMPT_TITLE_TEMPLATE = "Do not refer to the context of the previous conversation records, " \
"start the conversation anew.\n\n" \
"Formation: {formation}\n\n" \
"{statements}\n" \
"Constraint: Writing in {language}.\n" \
"Answer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" " \
"and \"[TEACHING_PLAN_END]\" tags.\n" \
"[LESSON_BEGIN]\n" \
"{lesson}\n" \
"[LESSON_END]"
PROMPT_TITLE_TEMPLATE = (
"Do not refer to the context of the previous conversation records, "
"start the conversation anew.\n\n"
"Formation: {formation}\n\n"
"{statements}\n"
"Constraint: Writing in {language}.\n"
'Answer options: Encloses the lesson title with "[TEACHING_PLAN_BEGIN]" '
'and "[TEACHING_PLAN_END]" tags.\n'
"[LESSON_BEGIN]\n"
"{lesson}\n"
"[LESSON_END]"
)
# Teaching plan parts:
PROMPT_TEMPLATE = "Do not refer to the context of the previous conversation records, " \
"start the conversation anew.\n\n" \
"Formation: {formation}\n\n" \
"Capacity and role: {role}\n" \
"Statement: Write the \"{topic}\" part of teaching plan, " \
"WITHOUT ANY content unrelated to \"{topic}\"!!\n" \
"{statements}\n" \
"Answer options: Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" " \
"and \"[TEACHING_PLAN_END]\" tags.\n" \
"Answer options: Using proper markdown format from second-level header format.\n" \
"Constraint: Writing in {language}.\n" \
"[LESSON_BEGIN]\n" \
"{lesson}\n" \
"[LESSON_END]"
PROMPT_TEMPLATE = (
"Do not refer to the context of the previous conversation records, "
"start the conversation anew.\n\n"
"Formation: {formation}\n\n"
"Capacity and role: {role}\n"
'Statement: Write the "{topic}" part of teaching plan, '
'WITHOUT ANY content unrelated to "{topic}"!!\n'
"{statements}\n"
'Answer options: Enclose the teaching plan content with "[TEACHING_PLAN_BEGIN]" '
'and "[TEACHING_PLAN_END]" tags.\n'
"Answer options: Using proper markdown format from second-level header format.\n"
"Constraint: Writing in {language}.\n"
"[LESSON_BEGIN]\n"
"{lesson}\n"
"[LESSON_END]"
)
DATA_BEGIN_TAG = "[TEACHING_PLAN_BEGIN]"
DATA_END_TAG = "[TEACHING_PLAN_END]"