From caa42ff7c17801ffec119c6b7be1947b2ec0bf65 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Wed, 8 May 2024 15:12:40 +0800 Subject: [PATCH 1/3] terminal stream opt --- metagpt/tools/libs/terminal.py | 38 +++++++++++++++++----------------- metagpt/utils/report.py | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index 5f5989e1a..85aacbef8 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -26,8 +26,6 @@ class Terminal: stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True, - bufsize=1, # Line buffered executable="/bin/bash", ) self.stdout_queue = Queue() @@ -58,9 +56,9 @@ class Terminal: """ # Send the command - self.process.stdin.write(cmd + self.command_terminator) + self.process.stdin.write((cmd + self.command_terminator).encode()) self.process.stdin.write( - f'echo "{END_MARKER_VALUE}"' + self.command_terminator # write EOF + (f'echo "{END_MARKER_VALUE}"{self.command_terminator}').encode() # write EOF ) # Unique marker to signal command end self.process.stdin.flush() if daemon: @@ -100,22 +98,24 @@ class Terminal: # report the command # Read the output until the unique marker is found + tmp = b"" while True: - line = self.process.stdout.readline() - ix = line.rfind(END_MARKER_VALUE) - if ix >= 0: - line = line[0:ix] - if line: - observer.report(line, "output") - # report stdout in real-time - cmd_output.append(line) - break - # log stdout in real-time - observer.report(line, "output") - cmd_output.append(line) - self.stdout_queue.put(line) - - return "".join(cmd_output) + output = tmp + self.process.stdout.read(1) + *lines, tmp = output.splitlines(True) + for line in lines: + line = line.decode() + ix = line.rfind(END_MARKER_VALUE) + if ix >= 0: + line = line[0:ix] + if line: + observer.report(line, "output") + # report stdout in real-time + cmd_output.append(line) + return "".join(cmd_output) + # log stdout in real-time + observer.report(line, "output") + cmd_output.append(line) + self.stdout_queue.put(line) def close(self): """Close the persistent shell process.""" diff --git a/metagpt/utils/report.py b/metagpt/utils/report.py index 20f9ccd01..a61c77381 100644 --- a/metagpt/utils/report.py +++ b/metagpt/utils/report.py @@ -42,7 +42,7 @@ class BlockType(str, Enum): END_MARKER_NAME = "end_marker" -END_MARKER_VALUE = "\x18\x19\x1B\x18" +END_MARKER_VALUE = "\x18\x19\x1B\x18\n" class ResourceReporter(BaseModel): From 32d6e5a1cf1416ff43354c308deaf9ae36392c3d Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Wed, 8 May 2024 15:13:13 +0800 Subject: [PATCH 2/3] add qa/mermaid reporter --- metagpt/actions/design_api.py | 5 ++++- metagpt/actions/write_code.py | 2 +- metagpt/actions/write_prd.py | 5 ++++- metagpt/environment/mgx/mgx_env.py | 10 +++++++++- metagpt/roles/qa_engineer.py | 12 +++++++++--- metagpt/schema.py | 2 +- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index b71c66543..83139035a 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -26,7 +26,7 @@ from metagpt.const import DATA_API_DESIGN_FILE_REPO, SEQ_FLOW_FILE_REPO from metagpt.logs import logger from metagpt.schema import Document, Documents, Message from metagpt.utils.mermaid import mermaid_to_file -from metagpt.utils.report import DocsReporter +from metagpt.utils.report import DocsReporter, GalleryReporter NEW_REQ_TEMPLATE = """ ### Legacy Content @@ -122,3 +122,6 @@ class WriteDesign(Action): async def _save_mermaid_file(self, data: str, pathname: Path): pathname.parent.mkdir(parents=True, exist_ok=True) await mermaid_to_file(self.config.mermaid.engine, data, pathname) + image_path = pathname.parent / f"{pathname.name}.png" + if image_path.exists(): + await GalleryReporter().async_report(image_path, "path") diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 6cfde2385..029a290fa 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -141,7 +141,7 @@ class WriteCode(Action): ) logger.info(f"Writing {coding_context.filename}..") async with EditorReporter(enable_llm_stream=True) as reporter: - await reporter.async_report({"filename": coding_context.filename}, "meta") + await reporter.async_report({"type": "code", "filename": coding_context.filename}, "meta") code = await self.write_code(prompt) if not coding_context.code_doc: # avoid root_path pydantic ValidationError if use WriteCode alone diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 3ba7e9543..02bd1d6d5 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -37,7 +37,7 @@ from metagpt.schema import BugFixContext, Document, Documents, Message from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.mermaid import mermaid_to_file -from metagpt.utils.report import DocsReporter +from metagpt.utils.report import DocsReporter, GalleryReporter CONTEXT_TEMPLATE = """ ### Project Name @@ -169,6 +169,9 @@ class WritePRD(Action): pathname = self.repo.workdir / COMPETITIVE_ANALYSIS_FILE_REPO / Path(prd_doc.filename).stem pathname.parent.mkdir(parents=True, exist_ok=True) await mermaid_to_file(self.config.mermaid.engine, quadrant_chart, pathname) + image_path = pathname.parent / f"{pathname.name}.png" + if image_path.exists(): + await GalleryReporter().async_report(image_path, "path") async def _rename_workspace(self, prd): if not self.project_name: diff --git a/metagpt/environment/mgx/mgx_env.py b/metagpt/environment/mgx/mgx_env.py index c4ed8c4f1..107f449a6 100644 --- a/metagpt/environment/mgx/mgx_env.py +++ b/metagpt/environment/mgx/mgx_env.py @@ -52,7 +52,15 @@ class MGXEnv(Environment): self._publish_message(message) if self.is_software_task_finished(message): tl.rc.memory.add(self.move_message_info_to_content(message)) - tl.finish_current_task() + from metagpt.utils.report import CURRENT_ROLE + + role = CURRENT_ROLE.get(None) + if role: + CURRENT_ROLE.set(tl) + tl.finish_current_task() + CURRENT_ROLE.set(role) + else: + tl.finish_current_task() elif publicer == tl.profile: if message.send_to == {"no one"}: diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index ed9c455a6..c5f22174f 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -21,6 +21,7 @@ from metagpt.actions.summarize_code import SummarizeCode from metagpt.const import MESSAGE_ROUTE_TO_NONE from metagpt.logs import logger from metagpt.roles import Role +from metagpt.utils.report import EditorReporter from metagpt.schema import AIMessage, Document, Message, RunCodeContext, TestingContext from metagpt.utils.common import ( any_to_str, @@ -75,10 +76,15 @@ class QaEngineer(Role): ) logger.info(f"Writing {test_doc.filename}..") context = TestingContext(filename=test_doc.filename, test_doc=test_doc, code_doc=code_doc) + context = await WriteTest(i_context=context, context=self.context, llm=self.llm).run() - await self.project_repo.tests.save_doc( - doc=context.test_doc, dependencies={context.code_doc.root_relative_path} - ) + async with EditorReporter(enable_llm_stream=True) as reporter: + await reporter.async_report({"type": "test", "filename": test_doc.filename}, "meta") + + doc = await self.project_repo.tests.save_doc( + doc=context.test_doc, dependencies={context.code_doc.root_relative_path} + ) + await reporter.async_report(self.project_repo.workdir / doc.root_relative_path, "path") # prepare context for run tests in next round run_code_context = RunCodeContext( diff --git a/metagpt/schema.py b/metagpt/schema.py index cfe991cd9..5fc1ae242 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -579,7 +579,7 @@ class Plan(BaseModel): current_task_id = task.task_id break self.current_task_id = current_task_id - TaskReporter().report({"tasks": self.tasks, "current_task_id": current_task_id}) + TaskReporter().report({"tasks": [i.model_dump() for i in self.tasks], "current_task_id": current_task_id}) @property def current_task(self) -> Task: From e72246e0df544b406fcdab4a2ad1f2a7c994be7d Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Wed, 8 May 2024 16:30:00 +0800 Subject: [PATCH 3/3] add comment for terminal tool output parser --- metagpt/tools/libs/terminal.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/tools/libs/terminal.py b/metagpt/tools/libs/terminal.py index 85aacbef8..68bd95901 100644 --- a/metagpt/tools/libs/terminal.py +++ b/metagpt/tools/libs/terminal.py @@ -95,9 +95,10 @@ class Terminal: with self.observer as observer: cmd_output = [] observer.report(cmd + self.command_terminator, "cmd") - # report the command - - # Read the output until the unique marker is found + # report the comman + # Read the output until the unique marker is found. + # We read bytes directly from stdout instead of text because when reading text, + # '\r' is changed to '\n', resulting in excessive output. tmp = b"" while True: output = tmp + self.process.stdout.read(1)