Merge branch 'feat-reporter' into 'mgx_ops'

Feat reporter

See merge request pub/MetaGPT!91

metagpt/actions/design_api.py

@@ -26,7 +26,7 @@ from metagpt.const import DATA_API_DESIGN_FILE_REPO, SEQ_FLOW_FILE_REPO
 from metagpt.logs import logger
 from metagpt.schema import Document, Documents, Message
 from metagpt.utils.mermaid import mermaid_to_file
-from metagpt.utils.report import DocsReporter
+from metagpt.utils.report import DocsReporter, GalleryReporter
 
 NEW_REQ_TEMPLATE = """
 ### Legacy Content
@@ -122,3 +122,6 @@ class WriteDesign(Action):
     async def _save_mermaid_file(self, data: str, pathname: Path):
         pathname.parent.mkdir(parents=True, exist_ok=True)
         await mermaid_to_file(self.config.mermaid.engine, data, pathname)
+        image_path = pathname.parent / f"{pathname.name}.png"
+        if image_path.exists():
+            await GalleryReporter().async_report(image_path, "path")

metagpt/actions/write_code.py

@@ -146,7 +146,7 @@ class WriteCode(Action):
             )
         logger.info(f"Writing {coding_context.filename}..")
         async with EditorReporter(enable_llm_stream=True) as reporter:
-            await reporter.async_report({"filename": coding_context.filename}, "meta")
+            await reporter.async_report({"type": "code", "filename": coding_context.filename}, "meta")
             code = await self.write_code(prompt)
             if not coding_context.code_doc:
                 # avoid root_path pydantic ValidationError if use WriteCode alone

metagpt/actions/write_prd.py

@@ -37,7 +37,7 @@ from metagpt.schema import BugFixContext, Document, Documents, Message
 from metagpt.utils.common import CodeParser
 from metagpt.utils.file_repository import FileRepository
 from metagpt.utils.mermaid import mermaid_to_file
-from metagpt.utils.report import DocsReporter
+from metagpt.utils.report import DocsReporter, GalleryReporter
 
 CONTEXT_TEMPLATE = """
 ### Project Name
@@ -169,6 +169,9 @@ class WritePRD(Action):
         pathname = self.repo.workdir / COMPETITIVE_ANALYSIS_FILE_REPO / Path(prd_doc.filename).stem
         pathname.parent.mkdir(parents=True, exist_ok=True)
         await mermaid_to_file(self.config.mermaid.engine, quadrant_chart, pathname)
+        image_path = pathname.parent / f"{pathname.name}.png"
+        if image_path.exists():
+            await GalleryReporter().async_report(image_path, "path")
 
     async def _rename_workspace(self, prd):
         if not self.project_name:

metagpt/environment/mgx/mgx_env.py

@@ -52,7 +52,15 @@ class MGXEnv(Environment):
             self._publish_message(message)
             if self.is_software_task_finished(message):
                 tl.rc.memory.add(self.move_message_info_to_content(message))
-                tl.finish_current_task()
+                from metagpt.utils.report import CURRENT_ROLE
+
+                role = CURRENT_ROLE.get(None)
+                if role:
+                    CURRENT_ROLE.set(tl)
+                    tl.finish_current_task()
+                    CURRENT_ROLE.set(role)
+                else:
+                    tl.finish_current_task()
 
         elif publicer == tl.profile:
             if message.send_to == {"no one"}:

metagpt/roles/qa_engineer.py

@@ -21,6 +21,7 @@ from metagpt.actions.summarize_code import SummarizeCode
 from metagpt.const import MESSAGE_ROUTE_TO_NONE
 from metagpt.logs import logger
 from metagpt.roles import Role
+from metagpt.utils.report import EditorReporter
 from metagpt.schema import AIMessage, Document, Message, RunCodeContext, TestingContext
 from metagpt.utils.common import (
     any_to_str,
@@ -75,10 +76,15 @@ class QaEngineer(Role):
                 )
             logger.info(f"Writing {test_doc.filename}..")
             context = TestingContext(filename=test_doc.filename, test_doc=test_doc, code_doc=code_doc)
+
             context = await WriteTest(i_context=context, context=self.context, llm=self.llm).run()
-            await self.project_repo.tests.save_doc(
-                doc=context.test_doc, dependencies={context.code_doc.root_relative_path}
-            )
+            async with EditorReporter(enable_llm_stream=True) as reporter:
+                await reporter.async_report({"type": "test", "filename": test_doc.filename}, "meta")
+
+                doc = await self.project_repo.tests.save_doc(
+                    doc=context.test_doc, dependencies={context.code_doc.root_relative_path}
+                )
+                await reporter.async_report(self.project_repo.workdir / doc.root_relative_path, "path")
 
             # prepare context for run tests in next round
             run_code_context = RunCodeContext(

metagpt/schema.py

@@ -579,7 +579,7 @@ class Plan(BaseModel):
                 current_task_id = task.task_id
                 break
         self.current_task_id = current_task_id
-        TaskReporter().report({"tasks": self.tasks, "current_task_id": current_task_id})
+        TaskReporter().report({"tasks": [i.model_dump() for i in self.tasks], "current_task_id": current_task_id})
 
     @property
     def current_task(self) -> Task:

metagpt/tools/libs/terminal.py

@@ -26,8 +26,6 @@ class Terminal:
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
-            text=True,
-            bufsize=1,  # Line buffered
             executable="/bin/bash",
         )
         self.stdout_queue = Queue()
@@ -64,9 +62,9 @@ class Terminal:
         """
 
         # Send the command
-        self.process.stdin.write(cmd + self.command_terminator)
+        self.process.stdin.write((cmd + self.command_terminator).encode())
         self.process.stdin.write(
-            f'echo "{END_MARKER_VALUE}"' + self.command_terminator  # write EOF
+            (f'echo "{END_MARKER_VALUE}"{self.command_terminator}').encode()  # write EOF
         )  # Unique marker to signal command end
         self.process.stdin.flush()
         if daemon:
@@ -103,25 +101,28 @@ class Terminal:
         with self.observer as observer:
             cmd_output = []
             observer.report(cmd + self.command_terminator, "cmd")
-            # report the command
-
-            # Read the output until the unique marker is found
+            # report the comman
+            # Read the output until the unique marker is found.
+            # We read bytes directly from stdout instead of text because when reading text,
+            # '\r' is changed to '\n', resulting in excessive output.
+            tmp = b""
             while True:
-                line = self.process.stdout.readline()
-                ix = line.rfind(END_MARKER_VALUE)
-                if ix >= 0:
-                    line = line[0:ix]
-                    if line:
-                        observer.report(line, "output")
-                        # report stdout in real-time
-                        cmd_output.append(line)
-                    break
-                # log stdout in real-time
-                observer.report(line, "output")
-                cmd_output.append(line)
-                self.stdout_queue.put(line)
-
-        return "".join(cmd_output)
+                output = tmp + self.process.stdout.read(1)
+                *lines, tmp = output.splitlines(True)
+                for line in lines:
+                    line = line.decode()
+                    ix = line.rfind(END_MARKER_VALUE)
+                    if ix >= 0:
+                        line = line[0:ix]
+                        if line:
+                            observer.report(line, "output")
+                            # report stdout in real-time
+                            cmd_output.append(line)
+                        return "".join(cmd_output)
+                    # log stdout in real-time
+                    observer.report(line, "output")
+                    cmd_output.append(line)
+                    self.stdout_queue.put(line)
 
     def close(self):
         """Close the persistent shell process."""

metagpt/utils/report.py

@@ -42,7 +42,7 @@ class BlockType(str, Enum):
 
 
 END_MARKER_NAME = "end_marker"
-END_MARKER_VALUE = "\x18\x19\x1B\x18"
+END_MARKER_VALUE = "\x18\x19\x1B\x18\n"
 
 
 class ResourceReporter(BaseModel):