Merge branch 'feat-cr' into 'mgx_ops'

Feat cr

See merge request pub/MetaGPT!226

metagpt/environment/mgx/mgx_env.py

@@ -27,7 +27,7 @@ class MGXEnv(Environment):
 
     def publish_message(self, message: Message, user_defined_recipient: str = "", publicer: str = "") -> bool:
         """let the team leader take over message publishing"""
-        tl = self.get_role("Tim")  # TeamLeader's name is Tim
+        tl = self.get_role("Mike")  # TeamLeader's name is Mike
 
         if user_defined_recipient:
             # human user's direct chat message to a certain role

metagpt/ext/cr/actions/code_review.py

@@ -57,7 +57,7 @@ Just print the PR Patch comments in json format like **Output Format**.
 """
 
 CODE_REVIEW_COMFIRM_SYSTEM_PROMPT = """
-You are a professional engineer with Java stack, and good at code review comment result judgement.
+You are a professional engineer with {code_language} stack, and good at code review comment result judgement.
 """
 
 CODE_REVIEW_COMFIRM_TEMPLATE = """
@@ -132,13 +132,20 @@ class CodeReview(Action):
                 code = get_code_block_from_patch(
                     patch, str(max(1, int(code_start_line) - 5)), str(int(code_end_line) + 5)
                 )
+            code_language = "Java"
+            code_file_ext = cmt.get("commented_file", ".java").split(".")[-1]
+            if code_file_ext == ".java":
+                code_language = "Java"
+            elif code_file_ext == ".py":
+                code_language = "Python"
             prompt = CODE_REVIEW_COMFIRM_TEMPLATE.format(
                 code=code,
                 comment=cmt.get("comment"),
                 desc=point.text,
                 example=point.yes_example + "\n" + point.no_example,
             )
-            resp = await self.llm.aask(prompt, system_msgs=[CODE_REVIEW_COMFIRM_SYSTEM_PROMPT])
+            system_prompt = [CODE_REVIEW_COMFIRM_SYSTEM_PROMPT.format(code_language=code_language)]
+            resp = await self.llm.aask(prompt, system_msgs=system_prompt)
             if "True" in resp or "true" in resp:
                 new_comments.append(cmt)
         logger.info(f"original comments num: {len(comments)}, confirmed comments num: {len(new_comments)}")
@@ -163,7 +170,11 @@ class CodeReview(Action):
             prompt = CODE_REVIEW_PROMPT_TEMPLATE.format(patch=str(patched_file), points=points_str)
             resp = await self.llm.aask(prompt)
             json_str = parse_json_code_block(resp)[0]
-            comments += json.loads(json_str)
+            comment = json.loads(json_str)
+            patched_file_path = patched_file.path
+            for c in comment:
+                c["commented_file"] = patched_file_path
+            comments += comment
 
         return comments
 

metagpt/ext/cr/actions/modify_code.py

@@ -81,17 +81,18 @@ class ModifyCode(Action):
         }
         resp = None
         for patched_file in patch:
-            patch_target_file_name = str(patched_file.target_file).split("/", maxsplit=1)[-1]
-            if patch_target_file_name not in grouped_comments:
+            patch_target_file_name = str(patched_file.path).split("/")[-1]
+            if patched_file.path not in grouped_comments:
                 continue
             comments_prompt = ""
             index = 1
-            for grouped_comment in grouped_comments[patch_target_file_name]:
+            for grouped_comment in grouped_comments[patched_file.path]:
                 comments_prompt += f"""
                     <comment{index}>
                     {grouped_comment}
                     </comment{index}>\n
                 """
+                index += 1
             prompt = MODIFY_CODE_PROMPT.format(patch=patched_file, comments=comments_prompt)
             output_dir = (
                 Path(output_dir)

metagpt/tools/libs/browser.py

@@ -24,6 +24,7 @@ from metagpt.utils.a11y_tree import (
     scroll_page,
     type_text,
 )
+from metagpt.utils.proxy_env import get_proxy_from_env
 from metagpt.utils.report import BrowserReporter
 
 
@@ -72,7 +73,7 @@ class Browser:
         self.page: Optional[Page] = None
         self.accessibility_tree: list = []
         self.headless: bool = True
-        self.proxy = None
+        self.proxy = get_proxy_from_env()
         self.is_empty_page = True
         self.reporter = BrowserReporter()
 
@@ -120,7 +121,7 @@ class Browser:
         await scroll_page(self.page, direction)
         return await self._wait_page()
 
-    async def goto(self, url: str, timeout: float = 30000):
+    async def goto(self, url: str, timeout: float = 90000):
         """Navigate to a specific URL."""
         if self.page is None:
             await self.start()

metagpt/tools/libs/cr.py

@@ -3,6 +3,7 @@ from pathlib import Path
 from typing import Optional
 
 import aiofiles
+from bs4 import BeautifulSoup
 from unidiff import PatchSet
 
 import metagpt.ext.cr
@@ -29,7 +30,7 @@ class CodeReview:
         Args:
             patch_path: The local path of the patch file or the url of the pull request. Example: "/data/xxx-pr-1.patch", "https://github.com/xx/XX/pull/1362"
             cr_output_file: Output file path where code review comments will be saved. Example: "cr/xxx-pr-1.json"
-            cr_point_file: File path for specifying code review points. Defaults to a predefined file.
+            cr_point_file: File path for specifying code review points. Set `None` to use a predefined file.
         """
         patch = await self._get_patch_content(patch_path)
         cr_point_file = cr_point_file if cr_point_file else Path(metagpt.ext.cr.__file__).parent / "points.json"
@@ -45,7 +46,7 @@ class CodeReview:
             )
             comments = await CodeReview_().run(patch, cr_points)
             cr_output_path.parent.mkdir(exist_ok=True, parents=True)
-            async with aiofiles.open(cr_output_path, "w") as f:
+            async with aiofiles.open(cr_output_path, "w", encoding="utf-8") as f:
                 await f.write(json.dumps(comments, ensure_ascii=False))
             await reporter.async_report(cr_output_path)
 
@@ -65,7 +66,7 @@ class CodeReview:
             output_dir: File path where code review comments are stored.
         """
         patch = await self._get_patch_content(patch_path)
-        async with aiofiles.open(cr_file, "r") as f:
+        async with aiofiles.open(cr_file, "r", encoding="utf-8") as f:
             comments = json.loads(await f.read())
         await ModifyCode(pr="").run(patch, comments, output_dir)
         return f"The fixed patch files store in {output_dir}"
@@ -75,12 +76,14 @@ class CodeReview:
             # async with aiohttp.ClientSession(trust_env=True) as client:
             #     async with client.get(f"{patch_path}.diff", ) as resp:
             #         patch_file_content = await resp.text()
-            browser = Browser()
-            browser.proxy = {"server": "http://127.0.0.1:20172"}
-            async with browser:
+            async with Browser() as browser:
                 await browser.goto(f"{patch_path}.diff")
                 patch_file_content = await browser.page.content()
-
+                if patch_file_content.startswith("<html>"):
+                    soup = BeautifulSoup(patch_file_content, "html.parser")
+                    pre = soup.find("pre")
+                    if pre:
+                        patch_file_content = pre.text
         else:
             async with aiofiles.open(patch_path) as f:
                 patch_file_content = await f.read()

metagpt/tools/libs/terminal.py

@@ -115,8 +115,9 @@ class Terminal:
             # '\r' is changed to '\n', resulting in excessive output.
             tmp = b""
             while True:
-                self.process.communicate()
                 output = tmp + await self.process.stdout.read(1)
+                if not output:
+                    continue
                 *lines, tmp = output.splitlines(True)
                 for line in lines:
                     line = line.decode()

metagpt/utils/proxy_env.py

@@ -0,0 +1,19 @@
+import os
+
+
+def get_proxy_from_env():
+    proxy_config = {}
+    server = None
+    for i in ("ALL_PROXY", "all_proxy", "HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"):
+        if os.environ.get(i):
+            server = os.environ.get(i)
+    if server:
+        proxy_config["server"] = server
+    no_proxy = os.environ.get("NO_PROXY") or os.environ.get("no_proxy")
+    if no_proxy:
+        proxy_config["bypass"] = no_proxy
+
+    if not proxy_config:
+        proxy_config = None
+
+    return proxy_config

metagpt/utils/token_counter.py

@@ -31,9 +31,9 @@ TOKEN_COSTS = {
     "gpt-4-0125-preview": {"prompt": 0.01, "completion": 0.03},
     "gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03},
     "gpt-4-vision-preview": {"prompt": 0.01, "completion": 0.03},  # TODO add extra image price calculator
-    "gpt-4-1106-vision-preview": {"prompt": 0.01, "completion": 0.03},
     "gpt-4o": {"prompt": 0.005, "completion": 0.015},
     "gpt-4o-2024-05-13": {"prompt": 0.005, "completion": 0.015},
+    "gpt-4-1106-vision-preview": {"prompt": 0.01, "completion": 0.03},
     "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0},
     "glm-3-turbo": {"prompt": 0.0007, "completion": 0.0007},  # 128k version, prompt + completion tokens=0.005￥/k-tokens
     "glm-4": {"prompt": 0.014, "completion": 0.014},  # 128k version, prompt + completion tokens=0.1￥/k-tokens
@@ -147,6 +147,8 @@ FIREWORKS_GRADE_TOKEN_COSTS = {
 
 # https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
 TOKEN_MAX = {
+    "gpt-4o-2024-05-13": 128000,
+    "gpt-4o": 128000,
     "gpt-4-0125-preview": 128000,
     "gpt-4-turbo-preview": 128000,
     "gpt-4-1106-preview": 128000,