From ff5e7deb215d0a5f4014808c12e527e74d7889b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 24 Jan 2024 10:52:30 +0800 Subject: [PATCH 1/7] add strip for result. --- metagpt/tools/libs/web_scraping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/tools/libs/web_scraping.py b/metagpt/tools/libs/web_scraping.py index e8e73f123..921fca809 100644 --- a/metagpt/tools/libs/web_scraping.py +++ b/metagpt/tools/libs/web_scraping.py @@ -19,4 +19,4 @@ async def scrape_web_playwright(url, *urls): web = await PlaywrightWrapper("chromium").run(url, *urls) # Return the inner text content of the web page - return {"inner_text": web.inner_text, "html": web.html} + return {"inner_text": web.inner_text.strip(), "html": web.html.strip()} From dfe49a3312ae457e6e2de51de25fdfaf42a99418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 24 Jan 2024 10:53:03 +0800 Subject: [PATCH 2/7] update return value. --- metagpt/roles/code_interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/code_interpreter.py b/metagpt/roles/code_interpreter.py index 3991862d1..b1526cd95 100644 --- a/metagpt/roles/code_interpreter.py +++ b/metagpt/roles/code_interpreter.py @@ -97,7 +97,7 @@ class CodeInterpreter(Role): if ReviewConst.CHANGE_WORD[0] in review: counter = 0 # redo the task again with help of human suggestions - return code["code"] if code["language"] != "markdown" else "", result, success + return code["code"] if code.get("language", None) != "markdown" else "", result, success async def _write_code(self): todo = WriteCodeByGenerate() if not self.use_tools else WriteCodeWithTools(selected_tools=self.tools) From 3f2b512d297e166b762c2af291096b9e3c21486f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 24 Jan 2024 10:53:37 +0800 Subject: [PATCH 3/7] new file: tests/metagpt/tools/libs/test_web_scraping.py --- tests/metagpt/tools/libs/test_web_scraping.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/metagpt/tools/libs/test_web_scraping.py diff --git a/tests/metagpt/tools/libs/test_web_scraping.py b/tests/metagpt/tools/libs/test_web_scraping.py new file mode 100644 index 000000000..c11960e68 --- /dev/null +++ b/tests/metagpt/tools/libs/test_web_scraping.py @@ -0,0 +1,23 @@ +import pytest + +from metagpt.tools.libs.web_scraping import scrape_web_playwright + + +@pytest.mark.asyncio +async def test_scrape_web_playwright(): + test_url = "https://www.deepwisdom.ai" + + result = await scrape_web_playwright(test_url) + + # Assert that the result is a dictionary + assert isinstance(result, dict) + + # Assert that the result contains 'inner_text' and 'html' keys + assert "inner_text" in result + assert "html" in result + + # Assert startswith and endswith + assert not result["inner_text"].startswith(" ") + assert not result["inner_text"].endswith(" ") + assert not result["html"].startswith(" ") + assert not result["html"].endswith(" ") From 0c8a844f5a2e4a7f2e93584bb64ad3bedae64c97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 24 Jan 2024 12:06:07 +0800 Subject: [PATCH 4/7] add strip for result. --- metagpt/actions/execute_code.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/execute_code.py b/metagpt/actions/execute_code.py index 6591f479f..6a4a9abb8 100644 --- a/metagpt/actions/execute_code.py +++ b/metagpt/actions/execute_code.py @@ -123,7 +123,10 @@ class ExecutePyCode(ExecuteCode, Action): return parsed_output for i, output in enumerate(outputs): - if output["output_type"] == "stream": + if output["output_type"] == "stream" and not any( + tag in output["text"] + for tag in ["| INFO | metagpt", "| ERROR | metagpt", "| WARNING | metagpt"] + ): parsed_output += output["text"] elif output["output_type"] == "display_data": if "image/png" in output["data"]: From 0353f36f0d9c04452a421e702ee984fbabc973c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 24 Jan 2024 15:21:32 +0800 Subject: [PATCH 5/7] new file: examples/crawle_webpage.py --- examples/crawle_webpage.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 examples/crawle_webpage.py diff --git a/examples/crawle_webpage.py b/examples/crawle_webpage.py new file mode 100644 index 000000000..2c616035f --- /dev/null +++ b/examples/crawle_webpage.py @@ -0,0 +1,22 @@ +# -*- encoding: utf-8 -*- +""" +@Date : 2024/01/24 15:11:27 +@Author : orange-crow +@File : crawle_webpage.py +""" + +from metagpt.roles.code_interpreter import CodeInterpreter + + +async def main(): + prompt = """Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/, + and save it to a csv file. paper title must include `multiagent` or `large language model`. *notice: print key data*""" + ci = CodeInterpreter(goal=prompt, use_tools=True) + + await ci.run(prompt) + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) From 526025bbe3e88d51b6bcc7cc97aa87f3549871fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Thu, 25 Jan 2024 15:01:05 +0800 Subject: [PATCH 6/7] change file name: crawle_webpage.py -> crawl_webpage.py --- examples/{crawle_webpage.py => crawl_webpage.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename examples/{crawle_webpage.py => crawl_webpage.py} (94%) diff --git a/examples/crawle_webpage.py b/examples/crawl_webpage.py similarity index 94% rename from examples/crawle_webpage.py rename to examples/crawl_webpage.py index 2c616035f..35413d2ff 100644 --- a/examples/crawle_webpage.py +++ b/examples/crawl_webpage.py @@ -2,7 +2,7 @@ """ @Date : 2024/01/24 15:11:27 @Author : orange-crow -@File : crawle_webpage.py +@File : crawl_webpage.py """ from metagpt.roles.code_interpreter import CodeInterpreter From 54a08747db5b16fb7680556b9051466cf121d3f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Thu, 25 Jan 2024 15:02:52 +0800 Subject: [PATCH 7/7] chore --- metagpt/roles/code_interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/code_interpreter.py b/metagpt/roles/code_interpreter.py index b1526cd95..7a4ced59b 100644 --- a/metagpt/roles/code_interpreter.py +++ b/metagpt/roles/code_interpreter.py @@ -97,7 +97,7 @@ class CodeInterpreter(Role): if ReviewConst.CHANGE_WORD[0] in review: counter = 0 # redo the task again with help of human suggestions - return code["code"] if code.get("language", None) != "markdown" else "", result, success + return code["code"] if code.get("language") != "markdown" else "", result, success async def _write_code(self): todo = WriteCodeByGenerate() if not self.use_tools else WriteCodeWithTools(selected_tools=self.tools)