fixbug: gbk UnicodeEncodeError

tests/metagpt/roles/test_tutorial_assistant.py

@@ -6,11 +6,11 @@
 @File    : test_tutorial_assistant.py
 """
 
-import aiofiles
 import pytest
 
 from metagpt.const import TUTORIAL_PATH
 from metagpt.roles.tutorial_assistant import TutorialAssistant
+from metagpt.utils.common import aread
 
 
 @pytest.mark.asyncio
@@ -20,9 +20,8 @@ async def test_tutorial_assistant(language: str, topic: str, context):
     msg = await role.run(topic)
     assert TUTORIAL_PATH.exists()
     filename = msg.content
-    async with aiofiles.open(filename, mode="r", encoding="utf-8") as reader:
-        content = await reader.read()
-        assert "pip" in content
+    content = await aread(filename=filename)
+    assert "pip" in content
 
 
 if __name__ == "__main__":

tests/metagpt/utils/test_common.py

@@ -13,7 +13,6 @@ import uuid
 from pathlib import Path
 from typing import Any, Set
 
-import aiofiles
 import pytest
 from pydantic import BaseModel
 
@@ -125,9 +124,7 @@ class TestGetProjectRoot:
     async def test_parse_data_exception(self, filename, want):
         pathname = Path(__file__).parent.parent.parent / "data/output_parser" / filename
         assert pathname.exists()
-        async with aiofiles.open(str(pathname), mode="r") as reader:
-            data = await reader.read()
-
+        data = await aread(filename=pathname)
         result = OutputParser.parse_data(data=data)
         assert want in result
 
@@ -198,12 +195,25 @@ class TestGetProjectRoot:
 
     @pytest.mark.asyncio
     async def test_read_write(self):
-        pathname = Path(__file__).parent / uuid.uuid4().hex / "test.tmp"
+        pathname = Path(__file__).parent / f"../../../workspace/unittest/{uuid.uuid4().hex}" / "test.tmp"
         await awrite(pathname, "ABC")
         data = await aread(pathname)
         assert data == "ABC"
         pathname.unlink(missing_ok=True)
 
+    @pytest.mark.asyncio
+    async def test_read_write_error_charset(self):
+        pathname = Path(__file__).parent / f"../../../workspace/unittest/{uuid.uuid4().hex}" / "test.txt"
+        content = "中国abc123\u27f6"
+        await awrite(filename=pathname, data=content)
+        data = await aread(filename=pathname)
+        assert data == content
+
+        content = "GB18030 是中国国家标准局发布的新一代中文字符集标准，是 GBK 的升级版，支持更广泛的字符范围。"
+        await awrite(filename=pathname, data=content, encoding="gb2312")
+        data = await aread(filename=pathname, encoding="utf-8")
+        assert data == content
+
 
 if __name__ == "__main__":
     pytest.main([__file__, "-s"])

tests/metagpt/utils/test_git_repository.py

@@ -10,15 +10,14 @@
 import shutil
 from pathlib import Path
 
-import aiofiles
 import pytest
 
+from metagpt.utils.common import awrite
 from metagpt.utils.git_repository import GitRepository
 
 
 async def mock_file(filename, content=""):
-    async with aiofiles.open(str(filename), mode="w") as file:
-        await file.write(content)
+    await awrite(filename=filename, data=content)
 
 
 async def mock_repo(local_path) -> (GitRepository, Path):

tests/metagpt/utils/test_s3.py

@@ -9,7 +9,6 @@ import uuid
 from pathlib import Path
 
 import aioboto3
-import aiofiles
 import pytest
 
 from metagpt.config2 import Config
@@ -37,7 +36,7 @@ async def test_s3(mocker):
     conn = S3(s3)
     object_name = "unittest.bak"
     await conn.upload_file(bucket=s3.bucket, local_path=__file__, object_name=object_name)
-    pathname = (Path(__file__).parent / uuid.uuid4().hex).with_suffix(".bak")
+    pathname = (Path(__file__).parent / "../../../workspace/unittest" / uuid.uuid4().hex).with_suffix(".bak")
     pathname.unlink(missing_ok=True)
     await conn.download_file(bucket=s3.bucket, object_name=object_name, local_path=str(pathname))
     assert pathname.exists()
@@ -45,8 +44,7 @@ async def test_s3(mocker):
     assert url
     bin_data = await conn.get_object(bucket=s3.bucket, object_name=object_name)
     assert bin_data
-    async with aiofiles.open(__file__, mode="r", encoding="utf-8") as reader:
-        data = await reader.read()
+    data = await aread(filename=__file__)
     res = await conn.cache(data, ".bak", "script")
     assert "http" in res
 
@@ -60,8 +58,6 @@ async def test_s3(mocker):
     except Exception:
         pass
 
-    await reader.close()
-
 
 if __name__ == "__main__":
     pytest.main([__file__, "-s"])