add aiohttp encapsulation

metagpt/utils/ahttp_client.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : pure async http_client
+
+from typing import Optional, Any, Mapping, Union
+
+from aiohttp.client import DEFAULT_TIMEOUT
+import aiohttp
+
+
+async def apost(url: str,
+                params: Optional[Mapping[str, str]] = None,
+                json: Any = None,
+                data: Any = None,
+                headers: Optional[dict] = None,
+                as_json: bool = False,
+                encoding: str = "utf-8",
+                timeout: int = DEFAULT_TIMEOUT.total) -> Union[str, dict]:
+    async with aiohttp.ClientSession() as session:
+        async with session.post(
+            url=url,
+            params=params,
+            json=json,
+            data=data,
+            headers=headers,
+            timeout=timeout
+        ) as resp:
+            if as_json:
+                data = await resp.json()
+            else:
+                data = await resp.read()
+                data = data.decode(encoding)
+    return data
+
+
+async def apost_stream(url: str,
+                       params: Optional[Mapping[str, str]] = None,
+                       json: Any = None,
+                       data: Any = None,
+                       headers: Optional[dict] = None,
+                       encoding: str = "utf-8",
+                       timeout: int = DEFAULT_TIMEOUT.total) -> Any:
+    """
+    usage:
+        result = astream(url="xx")
+        async for line in result:
+            deal_with(line)
+    """
+    async with aiohttp.ClientSession() as session:
+        async with session.post(
+            url=url,
+            params=params,
+            json=json,
+            data=data,
+            headers=headers,
+            timeout=timeout
+        ) as resp:
+            async for line in resp.content:
+                yield line.decode(encoding)

tests/metagpt/utils/test_ahttp_client.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : unittest of ahttp_client
+
+import pytest
+
+from metagpt.utils.ahttp_client import apost, apost_stream
+
+
+@pytest.mark.asyncio
+async def test_apost():
+    result = await apost(
+        url="https://www.baidu.com/"
+    )
+    assert "百度一下" in result
+
+    result = await apost(
+        url="http://aider.meizu.com/app/weather/listWeather",
+        data={"cityIds": "101240101"},
+        as_json=True
+    )
+    assert result["code"] == "200"
+
+
+@pytest.mark.asyncio
+async def test_apost_stream():
+    result = apost_stream(
+        url="https://www.baidu.com/"
+    )
+    async for line in result:
+        assert len(line) >= 0
+
+    result = apost_stream(
+        url="http://aider.meizu.com/app/weather/listWeather",
+        data={"cityIds": "101240101"}
+    )
+    async for line in result:
+        assert len(line) >= 0