mirror of
https://github.com/FoundationAgents/MetaGPT.git
synced 2026-05-03 04:42:38 +02:00
add serper search api
This commit is contained in:
leonzh0u
parent
d651ed2882
commit
6a4b980c40
9 changed files with 301 additions and 7 deletions
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
#### if OpenAI
|
||||
|
||||
#OPENAI_API_KEY: "YOUR_API_KEY"
|
||||
OPENAI_API_KEY: "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"
|
||||
#OPENAI_API_BASE: "YOUR_API_BASE"
|
||||
OPENAI_API_MODEL: "gpt-4"
|
||||
MAX_TOKENS: 1500
|
||||
|
|
@ -26,6 +26,8 @@ RPM: 10
|
|||
#GOOGLE_API_KEY: "YOUR_API_KEY"
|
||||
## Visit https://programmablesearchengine.google.com/controlpanel/create to get id.
|
||||
#GOOGLE_CSE_ID: "YOUR_CSE_ID"
|
||||
## Visit https://serper.dev/ to get key.
|
||||
# SERPER_API_KEY: "YOUR_API_KEY"
|
||||
|
||||
#### for TTS
|
||||
|
||||
|
|
|
|||
150
examples/Untitled.ipynb
Normal file
150
examples/Untitled.ipynb
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "b5359d43",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "ModuleNotFoundError",
|
||||
"evalue": "No module named 'metagpt'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[0;32mIn[3], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01masyncio\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmetagpt\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Config\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmetagpt\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mroles\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Searcher\n",
|
||||
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'metagpt'"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import asyncio\n",
|
||||
"from metagpt.config import Config\n",
|
||||
"from metagpt.roles import Searcher"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "c01e1f86",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Requirement already satisfied: metagpt in /Users/leon/miniconda3/lib/python3.10/site-packages/metagpt-0.1-py3.10.egg (0.1)\n",
|
||||
"Requirement already satisfied: aiohttp==3.8.4 in /Users/leon/miniconda3/lib/python3.10/site-packages (from metagpt) (3.8.4)\n",
|
||||
"Requirement already satisfied: channels==4.0.0 in /Users/leon/miniconda3/lib/python3.10/site-packages/channels-4.0.0-py3.10.egg (from metagpt) (4.0.0)\n",
|
||||
"Requirement already satisfied: duckduckgo_search==2.9.4 in /Users/leon/miniconda3/lib/python3.10/site-packages/duckduckgo_search-2.9.4-py3.10.egg (from metagpt) (2.9.4)\n",
|
||||
"Requirement already satisfied: faiss_cpu==1.7.4 in /Users/leon/miniconda3/lib/python3.10/site-packages/faiss_cpu-1.7.4-py3.10-macosx-11.1-arm64.egg (from metagpt) (1.7.4)\n",
|
||||
"Requirement already satisfied: fire==0.4.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from metagpt) (0.4.0)\n",
|
||||
"Requirement already satisfied: langchain==0.0.231 in /Users/leon/miniconda3/lib/python3.10/site-packages/langchain-0.0.231-py3.10.egg (from metagpt) (0.0.231)\n",
|
||||
"Requirement already satisfied: loguru==0.6.0 in /Users/leon/miniconda3/lib/python3.10/site-packages/loguru-0.6.0-py3.10.egg (from metagpt) (0.6.0)\n",
|
||||
"Requirement already satisfied: meilisearch==0.21.0 in /Users/leon/miniconda3/lib/python3.10/site-packages/meilisearch-0.21.0-py3.10.egg (from metagpt) (0.21.0)\n",
|
||||
"Requirement already satisfied: numpy==1.24.3 in /Users/leon/miniconda3/lib/python3.10/site-packages (from metagpt) (1.24.3)\n",
|
||||
"Requirement already satisfied: openai==0.27.8 in /Users/leon/miniconda3/lib/python3.10/site-packages (from metagpt) (0.27.8)\n",
|
||||
"Requirement already satisfied: openpyxl in /Users/leon/miniconda3/lib/python3.10/site-packages (from metagpt) (3.1.2)\n",
|
||||
"Requirement already satisfied: pandas==1.4.1 in /Users/leon/miniconda3/lib/python3.10/site-packages (from metagpt) (1.4.1)\n",
|
||||
"Requirement already satisfied: pydantic==1.10.7 in /Users/leon/miniconda3/lib/python3.10/site-packages (from metagpt) (1.10.7)\n",
|
||||
"Requirement already satisfied: pytest==7.2.2 in /Users/leon/miniconda3/lib/python3.10/site-packages/pytest-7.2.2-py3.10.egg (from metagpt) (7.2.2)\n",
|
||||
"Requirement already satisfied: python_docx==0.8.11 in /Users/leon/miniconda3/lib/python3.10/site-packages/python_docx-0.8.11-py3.10.egg (from metagpt) (0.8.11)\n",
|
||||
"Requirement already satisfied: PyYAML==6.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from metagpt) (6.0)\n",
|
||||
"Requirement already satisfied: setuptools==65.6.3 in /Users/leon/miniconda3/lib/python3.10/site-packages (from metagpt) (65.6.3)\n",
|
||||
"Requirement already satisfied: tenacity==8.2.2 in /Users/leon/miniconda3/lib/python3.10/site-packages/tenacity-8.2.2-py3.10.egg (from metagpt) (8.2.2)\n",
|
||||
"Requirement already satisfied: tiktoken==0.3.3 in /Users/leon/miniconda3/lib/python3.10/site-packages/tiktoken-0.3.3-py3.10-macosx-11.1-arm64.egg (from metagpt) (0.3.3)\n",
|
||||
"Requirement already satisfied: tqdm==4.64.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from metagpt) (4.64.0)\n",
|
||||
"Requirement already satisfied: attrs>=17.3.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from aiohttp==3.8.4->metagpt) (23.1.0)\n",
|
||||
"Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from aiohttp==3.8.4->metagpt) (2.0.4)\n",
|
||||
"Requirement already satisfied: multidict<7.0,>=4.5 in /Users/leon/miniconda3/lib/python3.10/site-packages (from aiohttp==3.8.4->metagpt) (6.0.4)\n",
|
||||
"Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/leon/miniconda3/lib/python3.10/site-packages (from aiohttp==3.8.4->metagpt) (4.0.2)\n",
|
||||
"Requirement already satisfied: yarl<2.0,>=1.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from aiohttp==3.8.4->metagpt) (1.9.2)\n",
|
||||
"Requirement already satisfied: frozenlist>=1.1.1 in /Users/leon/miniconda3/lib/python3.10/site-packages (from aiohttp==3.8.4->metagpt) (1.3.3)\n",
|
||||
"Requirement already satisfied: aiosignal>=1.1.2 in /Users/leon/miniconda3/lib/python3.10/site-packages (from aiohttp==3.8.4->metagpt) (1.3.1)\n",
|
||||
"Requirement already satisfied: Django>=3.2 in /Users/leon/miniconda3/lib/python3.10/site-packages (from channels==4.0.0->metagpt) (4.2.3)\n",
|
||||
"Requirement already satisfied: asgiref<4,>=3.5.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from channels==4.0.0->metagpt) (3.7.2)\n",
|
||||
"Requirement already satisfied: click>=8.1.3 in /Users/leon/miniconda3/lib/python3.10/site-packages (from duckduckgo_search==2.9.4->metagpt) (8.1.3)\n",
|
||||
"Requirement already satisfied: diskcache>=5.6.1 in /Users/leon/miniconda3/lib/python3.10/site-packages (from duckduckgo_search==2.9.4->metagpt) (5.6.1)\n",
|
||||
"Requirement already satisfied: requests>=2.29.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from duckduckgo_search==2.9.4->metagpt) (2.31.0)\n",
|
||||
"Requirement already satisfied: six in /Users/leon/miniconda3/lib/python3.10/site-packages (from fire==0.4.0->metagpt) (1.16.0)\n",
|
||||
"Requirement already satisfied: termcolor in /Users/leon/miniconda3/lib/python3.10/site-packages (from fire==0.4.0->metagpt) (2.3.0)\n",
|
||||
"Requirement already satisfied: SQLAlchemy<3,>=1.4 in /Users/leon/miniconda3/lib/python3.10/site-packages (from langchain==0.0.231->metagpt) (2.0.19)\n",
|
||||
"Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /Users/leon/miniconda3/lib/python3.10/site-packages (from langchain==0.0.231->metagpt) (0.5.9)\n",
|
||||
"Requirement already satisfied: langchainplus-sdk<0.0.21,>=0.0.20 in /Users/leon/miniconda3/lib/python3.10/site-packages (from langchain==0.0.231->metagpt) (0.0.20)\n",
|
||||
"Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /Users/leon/miniconda3/lib/python3.10/site-packages (from langchain==0.0.231->metagpt) (2.8.4)\n",
|
||||
"Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /Users/leon/miniconda3/lib/python3.10/site-packages (from langchain==0.0.231->metagpt) (1.2.4)\n",
|
||||
"Requirement already satisfied: camel-converter[pydantic] in /Users/leon/miniconda3/lib/python3.10/site-packages (from meilisearch==0.21.0->metagpt) (3.0.2)\n",
|
||||
"Requirement already satisfied: python-dateutil>=2.8.1 in /Users/leon/miniconda3/lib/python3.10/site-packages (from pandas==1.4.1->metagpt) (2.8.2)\n",
|
||||
"Requirement already satisfied: pytz>=2020.1 in /Users/leon/miniconda3/lib/python3.10/site-packages (from pandas==1.4.1->metagpt) (2023.3)\n",
|
||||
"Requirement already satisfied: typing-extensions>=4.2.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from pydantic==1.10.7->metagpt) (4.6.3)\n",
|
||||
"Requirement already satisfied: iniconfig in /Users/leon/miniconda3/lib/python3.10/site-packages (from pytest==7.2.2->metagpt) (2.0.0)\n",
|
||||
"Requirement already satisfied: packaging in /Users/leon/miniconda3/lib/python3.10/site-packages (from pytest==7.2.2->metagpt) (23.0)\n",
|
||||
"Requirement already satisfied: pluggy<2.0,>=0.12 in /Users/leon/miniconda3/lib/python3.10/site-packages (from pytest==7.2.2->metagpt) (1.0.0)\n",
|
||||
"Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /Users/leon/miniconda3/lib/python3.10/site-packages (from pytest==7.2.2->metagpt) (1.1.1)\n",
|
||||
"Requirement already satisfied: tomli>=1.0.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from pytest==7.2.2->metagpt) (2.0.1)\n",
|
||||
"Requirement already satisfied: lxml>=2.3.2 in /Users/leon/miniconda3/lib/python3.10/site-packages (from python_docx==0.8.11->metagpt) (4.9.3)\n",
|
||||
"Requirement already satisfied: regex>=2022.1.18 in /Users/leon/miniconda3/lib/python3.10/site-packages (from tiktoken==0.3.3->metagpt) (2023.6.3)\n",
|
||||
"Requirement already satisfied: et-xmlfile in /Users/leon/miniconda3/lib/python3.10/site-packages (from openpyxl->metagpt) (1.1.0)\n",
|
||||
"Requirement already satisfied: marshmallow<4.0.0,>=3.3.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.231->metagpt) (3.19.0)\n",
|
||||
"Requirement already satisfied: marshmallow-enum<2.0.0,>=1.5.1 in /Users/leon/miniconda3/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.231->metagpt) (1.5.1)\n",
|
||||
"Requirement already satisfied: typing-inspect>=0.4.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.231->metagpt) (0.9.0)\n",
|
||||
"Requirement already satisfied: sqlparse>=0.3.1 in /Users/leon/miniconda3/lib/python3.10/site-packages (from Django>=3.2->channels==4.0.0->metagpt) (0.4.4)\n",
|
||||
"Requirement already satisfied: idna<4,>=2.5 in /Users/leon/miniconda3/lib/python3.10/site-packages (from requests>=2.29.0->duckduckgo_search==2.9.4->metagpt) (3.4)\n",
|
||||
"Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/leon/miniconda3/lib/python3.10/site-packages (from requests>=2.29.0->duckduckgo_search==2.9.4->metagpt) (1.26.15)\n",
|
||||
"Requirement already satisfied: certifi>=2017.4.17 in /Users/leon/miniconda3/lib/python3.10/site-packages (from requests>=2.29.0->duckduckgo_search==2.9.4->metagpt) (2022.12.7)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/leon/miniconda3/lib/python3.10/site-packages (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.231->metagpt) (1.0.0)\n",
|
||||
"\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!pip install metagpt"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b48f4dab",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"async def main():\n",
|
||||
" await Searcher().run(\"What are some good sun protection products?\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if __name__ == '__main__':\n",
|
||||
" asyncio.run(main())"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
10
examples/serper_api.py
Normal file
10
examples/serper_api.py
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import asyncio
|
||||
from metagpt.config import Config
|
||||
from metagpt.roles import Searcher
|
||||
from metagpt.tools import SearchEngineType
|
||||
|
||||
async def main():
|
||||
await Searcher(engine = SearchEngineType.SERPER_GOOGLE).run("What are the best ski brands for advanced skiers?")
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
|
|
@ -110,10 +110,14 @@ class SearchAndSummarize(Action):
|
|||
super().__init__(name, context, llm)
|
||||
|
||||
async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYSTEM) -> str:
|
||||
if not self.config.serpapi_api_key or 'YOUR_API_KEY' == self.config.serpapi_api_key:
|
||||
logger.warning('Configure SERPAPI_API_KEY to unlock full feature')
|
||||
no_serpapi = not self.config.serpapi_api_key or 'YOUR_API_KEY' == self.config.serpapi_api_key
|
||||
no_serper = not self.config.serper_api_key or 'YOUR_API_KEY' == self.config.serper_api_key
|
||||
no_google= not self.config.google_api_key or 'YOUR_API_KEY' == self.config.google_api_key
|
||||
|
||||
if no_serpapi and no_google and no_serper:
|
||||
logger.warning('Configure one of SERPAPI_API_KEY, SERPER_API_KEY, GOOGLE_API_KEY to unlock full feature')
|
||||
return ""
|
||||
|
||||
|
||||
query = context[-1].content
|
||||
# logger.debug(query)
|
||||
rsp = await self.search_engine.run(query)
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ class Config(metaclass=Singleton):
|
|||
self.deployment_id = self._get('DEPLOYMENT_ID')
|
||||
|
||||
self.serpapi_api_key = self._get('SERPAPI_API_KEY')
|
||||
self.serper_api_key = self._get('SERPER_API_KEY')
|
||||
self.google_api_key = self._get('GOOGLE_API_KEY')
|
||||
self.google_cse_id = self._get('GOOGLE_CSE_ID')
|
||||
self.search_engine = self._get('SEARCH_ENGINE', SearchEngineType.SERPAPI_GOOGLE)
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ from metagpt.schema import Message
|
|||
|
||||
class Searcher(Role):
|
||||
def __init__(self, name='Alice', profile='Smart Assistant', goal='Provide search services for users',
|
||||
constraints='Answer is rich and complete', **kwargs):
|
||||
constraints='Answer is rich and complete', engine=SearchEngineType.SERPAPI_GOOGLE, **kwargs):
|
||||
super().__init__(name, profile, goal, constraints, **kwargs)
|
||||
self._init_actions([SearchAndSummarize])
|
||||
self._init_actions([SearchAndSummarize(engine = engine)])
|
||||
|
||||
def set_search_func(self, search_func):
|
||||
action = SearchAndSummarize("", engine=SearchEngineType.CUSTOM_ENGINE, search_func=search_func)
|
||||
|
|
@ -24,7 +24,6 @@ class Searcher(Role):
|
|||
|
||||
async def _act_sp(self) -> Message:
|
||||
logger.info(f"{self._setting}: ready to {self._rc.todo}")
|
||||
logger.info(self._rc.memory.get(k=0))
|
||||
response = await self._rc.todo.run(self._rc.memory.get(k=0))
|
||||
# logger.info(response)
|
||||
if isinstance(response, ActionOutput):
|
||||
|
|
|
|||
|
|
@ -13,4 +13,5 @@ from enum import Enum, auto
|
|||
class SearchEngineType(Enum):
|
||||
SERPAPI_GOOGLE = auto()
|
||||
DIRECT_GOOGLE = auto()
|
||||
SERPER_GOOGLE = auto()
|
||||
CUSTOM_ENGINE = auto()
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from duckduckgo_search import ddg
|
|||
|
||||
from metagpt.config import Config
|
||||
from metagpt.tools.search_engine_serpapi import SerpAPIWrapper
|
||||
from metagpt.tools.search_engine_serper import SerperWrapper
|
||||
|
||||
config = Config()
|
||||
from metagpt.tools import SearchEngineType
|
||||
|
|
@ -44,6 +45,12 @@ class SearchEngine:
|
|||
rsp = await api.run(query)
|
||||
elif self.engine == SearchEngineType.DIRECT_GOOGLE:
|
||||
rsp = SearchEngine.run_google(query, max_results)
|
||||
elif self.engine == SearchEngineType.SERPER_GOOGLE:
|
||||
api = SerperWrapper()
|
||||
if isinstance(query, list):
|
||||
rsp = await api.run(query)
|
||||
elif isinstance(query, str):
|
||||
rsp = await api.run([query])
|
||||
elif self.engine == SearchEngineType.CUSTOM_ENGINE:
|
||||
rsp = self.run_func(query)
|
||||
else:
|
||||
|
|
|
|||
120
metagpt/tools/search_engine_serper.py
Normal file
120
metagpt/tools/search_engine_serper.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
@Time : 2023/5/23 18:27
|
||||
@Author : alexanderwu
|
||||
@File : search_engine_serpapi.py
|
||||
"""
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from metagpt.logs import logger
|
||||
import aiohttp
|
||||
import json
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from metagpt.config import Config
|
||||
|
||||
|
||||
class SerperWrapper(BaseModel):
|
||||
"""Wrapper around SerpAPI.
|
||||
|
||||
To use, you should have the ``google-search-results`` python package installed,
|
||||
and the environment variable ``SERPAPI_API_KEY`` set with your API key, or pass
|
||||
`serpapi_api_key` as a named parameter to the constructor.
|
||||
"""
|
||||
|
||||
search_engine: Any #: :meta private:
|
||||
payload: dict = Field(
|
||||
default={
|
||||
"page": 1,
|
||||
"num": 10
|
||||
}
|
||||
)
|
||||
config = Config()
|
||||
serper_api_key: Optional[str] = config.serper_api_key
|
||||
aiosession: Optional[aiohttp.ClientSession] = None
|
||||
|
||||
class Config:
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
async def run(self, query: str, **kwargs: Any) -> str:
|
||||
"""Run query through Serper and parse result async."""
|
||||
return ";".join([self._process_response(res) for res in await self.results(query)])
|
||||
|
||||
async def results(self, queries: list[str]) -> dict:
|
||||
"""Use aiohttp to run query through Serper and return the results async."""
|
||||
|
||||
def construct_url_and_payload_and_headers() -> Tuple[str, Dict[str, str]]:
|
||||
payloads = self.get_payloads(queries)
|
||||
url = "https://google.serper.dev/search"
|
||||
headers = self.get_headers()
|
||||
return url, payloads, headers
|
||||
|
||||
url, payloads, headers = construct_url_and_payload_and_headers()
|
||||
if not self.aiosession:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(url, data=payloads, headers=headers) as response:
|
||||
res = await response.json()
|
||||
|
||||
else:
|
||||
async with self.aiosession.get.post(url, data=payloads, headers=headers) as response:
|
||||
res = await response.json()
|
||||
|
||||
return res
|
||||
|
||||
def get_payloads(self, queries: list[str]) -> Dict[str, str]:
|
||||
"""Get payloads for Serper."""
|
||||
payloads = []
|
||||
for query in queries:
|
||||
_payload = {
|
||||
"q": query,
|
||||
}
|
||||
payloads.append({**self.payload, **_payload})
|
||||
return json.dumps(payloads, sort_keys=True)
|
||||
|
||||
def get_headers(self) -> Dict[str, str]:
|
||||
headers = {
|
||||
'X-API-KEY': self.serper_api_key,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
return headers
|
||||
|
||||
@staticmethod
|
||||
def _process_response(res: dict) -> str:
|
||||
"""Process response from SerpAPI."""
|
||||
# logger.debug(res)
|
||||
focus = ['title', 'snippet', 'link']
|
||||
def get_focused(x): return {i: j for i, j in x.items() if i in focus}
|
||||
|
||||
if "error" in res.keys():
|
||||
raise ValueError(f"Got error from SerpAPI: {res['error']}")
|
||||
if "answer_box" in res.keys() and "answer" in res["answer_box"].keys():
|
||||
toret = res["answer_box"]["answer"]
|
||||
elif "answer_box" in res.keys() and "snippet" in res["answer_box"].keys():
|
||||
toret = res["answer_box"]["snippet"]
|
||||
elif (
|
||||
"answer_box" in res.keys()
|
||||
and "snippet_highlighted_words" in res["answer_box"].keys()
|
||||
):
|
||||
toret = res["answer_box"]["snippet_highlighted_words"][0]
|
||||
elif (
|
||||
"sports_results" in res.keys()
|
||||
and "game_spotlight" in res["sports_results"].keys()
|
||||
):
|
||||
toret = res["sports_results"]["game_spotlight"]
|
||||
elif (
|
||||
"knowledge_graph" in res.keys()
|
||||
and "description" in res["knowledge_graph"].keys()
|
||||
):
|
||||
toret = res["knowledge_graph"]["description"]
|
||||
elif "snippet" in res["organic"][0].keys():
|
||||
toret = res["organic"][0]["snippet"]
|
||||
else:
|
||||
toret = "No good search result found"
|
||||
|
||||
toret_l = []
|
||||
if "answer_box" in res.keys() and "snippet" in res["answer_box"].keys():
|
||||
toret_l += [get_focused(res["answer_box"])]
|
||||
if res.get("organic"):
|
||||
toret_l += [get_focused(i) for i in res.get("organic")]
|
||||
|
||||
return str(toret) + '\n' + str(toret_l)
|
||||
Loading…
Add table
Add a link
Reference in a new issue