apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-07-14 16:32:16 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

add crawler tools

Browse source
This commit is contained in:
shenchucheng 2024-05-30 20:04:02 +08:00
parent d1d44e9cea
commit 4f43b905a2
5 changed files with 83 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

4
metagpt/rag/engines/simple.py
View file

@ -4,6 +4,7 @@ import json
import os
from typing import Any, Optional, Union
from fsspec import AbstractFileSystem
from llama_index.core import SimpleDirectoryReader, VectorStoreIndex
from llama_index.core.callbacks.base import CallbackManager
from llama_index.core.embeddings import BaseEmbedding
@ -83,6 +84,7 @@ class SimpleEngine(RetrieverQueryEngine):
llm: LLM = None,
retriever_configs: list[BaseRetrieverConfig] = None,
ranker_configs: list[BaseRankerConfig] = None,
fs: Optional[AbstractFileSystem] = None,
) -> "SimpleEngine":
"""From docs.
@ -100,7 +102,7 @@ class SimpleEngine(RetrieverQueryEngine):
if not input_dir and not input_files:
raise ValueError("Must provide either `input_dir` or `input_files`.")
documents = SimpleDirectoryReader(input_dir=input_dir, input_files=input_files).load_data()
documents = SimpleDirectoryReader(input_dir=input_dir, input_files=input_files, fs=fs).load_data()
cls._fix_document_metadata(documents)
index = VectorStoreIndex.from_documents(