upgrade llama-index to v0.10

metagpt/const.py

@@ -49,6 +49,7 @@ METAGPT_ROOT = get_metagpt_root()  # Dependent on METAGPT_PROJECT_ROOT
 DEFAULT_WORKSPACE_ROOT = METAGPT_ROOT / "workspace"
 
 EXAMPLE_PATH = METAGPT_ROOT / "examples"
+EXAMPLE_DATA_PATH = EXAMPLE_PATH / "data"
 DATA_PATH = METAGPT_ROOT / "data"
 TEST_DATA_PATH = METAGPT_ROOT / "tests/data"
 RESEARCH_PATH = DATA_PATH / "research"

metagpt/document_store/faiss_store.py

@@ -40,7 +40,7 @@ class FaissStore(LocalStore):
             return None
         vector_store = FaissVectorStore.from_persist_dir(persist_dir=self.cache_dir)
         storage_context = StorageContext.from_defaults(persist_dir=self.cache_dir, vector_store=vector_store)
-        index = load_index_from_storage(storage_context)
+        index = load_index_from_storage(storage_context, embed_model=self.embedding)
 
         return index
 
@@ -54,7 +54,9 @@ class FaissStore(LocalStore):
         # doc_store.add_documents(nodes)
         vector_store = FaissVectorStore(faiss_index=faiss.IndexFlatL2(1536))
         storage_context = StorageContext.from_defaults(vector_store=vector_store)
-        index = VectorStoreIndex.from_documents(documents=documents, storage_context=storage_context)
+        index = VectorStoreIndex.from_documents(
+            documents=documents, storage_context=storage_context, embed_model=self.embedding
+        )
 
         return index
 

metagpt/rag/engines/simple.py

@@ -18,6 +18,7 @@ from llama_index.core.response_synthesizers import (
 )
 from llama_index.core.retrievers import BaseRetriever
 from llama_index.core.schema import (
+    BaseNode,
     NodeWithScore,
     QueryBundle,
     QueryType,
@@ -110,15 +111,22 @@ class SimpleEngine(RetrieverQueryEngine):
 
         documents = SimpleDirectoryReader(input_files=input_files).load_data()
         nodes = run_transformations(documents, transformations=self.index._transformations)
-        self.retriever.add_nodes(nodes)
+        self._save_nodes(nodes)
 
     def add_objs(self, objs: list[RAGObject]):
         """Adds objects to the retriever, storing each object's original form in metadata for future reference."""
         self._ensure_retriever_modifiable()
 
         nodes = [TextNode(text=obj.rag_key(), metadata={"obj": obj.model_dump()}) for obj in objs]
-        self.retriever.add_nodes(nodes)
+        self._save_nodes(nodes)
 
     def _ensure_retriever_modifiable(self):
         if not isinstance(self.retriever, ModifiableRAGRetriever):
             raise TypeError(f"the retriever is not modifiable: {type(self.retriever)}")
+
+    def _save_nodes(self, nodes: list[BaseNode]):
+        # for search in memory
+        self.retriever.add_nodes(nodes)
+
+        # for persist
+        self.index.insert_nodes(nodes)

metagpt/rag/retrievers/hybrid_retriever.py

@@ -22,7 +22,7 @@ class SimpleHybridRetriever(RAGRetriever):
         """
         all_nodes = []
         for retriever in self.retrievers:
-            # 防止retriever可能改变query的属性
+            # Prevent retriever changing query
             query_copy = copy.deepcopy(query)
             nodes = await retriever.aretrieve(query_copy, **kwargs)
             all_nodes.extend(nodes)