Merge branch 'geekan:main' into add_counter

.github/workflows/pre-commit.yaml

@@ -11,6 +11,7 @@ on:
 jobs:
   pre-commit-check:
     runs-on: ubuntu-latest
+    environment: pre-commit
     steps:
     - name: Checkout Source Code
       uses: actions/checkout@v2

config/config2.example.yaml

@@ -18,6 +18,7 @@ embedding:
   model: ""
   api_version: ""
   embed_batch_size: 100
+  dimensions: # output dimension of embedding model
 
 repair_llm_output: true  # when the output is not a valid json, try to repair it
 

examples/rag_pipeline.py

@@ -18,13 +18,13 @@ from metagpt.rag.schema import (
 )
 from metagpt.utils.exceptions import handle_exception
 
+LLM_TIP = "If you not sure, just answer I don't know."
+
 DOC_PATH = EXAMPLE_DATA_PATH / "rag/writer.txt"
-QUESTION = "What are key qualities to be a good writer?"
+QUESTION = f"What are key qualities to be a good writer? {LLM_TIP}"
 
 TRAVEL_DOC_PATH = EXAMPLE_DATA_PATH / "rag/travel.txt"
-TRAVEL_QUESTION = "What does Bob like?"
-
-LLM_TIP = "If you not sure, just answer I don't know."
+TRAVEL_QUESTION = f"What does Bob like? {LLM_TIP}"
 
 
 class Player(BaseModel):
@@ -40,21 +40,21 @@ class Player(BaseModel):
 
 
 class RAGExample:
-    """Show how to use RAG.
+    """Show how to use RAG."""
 
-    Default engine use LLM Reranker, if the answer from the LLM is incorrect, may encounter `IndexError: list index out of range`.
-    """
-
-    def __init__(self, engine: SimpleEngine = None):
+    def __init__(self, engine: SimpleEngine = None, use_llm_ranker: bool = True):
         self._engine = engine
+        self._use_llm_ranker = use_llm_ranker
 
     @property
     def engine(self):
         if not self._engine:
+            ranker_configs = [LLMRankerConfig()] if self._use_llm_ranker else None
+
             self._engine = SimpleEngine.from_docs(
                 input_files=[DOC_PATH],
                 retriever_configs=[FAISSRetrieverConfig()],
-                ranker_configs=[LLMRankerConfig()],
+                ranker_configs=ranker_configs,
             )
         return self._engine
 
@@ -105,7 +105,7 @@ class RAGExample:
         """
         self._print_title("Add Docs")
 
-        travel_question = f"{TRAVEL_QUESTION}{LLM_TIP}"
+        travel_question = f"{TRAVEL_QUESTION}"
         travel_filepath = TRAVEL_DOC_PATH
 
         logger.info("[Before add docs]")
@@ -240,8 +240,14 @@ class RAGExample:
 
 
 async def main():
-    """RAG pipeline."""
-    e = RAGExample()
+    """RAG pipeline.
+
+    Note:
+    1. If `use_llm_ranker` is True, then it will use LLM Reranker to get better result, but it is not always guaranteed that the output will be parseable for reranking,
+       prefer `gpt-4-turbo`, otherwise might encounter `IndexError: list index out of range` or `ValueError: invalid literal for int() with base 10`.
+    """
+    e = RAGExample(use_llm_ranker=False)
+
     await e.run_pipeline()
     await e.add_docs()
     await e.add_objects()

metagpt/configs/embedding_config.py

@@ -20,11 +20,13 @@ class EmbeddingConfig(YamlModel):
     ---------
     api_type: "openai"
     api_key: "YOU_API_KEY"
+    dimensions: "YOUR_MODEL_DIMENSIONS"
 
     api_type: "azure"
     api_key: "YOU_API_KEY"
     base_url: "YOU_BASE_URL"
     api_version: "YOU_API_VERSION"
+    dimensions: "YOUR_MODEL_DIMENSIONS"
 
     api_type: "gemini"
     api_key: "YOU_API_KEY"
@@ -32,6 +34,7 @@ class EmbeddingConfig(YamlModel):
     api_type: "ollama"
     base_url: "YOU_BASE_URL"
     model: "YOU_MODEL"
+    dimensions: "YOUR_MODEL_DIMENSIONS"
     """
 
     api_type: Optional[EmbeddingType] = None
@@ -41,6 +44,7 @@ class EmbeddingConfig(YamlModel):
 
     model: Optional[str] = None
     embed_batch_size: Optional[int] = None
+    dimensions: Optional[int] = None  # output dimension of embedding model
 
     @field_validator("api_type", mode="before")
     @classmethod

metagpt/provider/human_provider.py

@@ -33,6 +33,7 @@ class HumanProvider(BaseLLM):
         format_msgs: Optional[list[dict[str, str]]] = None,
         generator: bool = False,
         timeout=USE_CONFIG_TIMEOUT,
+        **kwargs
     ) -> str:
         return self.ask(msg, timeout=self.get_timeout(timeout))
 

metagpt/rag/retrievers/bm25_retriever.py

@@ -40,8 +40,10 @@ class DynamicBM25Retriever(BM25Retriever):
         self._corpus = [self._tokenizer(node.get_content()) for node in self._nodes]
         self.bm25 = BM25Okapi(self._corpus)
 
-        self._index.insert_nodes(nodes, **kwargs)
+        if self._index:
+            self._index.insert_nodes(nodes, **kwargs)
 
     def persist(self, persist_dir: str, **kwargs) -> None:
         """Support persist."""
-        self._index.storage_context.persist(persist_dir)
+        if self._index:
+            self._index.storage_context.persist(persist_dir)

metagpt/rag/schema.py

@@ -12,6 +12,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
 
 from metagpt.config2 import config
 from metagpt.configs.embedding_config import EmbeddingType
+from metagpt.logs import logger
 from metagpt.rag.interface import RAGObject
 
 
@@ -44,7 +45,13 @@ class FAISSRetrieverConfig(IndexRetrieverConfig):
     @model_validator(mode="after")
     def check_dimensions(self):
         if self.dimensions == 0:
-            self.dimensions = self._embedding_type_to_dimensions.get(config.embedding.api_type, 1536)
+            self.dimensions = config.embedding.dimensions or self._embedding_type_to_dimensions.get(
+                config.embedding.api_type, 1536
+            )
+            if not config.embedding.dimensions and config.embedding.api_type not in self._embedding_type_to_dimensions:
+                logger.warning(
+                    f"You didn't set dimensions in config when using {config.embedding.api_type}, default to 1536"
+                )
 
         return self