Merge branch 'geekan:main' into add_counter

metagpt/configs/embedding_config.py

@@ -20,11 +20,13 @@ class EmbeddingConfig(YamlModel):
     ---------
     api_type: "openai"
     api_key: "YOU_API_KEY"
+    dimensions: "YOUR_MODEL_DIMENSIONS"
 
     api_type: "azure"
     api_key: "YOU_API_KEY"
     base_url: "YOU_BASE_URL"
     api_version: "YOU_API_VERSION"
+    dimensions: "YOUR_MODEL_DIMENSIONS"
 
     api_type: "gemini"
     api_key: "YOU_API_KEY"
@@ -32,6 +34,7 @@ class EmbeddingConfig(YamlModel):
     api_type: "ollama"
     base_url: "YOU_BASE_URL"
     model: "YOU_MODEL"
+    dimensions: "YOUR_MODEL_DIMENSIONS"
     """
 
     api_type: Optional[EmbeddingType] = None
@@ -41,6 +44,7 @@ class EmbeddingConfig(YamlModel):
 
     model: Optional[str] = None
     embed_batch_size: Optional[int] = None
+    dimensions: Optional[int] = None  # output dimension of embedding model
 
     @field_validator("api_type", mode="before")
     @classmethod

metagpt/provider/human_provider.py

@@ -33,6 +33,7 @@ class HumanProvider(BaseLLM):
         format_msgs: Optional[list[dict[str, str]]] = None,
         generator: bool = False,
         timeout=USE_CONFIG_TIMEOUT,
+        **kwargs
     ) -> str:
         return self.ask(msg, timeout=self.get_timeout(timeout))
 

metagpt/rag/retrievers/bm25_retriever.py

@@ -40,8 +40,10 @@ class DynamicBM25Retriever(BM25Retriever):
         self._corpus = [self._tokenizer(node.get_content()) for node in self._nodes]
         self.bm25 = BM25Okapi(self._corpus)
 
-        self._index.insert_nodes(nodes, **kwargs)
+        if self._index:
+            self._index.insert_nodes(nodes, **kwargs)
 
     def persist(self, persist_dir: str, **kwargs) -> None:
         """Support persist."""
-        self._index.storage_context.persist(persist_dir)
+        if self._index:
+            self._index.storage_context.persist(persist_dir)

metagpt/rag/schema.py

@@ -12,6 +12,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
 
 from metagpt.config2 import config
 from metagpt.configs.embedding_config import EmbeddingType
+from metagpt.logs import logger
 from metagpt.rag.interface import RAGObject
 
 
@@ -44,7 +45,13 @@ class FAISSRetrieverConfig(IndexRetrieverConfig):
     @model_validator(mode="after")
     def check_dimensions(self):
         if self.dimensions == 0:
-            self.dimensions = self._embedding_type_to_dimensions.get(config.embedding.api_type, 1536)
+            self.dimensions = config.embedding.dimensions or self._embedding_type_to_dimensions.get(
+                config.embedding.api_type, 1536
+            )
+            if not config.embedding.dimensions and config.embedding.api_type not in self._embedding_type_to_dimensions:
+                logger.warning(
+                    f"You didn't set dimensions in config when using {config.embedding.api_type}, default to 1536"
+                )
 
         return self