Add support for local llm (mistral 7b) (#31)

embedding-server/.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "embedding server",
+      "cwd": "${workspaceFolder}/app",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "uvicorn",
+      "args": ["main:app","--reload", "--port", "8000"],
+    }
+  ]
+}

embedding-server/Dockerfile

@@ -35,9 +35,11 @@ RUN apt-get update && apt-get install -y \
   curl \
   && rm -rf /var/lib/apt/lists/*
 
-RUN python install.py
+# comment it out for now as we don't want to download the model every time we build the image
+# we will mount host cache to docker image to avoid downloading the model every time
+# see docker-compose file for more details
+
 # RUN python install.py && \
 #   find /root/.cache/torch/sentence_transformers/ -name onnx -exec rm -rf {} +
 
-
 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "80"]

embedding-server/app/load_models.py

@@ -2,7 +2,7 @@ import os
 import sentence_transformers
 from gliner import GLiNER
 
-def load_transformers(models = os.getenv("MODELS", "sentence-transformers/all-MiniLM-L6-v2")):
+def load_transformers(models = os.getenv("MODELS", "BAAI/bge-large-en-v1.5")):
     transformers = {}
 
     for model in models.split(','):