Add support for RAG

docker-compose.yml

@@ -1,5 +1,13 @@
 version: '3.8'
 
+volumes:
+  uploads:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ./data/uploads
+
 services:
   rowboat:
     build:
@@ -21,10 +29,11 @@ services:
       - COPILOT_API_URL=http://copilot:3002
       - COPILOT_API_KEY=${COPILOT_API_KEY}
       - REDIS_URL=redis://redis:6379
-      - USE_RAG=${USE_RAG}
-      - QDRANT_URL=${QDRANT_URL}
+      - USE_RAG=true
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
-      - USE_RAG_UPLOADS=${USE_RAG_UPLOADS}
+      - USE_RAG_UPLOADS=true
+      - USE_RAG_S3_UPLOADS=${USE_RAG_S3_UPLOADS}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
       - RAG_UPLOADS_S3_BUCKET=${RAG_UPLOADS_S3_BUCKET}
@@ -38,7 +47,10 @@ services:
       - MAX_PROJECTS_PER_USER=${MAX_PROJECTS_PER_USER}
       - VOICE_API_URL=${VOICE_API_URL}
       - PROVIDER_DEFAULT_MODEL=${PROVIDER_DEFAULT_MODEL}
+      - RAG_UPLOADS_DIR=/app/uploads
     restart: unless-stopped
+    volumes:
+      - uploads:/app/uploads
 
   rowboat_agents:
     build:
@@ -51,7 +63,7 @@ services:
       - API_KEY=${AGENTS_API_KEY}
       - REDIS_URL=redis://redis:6379
       - MONGODB_URI=mongodb://mongo:27017/rowboat
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
       - PROVIDER_BASE_URL=${PROVIDER_BASE_URL}
       - PROVIDER_API_KEY=${PROVIDER_API_KEY}
@@ -99,21 +111,21 @@ services:
     build:
       context: ./apps/rowboat
       dockerfile: scripts.Dockerfile
-    command: ["sh", "-c", "npm run setupQdrant && echo 'index created successfully'"]
-    profiles: [ "setup_qdrant" ]
+    command: ["sh", "-c", "npm run setupQdrant"]
     environment:
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
+      - EMBEDDING_VECTOR_SIZE=${EMBEDDING_VECTOR_SIZE}
     restart: no
 
   delete_qdrant:
     build:
       context: ./apps/rowboat
       dockerfile: scripts.Dockerfile
-    command: ["sh", "-c", "npm run deleteQdrant && echo 'index deleted successfully'"]
+    command: ["sh", "-c", "npm run deleteQdrant"]
     profiles: [ "delete_qdrant" ]
     environment:
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
     restart: no
 
@@ -125,15 +137,23 @@ services:
     profiles: [ "rag_files_worker" ]
     environment:
       - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - EMBEDDING_PROVIDER_BASE_URL=${EMBEDDING_PROVIDER_BASE_URL}
+      - EMBEDDING_PROVIDER_API_KEY=${EMBEDDING_PROVIDER_API_KEY}
+      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
       - MONGODB_CONNECTION_STRING=mongodb://mongo:27017/rowboat
+      - REDIS_URL=redis://redis:6379
       - GOOGLE_API_KEY=${GOOGLE_API_KEY}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
       - RAG_UPLOADS_S3_BUCKET=${RAG_UPLOADS_S3_BUCKET}
       - RAG_UPLOADS_S3_REGION=${RAG_UPLOADS_S3_REGION}
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
+      - RAG_UPLOADS_DIR=/app/uploads
+      - USE_GEMINI_FILE_PARSING=${USE_GEMINI_FILE_PARSING}
     restart: unless-stopped
+    volumes:
+      - uploads:/app/uploads
 
   rag_urls_worker:
     build:
@@ -143,9 +163,13 @@ services:
     profiles: [ "rag_urls_worker" ]
     environment:
       - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - EMBEDDING_PROVIDER_BASE_URL=${EMBEDDING_PROVIDER_BASE_URL}
+      - EMBEDDING_PROVIDER_API_KEY=${EMBEDDING_PROVIDER_API_KEY}
+      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
       - MONGODB_CONNECTION_STRING=mongodb://mongo:27017/rowboat
+      - REDIS_URL=redis://redis:6379
       - FIRECRAWL_API_KEY=${FIRECRAWL_API_KEY}
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
     restart: unless-stopped
 
@@ -157,8 +181,12 @@ services:
     profiles: [ "rag_text_worker" ]
     environment:
       - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - EMBEDDING_PROVIDER_BASE_URL=${EMBEDDING_PROVIDER_BASE_URL}
+      - EMBEDDING_PROVIDER_API_KEY=${EMBEDDING_PROVIDER_API_KEY}
+      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
       - MONGODB_CONNECTION_STRING=mongodb://mongo:27017/rowboat
-      - QDRANT_URL=${QDRANT_URL}
+      - REDIS_URL=redis://redis:6379
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
     restart: unless-stopped
 
@@ -209,3 +237,13 @@ services:
   #     - ROWBOAT_API_HOST=http://rowboat:3000
   #     - MONGODB_URI=mongodb://mongo:27017/rowboat
   #   restart: unless-stopped
+
+  qdrant:
+    image: qdrant/qdrant
+    ports:
+      - "6333:6333"
+    environment:
+      - QDRANT__STORAGE__STORAGE_PATH=/data/qdrant
+    restart: unless-stopped
+    volumes:
+      - ./data/qdrant:/data/qdrant