Add support for local llm (mistral 7b) (#31)

demos/weather-forecast/README.md

@@ -4,12 +4,12 @@ This demo shows how you can use intelligent prompt gateway to provide realtime w
 # Startig the demo
 1. Ensure that submodule is up to date
    ```sh
-   $ git submodule sync --recursive
+   git submodule sync --recursive
    ```
 1. Create `.env` file and set OpenAI key using env var `OPENAI_API_KEY`
 1. Start services
    ```sh
-   $ docker compose up
+   docker compose up
    ```
 1. Navigate to http://localhost:18080/
 1. You can type in queries like "how is the weather in Seattle"

demos/weather-forecast/docker-compose.yaml

@@ -17,8 +17,6 @@ services:
     volumes:
       - ./generated/envoy.yaml:/etc/envoy/envoy.yaml
       - /etc/ssl/cert.pem:/etc/ssl/cert.pem
-    networks:
-      - envoymesh
     depends_on:
       config-generator:
         condition: service_completed_successfully
@@ -35,17 +33,14 @@ services:
         test: ["CMD", "curl" ,"http://localhost:80/healthz"]
         interval: 5s
         retries: 20
-    networks:
-      - envoymesh
-
+    volumes:
+      - ~/.cache/huggingface:/root/.cache/huggingface
   qdrant:
     image: qdrant/qdrant
     hostname: vector-db
     ports:
       - 16333:6333
       - 16334:6334
-    networks:
-      - envoymesh
 
   chatbot-ui:
     build:
@@ -53,8 +48,6 @@ services:
       dockerfile: Dockerfile
     ports:
       - "18080:8080"
-    networks:
-      - envoymesh
     environment:
       - OPENAI_API_KEY=${OPENAI_API_KEY}
       - CHAT_COMPLETION_ENDPOINT=http://envoy:10000/v1/chat/completions
@@ -70,8 +63,6 @@ services:
     volumes:
       - ./prometheus:/etc/prometheus
       - ./prom_data:/prometheus
-    networks:
-      - envoymesh
 
   grafana:
     image: grafana/grafana
@@ -86,9 +77,3 @@ services:
       - ./grafana:/etc/grafana/provisioning/datasources
       - ./grafana/dashboard.yaml:/etc/grafana/provisioning/dashboards/main.yaml
       - ./grafana/dashboards:/var/lib/grafana/dashboards
-      # - ./grafana-data:/var/lib/grafana
-    networks:
-      - envoymesh
-
-networks:
-  envoymesh: {}

demos/weather-forecast/katanemo-config.yaml

@@ -8,9 +8,10 @@ embedding_provider:
 
 llm_providers:
 
-  - name: "open-ai-gpt-4"
-    api_key: "$OPEN_AI_API_KEY"
+  - name: open-ai-gpt-4
+    api_key: $OPEN_AI_API_KEY
     model: gpt-4
+    default: true
 
 prompt_targets: