Metrics (#3)

* Basic metrics working
* Add consumer & producer metrics
* Grafana & Prometheus in docker compose

docker-compose-vertexai.yaml

@@ -6,6 +6,8 @@ volumes:
   etcd:
   minio-data:
   milvus:
+  prometheus-data:
+  grafana-storage:
 
 services:
 
@@ -90,8 +92,34 @@ services:
       - "milvus:/var/lib/milvus"
     restart: on-failure:100
 
+  prometheus:
+    image: docker.io/prom/prometheus:v2.53.1
+    ports:
+      - "9090:9090"
+    volumes:
+      - "./prometheus:/etc/prometheus"
+      - "prometheus-data:/prometheus"
+    restart: on-failure:100
+
+  grafana:
+    image: docker.io/grafana/grafana:10.0.0
+    ports:
+      - "3000:3000"
+    volumes:
+      - "grafana-storage:/var/lib/grafana"
+      - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml"
+      - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml"
+      - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json"
+    environment:
+#       GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
+#       GF_AUTH_ANONYMOUS_ENABLED: true
+#       GF_ORG_ROLE: Admin
+      GF_ORG_NAME: trustgraph.ai
+#     GF_SERVER_ROOT_URL: https://example.com
+    restart: on-failure:100
+
   pdf-decoder:
-    image: docker.io/trustgraph/trustgraph-flow:0.3.3
+    image: docker.io/trustgraph/trustgraph-flow:0.4.1
     command:
       - "pdf-decoder"
       - "-p"
@@ -99,7 +127,7 @@ services:
     restart: on-failure:100
 
   chunker:
-    image: docker.io/trustgraph/trustgraph-flow:0.3.3
+    image: docker.io/trustgraph/trustgraph-flow:0.4.1
     command:
       - "chunker-recursive"
       - "-p"
@@ -107,7 +135,7 @@ services:
     restart: on-failure:100
 
   vectorize:
-    image: docker.io/trustgraph/trustgraph-flow:0.3.3
+    image: docker.io/trustgraph/trustgraph-flow:0.4.1
     command:
       - "embeddings-vectorize"
       - "-p"
@@ -115,15 +143,17 @@ services:
     restart: on-failure:100
     
   embeddings:
-    image: docker.io/trustgraph/trustgraph-flow:0.3.3
+    image: docker.io/trustgraph/trustgraph-flow:0.4.1
     command:
       - "embeddings-hf"
       - "-p"
       - "pulsar://pulsar:6650"
+      - "-m"
+      - "mixedbread-ai/mxbai-embed-large-v1"
     restart: on-failure:100
     
   kg-extract-definitions:
-    image: docker.io/trustgraph/trustgraph-flow:0.3.3
+    image: docker.io/trustgraph/trustgraph-flow:0.4.1
     command:
       - "kg-extract-definitions"
       - "-p"
@@ -131,7 +161,7 @@ services:
     restart: on-failure:100
     
   kg-extract-relationships:
-    image: docker.io/trustgraph/trustgraph-flow:0.3.3
+    image: docker.io/trustgraph/trustgraph-flow:0.4.1
     command:
       - "kg-extract-relationships"
       - "-p"
@@ -139,7 +169,7 @@ services:
     restart: on-failure:100
     
   vector-write:
-    image: docker.io/trustgraph/trustgraph-flow:0.3.3
+    image: docker.io/trustgraph/trustgraph-flow:0.4.1
     command:
       - "vector-write-milvus"
       - "-p"
@@ -149,7 +179,7 @@ services:
     restart: on-failure:100
     
   graph-write:
-    image: docker.io/trustgraph/trustgraph-flow:0.3.3
+    image: docker.io/trustgraph/trustgraph-flow:0.4.1
     command:
       - "graph-write-cassandra"
       - "-p"
@@ -159,7 +189,7 @@ services:
     restart: on-failure:100
     
   llm:
-    image: docker.io/trustgraph/trustgraph-flow:0.3.3
+    image: docker.io/trustgraph/trustgraph-flow:0.4.1
     command:
       - "llm-vertexai-text"
       - "-p"
@@ -173,7 +203,7 @@ services:
     restart: on-failure:100
 
   graph-rag:
-    image: docker.io/trustgraph/trustgraph-flow:0.3.3
+    image: docker.io/trustgraph/trustgraph-flow:0.4.1
     command:
       - "graph-rag"
       - "-p"