From 8c3ac009dae3f0e892eb357710c169a973949397 Mon Sep 17 00:00:00 2001
From: Cyber MacGeddon <cybermaggedon@gmail.com>
Date: Mon, 30 Sep 2024 12:00:20 +0100
Subject: [PATCH] Separate Prom metrics, different processors as different jobs

---
 prometheus/prometheus.yml  | 113 +++++++++++++++++++++++++++++++++++--
 scripts/tg-processor-state |   2 +-
 2 files changed, 109 insertions(+), 6 deletions(-)

diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml
index d1455738..c74f5df3 100644
--- a/prometheus/prometheus.yml
+++ b/prometheus/prometheus.yml
@@ -14,31 +14,134 @@ scrape_configs:
   # The job name is added as a label `job=<job_name>` to any timeseries
   # scraped from this config.
 
-  - job_name: 'trustgraph'
-
-    # Override the global default and scrape targets from this job every
-    # 5 seconds.
+  - job_name: 'pdf-decoder'
     scrape_interval: 5s
-
     static_configs:
       - targets:
         - 'pdf-decoder:8000'
+
+  - job_name: 'chunker'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'chunker:8000'
+
+
+  - job_name: 'vectorize'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'vectorize:8000'
+
+
+  - job_name: 'embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'embeddings:8000'
+
+
+  - job_name: 'kg-extract-definitions'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'kg-extract-definitions:8000'
+
+
+  - job_name: 'kg-extract-topics'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'kg-extract-topics:8000'
+
+
+  - job_name: 'kg-extract-relationships'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'kg-extract-relationships:8000'
+
+
+  - job_name: 'metering'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'metering:8000'
+
+
+  - job_name: 'metering-rag'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'metering-rag:8000'
+
+
+  - job_name: 'store-graph-embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'store-graph-embeddings:8000'
+
+
+  - job_name: 'store-triples'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'store-triples:8000'
+
+
+  - job_name: 'text-completion'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'text-completion:8000'
+
+
+  - job_name: 'text-completion-rag'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'text-completion-rag:8000'
+
+
+  - job_name: 'graph-rag'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'graph-rag:8000'
+
+
+  - job_name: 'prompt'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'prompt:8000'
+
+
+  - job_name: 'prompt-rag'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'prompt-rag:8000'
+
+
+  - job_name: 'query-graph-embeddings'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'query-graph-embeddings:8000'
+
+
+  - job_name: 'query-triples'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'query-triples:8000'
+
+
+  - job_name: 'pulsar'
+    scrape_interval: 5s
+    static_configs:
+      - targets:
         - 'pulsar:8080'
 
diff --git a/scripts/tg-processor-state b/scripts/tg-processor-state
index f9945aac..1b9399c7 100755
--- a/scripts/tg-processor-state
+++ b/scripts/tg-processor-state
@@ -11,7 +11,7 @@ obj = resp.json()
 
 tbl = [
     [
-        m["metric"]["instance"],
+        m["metric"]["job"],
         "running" if int(m["value"][1]) > 0 else "down"
     ]
     for m in obj["data"]["result"]