trustgraph/ts/deploy/grafana/dashboards/llm-metrics.json

{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "grafana",
          "uid": "-- Grafana --"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "panels": [
    {
      "title": "LLM Request Latency by Provider",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "tg-prometheus"
      },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
      "id": 1,
      "targets": [
        {
          "expr": "histogram_quantile(0.50, sum(rate(tg_consumer_request_duration_seconds_bucket{job=~\".*text-completion.*\"}[5m])) by (le, job))",
          "legendFormat": "{{job}} p50",
          "refId": "A"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(tg_consumer_request_duration_seconds_bucket{job=~\".*text-completion.*\"}[5m])) by (le, job))",
          "legendFormat": "{{job}} p95",
          "refId": "B"
        },
        {
          "expr": "histogram_quantile(0.99, sum(rate(tg_consumer_request_duration_seconds_bucket{job=~\".*text-completion.*\"}[5m])) by (le, job))",
          "legendFormat": "{{job}} p99",
          "refId": "C"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "latency",
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 5,
            "gradientMode": "scheme",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": { "type": "linear" },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": { "group": "A", "mode": "none" },
            "thresholdsStyle": { "mode": "off" }
          },
          "unit": "s",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "yellow", "value": 5 },
              { "color": "red", "value": 30 }
            ]
          }
        },
        "overrides": []
      },
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
        "tooltip": { "mode": "multi", "sort": "desc" }
      }
    },
    {
      "title": "Token Usage (Input vs Output)",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "tg-prometheus"
      },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
      "id": 2,
      "targets": [
        {
          "expr": "sum(rate(tg_llm_input_tokens_total[5m])) by (job)",
          "legendFormat": "{{job}} input tokens/s",
          "refId": "A"
        },
        {
          "expr": "sum(rate(tg_llm_output_tokens_total[5m])) by (job)",
          "legendFormat": "{{job}} output tokens/s",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "tokens/s",
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 20,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": { "type": "linear" },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": { "group": "A", "mode": "normal" },
            "thresholdsStyle": { "mode": "off" }
          },
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null }
            ]
          }
        },
        "overrides": [
          {
            "matcher": { "id": "byRegexp", "options": ".*input.*" },
            "properties": [
              { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }
            ]
          },
          {
            "matcher": { "id": "byRegexp", "options": ".*output.*" },
            "properties": [
              { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }
            ]
          }
        ]
      },
      "options": {
        "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "bottom", "showLegend": true },
        "tooltip": { "mode": "multi", "sort": "desc" }
      }
    },
    {
      "title": "Rate Limit Events",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "tg-prometheus"
      },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
      "id": 3,
      "targets": [
        {
          "expr": "sum(rate(tg_consumer_rate_limit_total[5m])) by (job)",
          "legendFormat": "{{job}} rate limits/s",
          "refId": "A"
        },
        {
          "expr": "sum(increase(tg_consumer_rate_limit_total[1h])) by (job)",
          "legendFormat": "{{job}} total (1h)",
          "refId": "B"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "drawStyle": "bars",
            "fillOpacity": 50,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": { "type": "linear" },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": { "group": "A", "mode": "none" },
            "thresholdsStyle": { "mode": "line" }
          },
          "unit": "short",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "red", "value": 1 }
            ]
          }
        },
        "overrides": [
          {
            "matcher": { "id": "byRegexp", "options": ".*total.*" },
            "properties": [
              { "id": "custom.drawStyle", "value": "line" },
              { "id": "custom.axisPlacement", "value": "right" },
              { "id": "custom.fillOpacity", "value": 0 },
              { "id": "custom.lineWidth", "value": 2 }
            ]
          }
        ]
      },
      "options": {
        "legend": { "calcs": ["sum", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
        "tooltip": { "mode": "multi", "sort": "desc" }
      }
    },
    {
      "title": "Streaming Chunk Latency",
      "type": "timeseries",
      "datasource": {
        "type": "prometheus",
        "uid": "tg-prometheus"
      },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
      "id": 4,
      "targets": [
        {
          "expr": "histogram_quantile(0.50, sum(rate(tg_llm_stream_chunk_duration_seconds_bucket[5m])) by (le, job))",
          "legendFormat": "{{job}} chunk p50",
          "refId": "A"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(tg_llm_stream_chunk_duration_seconds_bucket[5m])) by (le, job))",
          "legendFormat": "{{job}} chunk p95",
          "refId": "B"
        },
        {
          "expr": "histogram_quantile(0.50, sum(rate(tg_llm_time_to_first_token_seconds_bucket[5m])) by (le, job))",
          "legendFormat": "{{job}} TTFT p50",
          "refId": "C"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(tg_llm_time_to_first_token_seconds_bucket[5m])) by (le, job))",
          "legendFormat": "{{job}} TTFT p95",
          "refId": "D"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "latency",
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 5,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": { "type": "linear" },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": { "group": "A", "mode": "none" },
            "thresholdsStyle": { "mode": "off" }
          },
          "unit": "s",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "yellow", "value": 0.5 },
              { "color": "red", "value": 2 }
            ]
          }
        },
        "overrides": [
          {
            "matcher": { "id": "byRegexp", "options": ".*TTFT.*" },
            "properties": [
              { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }
            ]
          }
        ]
      },
      "options": {
        "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true },
        "tooltip": { "mode": "multi", "sort": "desc" }
      }
    }
  ],
  "schemaVersion": 39,
  "tags": ["trustgraph", "llm"],
  "templating": {
    "list": []
  },
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "browser",
  "title": "TrustGraph - LLM Performance",
  "uid": "tg-llm-metrics",
  "version": 1
}