apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-20 15:28:07 +02:00
Code Issues Projects Releases Packages Wiki Activity

update dashboard

Browse source
This commit is contained in:
Adil Hafeez 2024-10-18 15:07:23 -07:00
parent 0e50e61a87
commit 87f5efd790
2 changed files with 487 additions and 367 deletions
Download patch file Download diff file Expand all files Collapse all files

740
arch/envoy.template.yaml
View file

@ -1,361 +1,379 @@
{ admin:
"annotations": { address:
"list": [ socket_address: { address: 0.0.0.0, port_value: 9901 }
{ static_resources:
"builtIn": 1, listeners:
"datasource": { - name: arch_listener_http
"type": "grafana", address:
"uid": "-- Grafana --" socket_address:
}, address: 0.0.0.0
"enable": true, port_value: 10000
"hide": true, traffic_direction: INBOUND
"iconColor": "rgba(0, 211, 255, 1)", filter_chains:
"name": "Annotations & Alerts", - filters:
"type": "dashboard" - name: envoy.filters.network.http_connection_manager
} typed_config:
] "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
}, {% if "random_sampling" in arch_tracing and arch_tracing["random_sampling"] > 0 %}
"editable": true, generate_request_id: true
"fiscalYearStartMonth": 0, tracing:
"graphTooltip": 1, provider:
"links": [], name: envoy.tracers.opentelemetry
"panels": [ typed_config:
{ "@type": type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig
"datasource": { grpc_service:
"default": true, envoy_grpc:
"type": "prometheus", cluster_name: opentelemetry_collector
"uid": "PBFA97CFB590B2093" timeout: 0.250s
}, service_name: arch
"fieldConfig": { random_sampling:
"defaults": { value: {{ arch_tracing.random_sampling }}
"color": { {% endif %}
"mode": "palette-classic" stat_prefix: arch_listener_http
}, codec_type: AUTO
"custom": { scheme_header_transformation:
"axisBorderShow": false, scheme_to_overwrite: https
"axisCenteredZero": false, access_log:
"axisColorMode": "text", - name: envoy.access_loggers.file
"axisLabel": "", typed_config:
"axisPlacement": "auto", "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
"barAlignment": 0, path: "/var/log/access_ingress.log"
"barWidthFactor": 0.6, route_config:
"drawStyle": "bars", name: local_routes
"fillOpacity": 54, virtual_hosts:
"gradientMode": "none", - name: local_service
"hideFrom": { domains:
"legend": false, - "*"
"tooltip": false, routes:
"viz": false - match:
}, prefix: "/"
"insertNulls": false, route:
"lineInterpolation": "linear", auto_host_rewrite: true
"lineWidth": 1, cluster: arch_llm_listener
"pointSize": 5, timeout: 60s
"scaleDistribution": { http_filters:
"type": "linear" - name: envoy.filters.http.wasm
}, typed_config:
"showPoints": "auto", "@type": type.googleapis.com/udpa.type.v1.TypedStruct
"spanNulls": false, type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
"stacking": { value:
"group": "A", config:
"mode": "normal" name: "http_config"
}, root_id: prompt_gateway
"thresholdsStyle": { configuration:
"mode": "off" "@type": "type.googleapis.com/google.protobuf.StringValue"
} value: |
}, {{ arch_config | indent(32) }}
"mappings": [], vm_config:
"thresholds": { runtime: "envoy.wasm.runtime.v8"
"mode": "absolute", code:
"steps": [ local:
{ filename: "/etc/envoy/proxy-wasm-plugins/prompt_gateway.wasm"
"color": "green", - name: envoy.filters.http.router
"value": null typed_config:
}, "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
{
"color": "red", - name: arch_internal
"value": 80 address:
} socket_address:
] address: 0.0.0.0
} port_value: 11000
}, traffic_direction: OUTBOUND
"overrides": [] filter_chains:
}, - filters:
"gridPos": { - name: envoy.filters.network.http_connection_manager
"h": 8, typed_config:
"w": 12, "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
"x": 0, {% if "random_sampling" in arch_tracing and arch_tracing["random_sampling"] > 0 %}
"y": 0 generate_request_id: true
}, tracing:
"id": 1, provider:
"options": { name: envoy.tracers.opentelemetry
"legend": { typed_config:
"calcs": [], "@type": type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig
"displayMode": "list", grpc_service:
"placement": "bottom", envoy_grpc:
"showLegend": true cluster_name: opentelemetry_collector
}, timeout: 0.250s
"tooltip": { service_name: arch
"mode": "single", random_sampling:
"sort": "none" value: {{ arch_tracing.random_sampling }}
} {% endif %}
}, stat_prefix: arch_internal
"targets": [ codec_type: AUTO
{ scheme_header_transformation:
"datasource": { scheme_to_overwrite: https
"type": "prometheus", access_log:
"uid": "PBFA97CFB590B2093" - name: envoy.access_loggers.file
}, typed_config:
"disableTextWrap": false, "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
"editorMode": "code", path: "/var/log/access_internal.log"
"expr": "avg(rate(envoy_cluster_external_upstream_rq_time_sum{envoy_cluster_name!=\"openai\",envoy_cluster_name!=\"arch_llm_listener\"}[1m]) / rate(envoy_cluster_external_upstream_rq_time_count[1m])) by (envoy_cluster_name)", route_config:
"fullMetaSearch": false, name: local_routes
"hide": false, virtual_hosts:
"includeNullMetadata": true, - name: local_service
"instant": false, domains:
"legendFormat": "__auto", - "*"
"range": true, routes:
"refId": "A",
"useBackend": false {% for internal_clustrer in ["embeddings", "zeroshot", "guard", "arch_fc", "hallucination"] %}
} - match:
], prefix: "/"
"title": "request latency - internal (ms)", headers:
"type": "timeseries" - name: "x-arch-upstream"
}, string_match:
{ exact: {{ internal_clustrer }}
"datasource": { route:
"default": true, auto_host_rewrite: true
"type": "prometheus", cluster: {{ internal_clustrer }}
"uid": "PBFA97CFB590B2093" timeout: 60s
}, {% endfor %}
"fieldConfig": {
"defaults": { {% for _, cluster in arch_clusters.items() %}
"color": { - match:
"mode": "palette-classic" prefix: "/"
}, headers:
"custom": { - name: "x-arch-upstream"
"axisBorderShow": false, string_match:
"axisCenteredZero": false, exact: {{ cluster.name }}
"axisColorMode": "text", route:
"axisLabel": "", auto_host_rewrite: true
"axisPlacement": "auto", cluster: {{ cluster.name }}
"barAlignment": 0, timeout: 60s
"barWidthFactor": 0.6, {% endfor %}
"drawStyle": "line", http_filters:
"fillOpacity": 0, - name: envoy.filters.http.router
"gradientMode": "none", typed_config:
"hideFrom": { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
"legend": false,
"tooltip": false, - name: arch_listener_llm
"viz": false address:
}, socket_address:
"insertNulls": false, address: 0.0.0.0
"lineInterpolation": "linear", port_value: 12000
"lineWidth": 1, filter_chains:
"pointSize": 5, - filters:
"scaleDistribution": { - name: envoy.filters.network.http_connection_manager
"type": "linear" typed_config:
}, "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
"showPoints": "auto", {% if "random_sampling" in arch_tracing and arch_tracing["random_sampling"] > 0 %}
"spanNulls": false, generate_request_id: true
"stacking": { tracing:
"group": "A", provider:
"mode": "none" name: envoy.tracers.opentelemetry
}, typed_config:
"thresholdsStyle": { "@type": type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig
"mode": "off" grpc_service:
} envoy_grpc:
}, cluster_name: opentelemetry_collector
"mappings": [], timeout: 0.250s
"thresholds": { service_name: arch
"mode": "absolute", random_sampling:
"steps": [ value: {{ arch_tracing.random_sampling }}
{ {% endif %}
"color": "green", stat_prefix: arch_listener_http
"value": null codec_type: AUTO
}, scheme_header_transformation:
{ scheme_to_overwrite: https
"color": "red", access_log:
"value": 80 - name: envoy.access_loggers.file
} typed_config:
] "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
} path: "/var/log/access_llm.log"
}, route_config:
"overrides": [] name: local_routes
}, virtual_hosts:
"gridPos": { - name: local_service
"h": 8, domains:
"w": 12, - "*"
"x": 12, routes:
"y": 0 {% for provider in arch_llm_providers %}
}, - match:
"id": 4, prefix: "/"
"options": { headers:
"legend": { - name: "x-arch-llm-provider"
"calcs": [], string_match:
"displayMode": "list", exact: {{ provider.name }}
"placement": "bottom", route:
"showLegend": true auto_host_rewrite: true
}, cluster: {{ provider.provider }}
"tooltip": { timeout: 60s
"mode": "single", {% endfor %}
"sort": "none" - match:
} prefix: "/"
}, direct_response:
"targets": [ status: 400
{ body:
"datasource": { inline_string: "x-arch-llm-provider header not set, llm gateway cannot perform routing\n"
"type": "prometheus", http_filters:
"uid": "PBFA97CFB590B2093" - name: envoy.filters.http.wasm
}, typed_config:
"disableTextWrap": false, "@type": type.googleapis.com/udpa.type.v1.TypedStruct
"editorMode": "code", type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
"expr": "avg(rate(envoy_cluster_external_upstream_rq_time_sum{envoy_cluster_name!=\"hallucination\", envoy_cluster_name!=\"zeroshot\", envoy_cluster_name!=\"embeddings\", envoy_cluster_name!=\"arch_fc\", envoy_cluster_name!=\"api_server\"}[1m]) / rate(envoy_cluster_external_upstream_rq_time_count[1m])) by (envoy_cluster_name)", value:
"fullMetaSearch": false, config:
"hide": false, name: "http_config"
"includeNullMetadata": true, root_id: llm_gateway
"instant": false, configuration:
"legendFormat": "__auto", "@type": "type.googleapis.com/google.protobuf.StringValue"
"range": true, value: |
"refId": "A", {{ arch_llm_config | indent(32) }}
"useBackend": false vm_config:
} runtime: "envoy.wasm.runtime.v8"
], code:
"title": "request latency - external (ms)", local:
"type": "timeseries" filename: "/etc/envoy/proxy-wasm-plugins/llm_gateway.wasm"
}, - name: envoy.filters.http.router
{ typed_config:
"datasource": { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
"default": true,
"type": "prometheus", clusters:
"uid": "PBFA97CFB590B2093" - name: openai
}, connect_timeout: 5s
"fieldConfig": { type: LOGICAL_DNS
"defaults": { dns_lookup_family: V4_ONLY
"color": { lb_policy: ROUND_ROBIN
"mode": "palette-classic" load_assignment:
}, cluster_name: openai
"custom": { endpoints:
"axisBorderShow": false, - lb_endpoints:
"axisCenteredZero": false, - endpoint:
"axisColorMode": "text", address:
"axisLabel": "", socket_address:
"axisPlacement": "auto", address: api.openai.com
"barAlignment": 0, port_value: 443
"barWidthFactor": 0.6, hostname: "api.openai.com"
"drawStyle": "line", transport_socket:
"fillOpacity": 0, name: envoy.transport_sockets.tls
"gradientMode": "none", typed_config:
"hideFrom": { "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
"legend": false, sni: api.openai.com
"tooltip": false, common_tls_context:
"viz": false tls_params:
}, tls_minimum_protocol_version: TLSv1_2
"insertNulls": false, tls_maximum_protocol_version: TLSv1_3
"lineInterpolation": "linear", - name: mistral
"lineWidth": 1, connect_timeout: 5s
"pointSize": 5, type: LOGICAL_DNS
"scaleDistribution": { dns_lookup_family: V4_ONLY
"type": "linear" lb_policy: ROUND_ROBIN
}, load_assignment:
"showPoints": "auto", cluster_name: mistral
"spanNulls": false, endpoints:
"stacking": { - lb_endpoints:
"group": "A", - endpoint:
"mode": "none" address:
}, socket_address:
"thresholdsStyle": { address: api.mistral.ai
"mode": "off" port_value: 443
} hostname: "api.mistral.ai"
}, transport_socket:
"mappings": [], name: envoy.transport_sockets.tls
"thresholds": { typed_config:
"mode": "absolute", "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
"steps": [ sni: api.mistral.ai
{ {% for internal_clustrer in ["embeddings", "zeroshot", "guard", "arch_fc", "hallucination"] %}
"color": "green", - name: {{ internal_clustrer }}
"value": null connect_timeout: 5s
}, type: STRICT_DNS
{ dns_lookup_family: V4_ONLY
"color": "red", lb_policy: ROUND_ROBIN
"value": 80 load_assignment:
} cluster_name: {{ internal_clustrer }}
] endpoints:
} - lb_endpoints:
}, - endpoint:
"overrides": [] address:
}, socket_address:
"gridPos": { address: host.docker.internal
"h": 8, port_value: 51000
"w": 12, hostname: {{ internal_clustrer }}
"x": 0, {% endfor %}
"y": 8 - name: mistral_7b_instruct
}, connect_timeout: 5s
"id": 3, type: STRICT_DNS
"options": { dns_lookup_family: V4_ONLY
"legend": { lb_policy: ROUND_ROBIN
"calcs": [], load_assignment:
"displayMode": "list", cluster_name: mistral_7b_instruct
"placement": "bottom", endpoints:
"showLegend": true - lb_endpoints:
}, - endpoint:
"tooltip": { address:
"mode": "single", socket_address:
"sort": "none" address: mistral_7b_instruct
} port_value: 10001
}, hostname: "mistral_7b_instruct"
"targets": [ {% for _, cluster in arch_clusters.items() %}
{ - name: {{ cluster.name }}
"datasource": { {% if cluster.connect_timeout -%}
"type": "prometheus", connect_timeout: {{ cluster.connect_timeout }}
"uid": "PBFA97CFB590B2093" {% else -%}
}, connect_timeout: 5s
"disableTextWrap": false, {% endif -%}
"editorMode": "code", type: LOGICAL_DNS
"expr": "avg(rate(envoy_cluster_internal_upstream_rq_completed{envoy_cluster_name !=\"opentelemetry_collector\"}[1m])) by (envoy_cluster_name)", dns_lookup_family: V4_ONLY
"fullMetaSearch": false, lb_policy: ROUND_ROBIN
"includeNullMetadata": true, load_assignment:
"instant": false, cluster_name: {{ cluster.name }}
"legendFormat": "__auto", endpoints:
"range": true, - lb_endpoints:
"refId": "A", - endpoint:
"useBackend": false address:
}, socket_address:
{ address: {{ cluster.endpoint }}
"datasource": { port_value: {{ cluster.port }}
"type": "prometheus", hostname: {{ cluster.name }}
"uid": "PBFA97CFB590B2093" {% endfor %}
}, - name: arch_internal
"disableTextWrap": false, connect_timeout: 5s
"editorMode": "code", type: LOGICAL_DNS
"expr": "avg(rate(envoy_cluster_external_upstream_rq_completed{envoy_cluster_name !=\"opentelemetry_collector\"}[1m])) by (envoy_cluster_name)", dns_lookup_family: V4_ONLY
"fullMetaSearch": false, lb_policy: ROUND_ROBIN
"hide": false, load_assignment:
"includeNullMetadata": true, cluster_name: arch_internal
"instant": false, endpoints:
"legendFormat": "__auto", - lb_endpoints:
"range": true, - endpoint:
"refId": "B", address:
"useBackend": false socket_address:
} address: 0.0.0.0
], port_value: 11000
"title": "Upstream request count", hostname: arch_internal
"type": "timeseries"
} - name: arch_llm_listener
], connect_timeout: 5s
"schemaVersion": 39, type: LOGICAL_DNS
"tags": [], dns_lookup_family: V4_ONLY
"templating": { lb_policy: ROUND_ROBIN
"list": [] load_assignment:
}, cluster_name: arch_llm_listener
"time": { endpoints:
"from": "now-15m", - lb_endpoints:
"to": "now" - endpoint:
}, address:
"timepicker": {}, socket_address:
"timezone": "browser", address: 0.0.0.0
"title": "Intelligent Gateway Overview", port_value: 12000
"uid": "adt6uhx5lk8aob", hostname: arch_llm_listener
"version": 2,
"weekStart": "" {% if "random_sampling" in arch_tracing and arch_tracing["random_sampling"] > 0 %}
} - name: opentelemetry_collector
type: STRICT_DNS
dns_lookup_family: V4_ONLY
lb_policy: ROUND_ROBIN
typed_extension_protocol_options:
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
"@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
explicit_http_config:
http2_protocol_options: {}
load_assignment:
cluster_name: opentelemetry_collector
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: host.docker.internal
port_value: 4317
{% endif %}

114
demos/function_calling/grafana/dashboards/envoy_overview.json
View file

@ -87,7 +87,7 @@
"x": 0, "x": 0,
"y": 0 "y": 0
}, },
"id": 1, "id": 5,
"options": { "options": {
"legend": { "legend": {
"calcs": [], "calcs": [],
@ -108,7 +108,7 @@
}, },
"disableTextWrap": false, "disableTextWrap": false,
"editorMode": "code", "editorMode": "code",
"expr": "avg(rate(envoy_cluster_external_upstream_rq_time_sum{envoy_cluster_name!=\"openai\",envoy_cluster_name!=\"arch_llm_listener\"}[1m]) / rate(envoy_cluster_external_upstream_rq_time_count[1m])) by (envoy_cluster_name)", "expr": "avg(rate(envoy_cluster_external_upstream_rq_time_sum{envoy_cluster_name=\"api_server\"}[1m]) / rate(envoy_cluster_external_upstream_rq_time_count[1m])) by (envoy_cluster_name)",
"fullMetaSearch": false, "fullMetaSearch": false,
"hide": false, "hide": false,
"includeNullMetadata": true, "includeNullMetadata": true,
@ -119,7 +119,7 @@
"useBackend": false "useBackend": false
} }
], ],
"title": "request latency - internal (ms)", "title": "request latency - developer api server (ms)",
"type": "timeseries" "type": "timeseries"
}, },
{ {
@ -224,6 +224,108 @@
"title": "request latency - external (ms)", "title": "request latency - external (ms)",
"type": "timeseries" "type": "timeseries"
}, },
{
"datasource": {
"default": true,
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
"fillOpacity": 54,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "avg(rate(envoy_cluster_external_upstream_rq_time_sum{envoy_cluster_name!=\"openai\",envoy_cluster_name!=\"arch_llm_listener\",envoy_cluster_name!=\"api_server\"}[1m]) / rate(envoy_cluster_external_upstream_rq_time_count[1m])) by (envoy_cluster_name)",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "request latency - internal arch services (ms)",
"type": "timeseries"
},
{ {
"datasource": { "datasource": {
"default": true, "default": true,
@ -289,7 +391,7 @@
"h": 8, "h": 8,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 8 "y": 16
}, },
"id": 3, "id": 3,
"options": { "options": {
@ -339,7 +441,7 @@
"useBackend": false "useBackend": false
} }
], ],
"title": "Upstream request count", "title": "Request count (per sec)",
"type": "timeseries" "type": "timeseries"
} }
], ],
@ -356,6 +458,6 @@
"timezone": "browser", "timezone": "browser",
"title": "Intelligent Gateway Overview", "title": "Intelligent Gateway Overview",
"uid": "adt6uhx5lk8aob", "uid": "adt6uhx5lk8aob",
"version": 2, "version": 14,
"weekStart": "" "weekStart": ""
} }