apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-04-30 11:26:27 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Introduce brightstaff a new terminal service for llm routing (#477)

Browse source
This commit is contained in:
Adil Hafeez 2025-05-19 09:59:22 -07:00 • committed by GitHub
parent 1f95fac4af
commit 27c0f2fdce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
36 changed files with 2817 additions and 150 deletions
Download patch file Download diff file Expand all files Collapse all files

44
arch/envoy.template.yaml
View file

@ -328,11 +328,15 @@ static_resources:
domains:
- "*"
routes:
- match:
prefix: "/healthz"
direct_response:
status: 200
- match:
prefix: "/"
route:
auto_host_rewrite: true
cluster: arch_listener_llm
cluster: bright_staff
timeout: {{ llm_gateway_listener.timeout }}
http_filters:
- name: envoy.filters.http.router
@ -380,12 +384,6 @@ static_resources:
domains:
- "*"
routes:
- match:
prefix: "/healthz"
route:
auto_host_rewrite: true
cluster: openai
timeout: 60s
{% for provider in arch_llm_providers %}
# if endpoint is set then use custom cluster for upstream llm
{% if provider.endpoint %}
@ -615,6 +613,38 @@ static_resources:
port_value: 11000
hostname: arch_internal
- name: bright_staff
connect_timeout: 0.5s
type: LOGICAL_DNS
dns_lookup_family: V4_ONLY
lb_policy: ROUND_ROBIN
load_assignment:
cluster_name: bright_staff
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: 0.0.0.0
port_value: 9091
hostname: localhost
- name: router_model_host
connect_timeout: 0.5s
type: LOGICAL_DNS
dns_lookup_family: V4_ONLY
lb_policy: ROUND_ROBIN
load_assignment:
cluster_name: router_model_host
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: 34.30.16.38
port_value: 8000
hostname: router_model_host
- name: arch_prompt_gateway_listener
connect_timeout: 0.5s
type: LOGICAL_DNS