apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-04-27 09:46:28 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert "Revert "Add support for multiple LLM Providers (#60)"" (#83)

Browse source 
* Revert "Revert "Add support for multiple LLM Providers (#60)""

This reverts commit 43d6bc80e9.

* wip

Signed-off-by: José Ulises Niño Rivera <junr03@users.noreply.github.com>

* Revert "wip"

This reverts commit 7c4dde5d1f.

* fix parameter name

Signed-off-by: José Ulises Niño Rivera <junr03@users.noreply.github.com>

* force use openai

---------

Signed-off-by: José Ulises Niño Rivera <junr03@users.noreply.github.com>
Co-authored-by: Adil Hafeez <adil@katanemo.com>
This commit is contained in:
José Ulises Niño Rivera 2024-09-26 00:15:17 -06:00 • committed by GitHub
parent 370f3bb2c5
commit 9ea6bb0d73
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 457 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

55
envoyfilter/envoy.template.yaml
View file

@ -19,15 +19,6 @@ static_resources:
route_config:
name: local_routes
virtual_hosts:
- name: openai
domains:
- "api.openai.com"
routes:
- match:
prefix: "/"
route:
auto_host_rewrite: true
cluster: openai
- name: local_service
domains:
- "*"
@ -48,28 +39,23 @@ static_resources:
- match:
prefix: "/v1/chat/completions"
headers:
name: "Authorization"
present_match: true
- name: "x-bolt-llm-provider"
string_match:
exact: openai
route:
auto_host_rewrite: true
cluster: openai
timeout: 60s
- match:
prefix: "/v1/chat/completions"
headers:
- name: "x-bolt-llm-provider"
string_match:
exact: mistral
route:
auto_host_rewrite: true
cluster: mistral_7b_instruct
cluster: mistral
timeout: 60s
- match:
prefix: "/embeddings"
route:
cluster: model_server
- match:
prefix: "/"
direct_response:
status: 200
body:
inline_string: "Inspect the HTTP header: custom-header.\n"
http_filters:
- name: envoy.filters.http.wasm
typed_config:
@ -122,6 +108,31 @@ static_resources:
tls_params:
tls_minimum_protocol_version: TLSv1_2
tls_maximum_protocol_version: TLSv1_3
- name: mistral
connect_timeout: 5s
dns_lookup_family: V4_ONLY
type: LOGICAL_DNS
lb_policy: ROUND_ROBIN
typed_extension_protocol_options:
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
"@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
explicit_http_config:
http2_protocol_options: {}
load_assignment:
cluster_name: mistral
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: api.mistral.ai
port_value: 443
hostname: "api.mistral.ai"
transport_socket:
name: envoy.transport_sockets.tls
typed_config:
"@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
sni: api.mistral.ai
- name: model_server
connect_timeout: 5s
type: STRICT_DNS