apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-05 14:45:15 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add function calling support using bolt-fc-1b (#35)

Browse source
This commit is contained in:
Adil Hafeez 2024-09-10 14:24:46 -07:00 • committed by GitHub
parent fdfad87347
commit 7b5203a2ce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
39 changed files with 1763 additions and 416 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -11,3 +11,6 @@ qdrant_data
generated
.DS_Store
*.gguf
venv
demos/function-calling/ollama/models/
demos/function-calling/ollama/id_ed*

2
.gitmodules vendored
View file

@ -1,3 +1,3 @@
[submodule "open-message-format"]
path = open-message-format
url = git@github.com:open-llm-initiative/open-message-format.git
url = git@github.com:open-llm-initiative/open-message-format-private.git

30
chatbot-ui/app/run.py
View file

@ -15,9 +15,14 @@ load_dotenv()
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
CHAT_COMPLETION_ENDPOINT = os.getenv("CHAT_COMPLETION_ENDPOINT", "https://api.openai.com/v1/chat/completions")
class Message(BaseModel):
role: str
content: str
# model is additional state we maintin on client side so that bolt gateway can know which model responded to user prompt
model: str
resolver: str
async def make_completion(messages:List[Message], nb_retries:int=3, delay:int=120) -> Optional[str]:
"""
@ -52,7 +57,20 @@ async def make_completion(messages:List[Message], nb_retries:int=3, delay:int=12
)
logger.debug(f"Status Code : {resp.status_code}")
if resp.status_code == 200:
return resp.json()["choices"][0]["message"]["content"]
resp_json = resp.json()
model = resp_json["model"]
msg = {}
msg["role"] = "assistant"
msg["model"] = model
if "resolver_name" in resp_json:
msg["resolver"] = resp_json["resolver_name"]
if "choices" in resp_json:
msg["content"] = resp_json["choices"][0]["message"]["content"]
return msg
elif "message" in resp_json:
msg["content"] = resp_json["message"]["content"]
return msg
keep_loop = False
else:
logger.warning(resp.content)
keep_loop = False
@ -70,20 +88,22 @@ async def predict(input, history):
"""
history.append({"role": "user", "content": input})
response = await make_completion(history)
print(response)
if response is not None:
history.append({"role": "assistant", "content": response})
history.append(response)
messages = [(history[i]["content"], history[i+1]["content"]) for i in range(0, len(history)-1, 2)]
return messages, history
"""
Gradio Blocks low-level API that allows to create custom web applications (here our chat app)
"""
with gr.Blocks() as demo:
# with fill_height=true the chatbot to fill the height of the page
with gr.Blocks(fill_height=True, css="footer {visibility: hidden}") as demo:
logger.info("Starting Demo...")
chatbot = gr.Chatbot(label="WebGPT")
chatbot = gr.Chatbot(label="Bolt Chatbot", scale=1)
state = gr.State([])
with gr.Row():
txt = gr.Textbox(show_label=False, placeholder="Enter text and press enter")
txt = gr.Textbox(show_label=False, placeholder="Enter text and press enter", scale=1)
txt.submit(predict, [txt, state], [chatbot, state])
demo.launch(server_name="0.0.0.0", server_port=8080)

3
chatbot-ui/requirements.txt
View file

@ -1,6 +1,7 @@
gradio==4.39.0
gradio==4.43.0
async_timeout==4.0.3
loguru==0.7.2
asyncio==3.4.3
httpx==0.27.0
python-dotenv==1.0.1
pydantic==2.8.2

25
demos/function-calling/Bolt-FC-1B-Q3_K_L.model_file Normal file
View file

@ -0,0 +1,25 @@
FROM Bolt-Function-Calling-1B-Q3_K_L.gguf
# Set the size of the context window used to generate the next token
# PARAMETER num_ctx 16384
PARAMETER num_ctx 4096
# Set parameters for response generation
PARAMETER num_predict 1024
PARAMETER temperature 0.1
PARAMETER top_p 0.5
PARAMETER top_k 32022
PARAMETER repeat_penalty 1.0
PARAMETER stop "<|EOT|>"
# Set the random number seed to use for generation
PARAMETER seed 42
# Set the prompt template to be passed into the model
TEMPLATE """{{ if .System }}<｜begin▁of▁sentence｜>
{{ .System }}
{{ end }}{{ if .Prompt }}### Instruction:
{{ .Prompt }}
{{ end }}### Response:
{{ .Response }}
<|EOT|>"""

24
demos/function-calling/Bolt-FC-1B-Q4_K_M.model_file Normal file
View file

@ -0,0 +1,24 @@
FROM Bolt-Function-Calling-1B-Q4_K_M.gguf
# Set the size of the context window used to generate the next token
PARAMETER num_ctx 4096
# Set parameters for response generation
PARAMETER num_predict 1024
PARAMETER temperature 0.1
PARAMETER top_p 0.5
PARAMETER top_k 32022
PARAMETER repeat_penalty 1.0
PARAMETER stop "<|EOT|>"
# Set the random number seed to use for generation
PARAMETER seed 42
# Set the prompt template to be passed into the model
TEMPLATE """{{ if .System }}<｜begin▁of▁sentence｜>
{{ .System }}
{{ end }}{{ if .Prompt }}### Instruction:
{{ .Prompt }}
{{ end }}### Response:
{{ .Response }}
<|EOT|>"""

24
demos/function-calling/README.md Normal file
View file

@ -0,0 +1,24 @@
# Function calling
This demo shows how you can use intelligent prompt gateway to do function calling. This demo assumes you are using ollama running natively. If you want to run ollama running inside docker then please update ollama endpoint in docker-compose file.
# Startig the demo
1. Ensure that submodule is up to date
```sh
git submodule sync --recursive
```
1. Create `.env` file and set OpenAI key using env var `OPENAI_API_KEY`
1. Start services
```sh
docker compose up
```
1. Download Bolt-FC model. This demo assumes we have downloaded `Bolt-Function-Calling-1B:Q4_K_M` to local folder
2. Create model file in ollama repository
```sh
ollama create Bolt-Function-Calling-1B:Q4_K_M -f Bolt-FC-1B-Q4_K_M.model_file
```
3. Navigate to http://localhost:18080/
4. You can type in queries like "how is the weather in Seattle"
1. You can also ask follow up questions like "show me sunny days"
5. To see metrics navigate to "http://localhost:3000/" (use admin/grafana for login)
1. Open up dahsboard named "Intelligent Gateway Overview"
2. On this dashboard you can see reuqest latency and number of requests

115
demos/function-calling/docker-compose.yaml Normal file
View file

@ -0,0 +1,115 @@
services:
config-generator:
build:
context: ../../
dockerfile: config_generator/Dockerfile
volumes:
- ../../envoyfilter/envoy.template.yaml:/usr/src/app/envoy.template.yaml
- ./katanemo-config.yaml:/usr/src/app/katanemo-config.yaml
- ./generated:/usr/src/app/out
envoy:
build:
context: ../../
dockerfile: envoyfilter/Dockerfile
hostname: envoy
ports:
- "10000:10000"
- "19901:9901"
volumes:
- ./generated/envoy.yaml:/etc/envoy/envoy.yaml
- /etc/ssl/cert.pem:/etc/ssl/cert.pem
depends_on:
config-generator:
condition: service_completed_successfully
embeddingserver:
condition: service_healthy
environment:
- LOG_LEVEL=debug
embeddingserver:
build:
context: ../../embedding-server
dockerfile: Dockerfile
ports:
- "18081:80"
healthcheck:
test: ["CMD", "curl" ,"http://localhost:80/healthz"]
interval: 5s
retries: 20
volumes:
- ~/.cache/huggingface:/root/.cache/huggingface
functionresolver:
build:
context: ../../function_resolver
dockerfile: Dockerfile
ports:
- "18082:80"
healthcheck:
test: ["CMD", "curl" ,"http://localhost:80/healthz"]
interval: 5s
retries: 20
volumes:
- ~/.cache/huggingface:/root/.cache/huggingface
environment:
# use ollama endpoint that is hosted by host machine (no virtualization)
- OLLAMA_ENDPOINT=host.docker.internal
# uncomment following line to use ollama endpoint that is hosted by docker
# - OLLAMA_ENDPOINT=ollama
ollama:
image: ollama/ollama
container_name: ollama
volumes:
- ./ollama:/root/.ollama
restart: unless-stopped
ports:
- '11434:11434'
profiles:
- manual
qdrant:
image: qdrant/qdrant
hostname: vector-db
ports:
- 16333:6333
- 16334:6334
chatbot-ui:
build:
context: ../../chatbot-ui
dockerfile: Dockerfile
ports:
- "18080:8080"
environment:
- OPENAI_API_KEY=${OPENAI_API_KEY}
- CHAT_COMPLETION_ENDPOINT=http://envoy:10000/v1/chat/completions
prometheus:
image: prom/prometheus
container_name: prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yaml'
ports:
- 9090:9090
restart: unless-stopped
volumes:
- ./prometheus:/etc/prometheus
- ./prom_data:/prometheus
grafana:
image: grafana/grafana
container_name: grafana
ports:
- 3000:3000
restart: unless-stopped
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=grafana
volumes:
- ./grafana:/etc/grafana/provisioning/datasources
- ./grafana/dashboard.yaml:/etc/grafana/provisioning/dashboards/main.yaml
- ./grafana/dashboards:/var/lib/grafana/dashboards

12
demos/function-calling/grafana/dashboard.yaml Normal file
View file

@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: "Dashboard provider"
orgId: 1
type: file
disableDeletion: false
updateIntervalSeconds: 10
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
foldersFromFilesStructure: true

355
demos/function-calling/grafana/dashboards/envoy_overview.json Normal file
View file

@ -0,0 +1,355 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "avg(rate(envoy_cluster_internal_upstream_rq_time_sum[1m]) / rate(envoy_cluster_internal_upstream_rq_time_count[1m])) by (envoy_cluster_name)",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "request latency - internal (ms)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "avg(rate(envoy_cluster_external_upstream_rq_time_sum[1m]) / rate(envoy_cluster_external_upstream_rq_time_count[1m])) by (envoy_cluster_name)",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "request latency - external (ms)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "avg(rate(envoy_cluster_internal_upstream_rq_completed[1m])) by (envoy_cluster_name)",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A",
"useBackend": false
},
{
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "avg(rate(envoy_cluster_external_upstream_rq_completed[1m])) by (envoy_cluster_name)",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "B",
"useBackend": false
}
],
"title": "Upstream request count",
"type": "timeseries"
}
],
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-15m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Intelligent Gateway Overview",
"uid": "adt6uhx5lk8aob",
"version": 3,
"weekStart": ""
}

9
demos/function-calling/grafana/datasource.yaml Normal file
View file

@ -0,0 +1,9 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
url: http://prometheus:9090
isDefault: true
access: proxy
editable: true

41
demos/function-calling/katanemo-config.yaml Normal file
View file

@ -0,0 +1,41 @@
default_prompt_endpoint: "127.0.0.1"
load_balancing: "round_robin"
timeout_ms: 5000
# should not be here
embedding_provider:
name: "bge-large-en-v1.5"
model: "BAAI/bge-large-en-v1.5"
llm_providers:
- name: open-ai-gpt-4
api_key: $OPEN_AI_API_KEY
model: gpt-4
default: true
prompt_targets:
- type: function_resolver
name: weather_forecast
description: This function resolver provides weather forecast information for a given city.
few_shot_examples:
- what is the weather in New York?
- how is the weather in San Francisco?
- what is the forecast in Chicago?
parameters:
- name: city
required: true
description: The city for which the weather forecast is requested.
- name: days
description: The number of days for which the weather forecast is requested.
- name: units
description: The units in which the weather forecast is requested.
endpoint:
cluster: weatherhost
path: /weather
system_prompt: |
You are a helpful weather forecaster. Use weater data that is provided to you. Please following following guidelines when responding to user queries:
- Use farenheight for temperature
- Use miles per hour for wind speed

23
demos/function-calling/prometheus/prometheus.yaml Normal file
View file

@ -0,0 +1,23 @@
global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 15s
alerting:
alertmanagers:
- static_configs:
- targets: []
scheme: http
timeout: 10s
api_version: v1
scrape_configs:
- job_name: envoy
honor_timestamps: true
scrape_interval: 15s
scrape_timeout: 10s
metrics_path: /stats
scheme: http
static_configs:
- targets:
- envoy:9901
params:
format: ['prometheus']

1
demos/weather-forecast-local-llm/docker-compose.yaml
View file

@ -4,6 +4,7 @@ services:
context: ../../
dockerfile: config_generator/Dockerfile
volumes:
- ../../envoyfilter/envoy.template.yaml:/usr/src/app/envoy.template.yaml
- ./katanemo-config.yaml:/usr/src/app/katanemo-config.yaml
- ./generated:/usr/src/app/out
envoy:

1
demos/weather-forecast/docker-compose.yaml
View file

@ -4,6 +4,7 @@ services:
context: ../../
dockerfile: config_generator/Dockerfile
volumes:
- ../../envoyfilter/envoy.template.yaml:/usr/src/app/envoy.template.yaml
- ./katanemo-config.yaml:/usr/src/app/katanemo-config.yaml
- ./generated:/usr/src/app/out
envoy:

386
envoyfilter/Cargo.lock generated
View file

@ -70,9 +70,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.86"
version = "1.0.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8"
[[package]]
name = "arbitrary"
@ -82,13 +82,13 @@ checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110"
[[package]]
name = "async-trait"
version = "0.1.81"
version = "0.1.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107"
checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
]
[[package]]
@ -202,18 +202,19 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.6.1"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952"
checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
[[package]]
name = "cc"
version = "1.1.6"
version = "1.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f"
checksum = "a93fe60e2fc87b6ba2c117f67ae14f66e3fc7d6a1e612a25adb238cc980eadb3"
dependencies = [
"jobserver",
"libc",
"shlex",
]
[[package]]
@ -261,24 +262,24 @@ dependencies = [
[[package]]
name = "core-foundation-sys"
version = "0.8.6"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "cpp_demangle"
version = "0.4.3"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119"
checksum = "96e58d342ad113c2b878f16d5d034c03be492ae460cdbc02b7f0f2284d310c7d"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "cpufeatures"
version = "0.2.12"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0"
dependencies = [
"libc",
]
@ -550,9 +551,9 @@ dependencies = [
[[package]]
name = "fastrand"
version = "2.1.0"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
[[package]]
name = "fnv"
@ -737,9 +738,9 @@ dependencies = [
[[package]]
name = "h2"
version = "0.4.5"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab"
checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
dependencies = [
"atomic-waker",
"bytes",
@ -876,9 +877,9 @@ dependencies = [
[[package]]
name = "hyper-rustls"
version = "0.27.2"
version = "0.27.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155"
checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
dependencies = [
"futures-util",
"http",
@ -909,9 +910,9 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956"
checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9"
dependencies = [
"bytes",
"futures-channel",
@ -945,9 +946,9 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.2.6"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5"
dependencies = [
"equivalent",
"hashbrown 0.14.5",
@ -1025,9 +1026,9 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.69"
version = "0.3.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
dependencies = [
"wasm-bindgen",
]
@ -1046,9 +1047,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
[[package]]
name = "libc"
version = "0.2.155"
version = "0.2.158"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
[[package]]
name = "libm"
@ -1145,9 +1146,9 @@ dependencies = [
[[package]]
name = "mio"
version = "1.0.1"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4"
checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
dependencies = [
"hermit-abi 0.3.9",
"libc",
@ -1195,9 +1196,9 @@ checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
[[package]]
name = "object"
version = "0.36.2"
version = "0.36.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e"
checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a"
dependencies = [
"crc32fast",
"hashbrown 0.14.5",
@ -1245,7 +1246,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
]
[[package]]
@ -1286,7 +1287,7 @@ dependencies = [
"libc",
"redox_syscall",
"smallvec",
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@ -1318,7 +1319,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
]
[[package]]
@ -1359,9 +1360,12 @@ dependencies = [
[[package]]
name = "ppv-lite86"
version = "0.2.17"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
dependencies = [
"zerocopy",
]
[[package]]
name = "proc-macro-error"
@ -1422,9 +1426,9 @@ dependencies = [
[[package]]
name = "psm"
version = "0.1.21"
version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874"
checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205"
dependencies = [
"cc",
]
@ -1439,9 +1443,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.36"
version = "1.0.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
dependencies = [
"proc-macro2",
]
@ -1507,9 +1511,9 @@ dependencies = [
[[package]]
name = "redox_users"
version = "0.4.5"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891"
checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
dependencies = [
"getrandom",
"libredox",
@ -1560,9 +1564,9 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
[[package]]
name = "reqwest"
version = "0.12.5"
version = "0.12.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37"
checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63"
dependencies = [
"base64 0.22.1",
"bytes",
@ -1599,7 +1603,7 @@ dependencies = [
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"winreg",
"windows-registry",
]
[[package]]
@ -1631,9 +1635,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustix"
version = "0.38.34"
version = "0.38.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36"
dependencies = [
"bitflags 2.6.0",
"errno",
@ -1657,9 +1661,9 @@ dependencies = [
[[package]]
name = "rustls-pemfile"
version = "2.1.2"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d"
checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425"
dependencies = [
"base64 0.22.1",
"rustls-pki-types",
@ -1667,15 +1671,15 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
version = "1.7.0"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
[[package]]
name = "rustls-webpki"
version = "0.102.6"
version = "0.102.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e"
checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56"
dependencies = [
"ring",
"rustls-pki-types",
@ -1690,9 +1694,9 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "scc"
version = "2.1.5"
version = "2.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fadf67e3cf23f8b11a6c8c48a16cb2437381503615acd91094ec7b4686a5a53"
checksum = "aeb7ac86243095b70a7920639507b71d51a63390d1ba26c4f60a552fbb914a37"
dependencies = [
"sdd",
]
@ -1714,9 +1718,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "sdd"
version = "1.7.0"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85f05a494052771fc5bd0619742363b5e24e5ad72ab3111ec2e27925b8edc5f3"
checksum = "0495e4577c672de8254beb68d01a9b62d0e8a13c099edecdbedccce3223cd29f"
[[package]]
name = "security-framework"
@ -1752,31 +1756,32 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.204"
version = "1.0.210"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.204"
version = "1.0.210"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
]
[[package]]
name = "serde_json"
version = "1.0.120"
version = "1.0.128"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
]
@ -1837,7 +1842,7 @@ checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
]
[[package]]
@ -1851,6 +1856,12 @@ dependencies = [
"digest",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "slab"
version = "0.4.9"
@ -1961,9 +1972,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.72"
version = "2.0.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed"
dependencies = [
"proc-macro2",
"quote",
@ -1975,23 +1986,26 @@ name = "sync_wrapper"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
dependencies = [
"futures-core",
]
[[package]]
name = "system-configuration"
version = "0.5.1"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
dependencies = [
"bitflags 1.3.2",
"bitflags 2.6.0",
"core-foundation",
"system-configuration-sys",
]
[[package]]
name = "system-configuration-sys"
version = "0.5.0"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
dependencies = [
"core-foundation-sys",
"libc",
@ -1999,20 +2013,21 @@ dependencies = [
[[package]]
name = "target-lexicon"
version = "0.12.15"
version = "0.12.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2"
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
[[package]]
name = "tempfile"
version = "3.10.1"
version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
dependencies = [
"cfg-if 1.0.0",
"fastrand",
"once_cell",
"rustix",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@ -2050,7 +2065,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
]
[[package]]
@ -2085,9 +2100,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.39.1"
version = "1.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a"
checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998"
dependencies = [
"backtrace",
"bytes",
@ -2121,9 +2136,9 @@ dependencies = [
[[package]]
name = "tokio-util"
version = "0.7.11"
version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1"
checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a"
dependencies = [
"bytes",
"futures-core",
@ -2134,9 +2149,9 @@ dependencies = [
[[package]]
name = "toml"
version = "0.8.16"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c"
checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
dependencies = [
"serde",
"serde_spanned",
@ -2146,18 +2161,18 @@ dependencies = [
[[package]]
name = "toml_datetime"
version = "0.6.7"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db"
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.22.17"
version = "0.22.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16"
checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
dependencies = [
"indexmap",
"serde",
@ -2183,15 +2198,15 @@ dependencies = [
[[package]]
name = "tower-layer"
version = "0.3.2"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
version = "0.3.2"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
@ -2268,9 +2283,9 @@ checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
[[package]]
name = "unicode-xid"
version = "0.2.4"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a"
[[package]]
name = "unsafe-libyaml"
@ -2340,34 +2355,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.92"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
dependencies = [
"cfg-if 1.0.0",
"once_cell",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.92"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.42"
version = "0.4.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed"
dependencies = [
"cfg-if 1.0.0",
"js-sys",
@ -2377,9 +2393,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.92"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -2387,22 +2403,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.92"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.92"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
[[package]]
name = "wasm-encoder"
@ -2415,9 +2431,9 @@ dependencies = [
[[package]]
name = "wasm-encoder"
version = "0.214.0"
version = "0.216.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff694f02a8d7a50b6922b197ae03883fbf18cdb2ae9fbee7b6148456f5f44041"
checksum = "04c23aebea22c8a75833ae08ed31ccc020835b12a41999e58c31464271b94a88"
dependencies = [
"leb128",
]
@ -2541,7 +2557,7 @@ dependencies = [
"anyhow",
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
"wasmtime-component-util",
"wasmtime-wit-bindgen",
"wit-parser",
@ -2671,7 +2687,7 @@ checksum = "99c02af2e9dbeb427304d1a08787d70ed0dbfec1af2236616f84c9f1f03e7969"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
]
[[package]]
@ -2705,31 +2721,31 @@ dependencies = [
[[package]]
name = "wast"
version = "214.0.0"
version = "216.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "694bcdb24c49c8709bd8713768b71301a11e823923eee355d530f1d8d0a7f8e9"
checksum = "f7eb1f2eecd913fdde0dc6c3439d0f24530a98ac6db6cb3d14d92a5328554a08"
dependencies = [
"bumpalo",
"leb128",
"memchr",
"unicode-width",
"wasm-encoder 0.214.0",
"wasm-encoder 0.216.0",
]
[[package]]
name = "wat"
version = "1.214.0"
version = "1.216.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "347249eb56773fa728df2656cfe3a8c19437ded61a922a0b5e0839d9790e278e"
checksum = "ac0409090fb5154f95fb5ba3235675fd9e579e731524d63b6a2f653e1280c82a"
dependencies = [
"wast",
]
[[package]]
name = "web-sys"
version = "0.3.69"
version = "0.3.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0"
dependencies = [
"js-sys",
"wasm-bindgen",
@ -2757,7 +2773,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.59.0",
]
[[package]]
@ -2784,12 +2800,33 @@ dependencies = [
]
[[package]]
name = "windows-sys"
version = "0.48.0"
name = "windows-registry"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-targets 0.48.5",
"windows-result",
"windows-strings",
"windows-targets",
]
[[package]]
name = "windows-result"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-strings"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result",
"windows-targets",
]
[[package]]
@ -2798,22 +2835,16 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.48.5"
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows_aarch64_gnullvm 0.48.5",
"windows_aarch64_msvc 0.48.5",
"windows_i686_gnu 0.48.5",
"windows_i686_msvc 0.48.5",
"windows_x86_64_gnu 0.48.5",
"windows_x86_64_gnullvm 0.48.5",
"windows_x86_64_msvc 0.48.5",
"windows-targets",
]
[[package]]
@ -2822,46 +2853,28 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
@ -2874,48 +2887,24 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
@ -2924,23 +2913,13 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
version = "0.6.16"
version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b480ae9340fc261e6be3e95a1ba86d54ae3f9171132a73ce8d4bbaf68339507c"
checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
dependencies = [
"memchr",
]
[[package]]
name = "winreg"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5"
dependencies = [
"cfg-if 1.0.0",
"windows-sys 0.48.0",
]
[[package]]
name = "wit-parser"
version = "0.212.0"
@ -2965,6 +2944,7 @@ version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
"byteorder",
"zerocopy-derive",
]
@ -2976,7 +2956,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.72",
"syn 2.0.77",
]
[[package]]
@ -2996,18 +2976,18 @@ dependencies = [
[[package]]
name = "zstd-safe"
version = "7.2.0"
version = "7.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa"
checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.12+zstd.1.5.6"
version = "2.0.13+zstd.1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13"
checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
dependencies = [
"cc",
"pkg-config",

2
envoyfilter/Dockerfile
View file

@ -5,6 +5,7 @@ COPY envoyfilter/src /envoyfilter/src
COPY envoyfilter/Cargo.toml /envoyfilter/
COPY envoyfilter/Cargo.lock /envoyfilter/
COPY open-message-format /open-message-format
COPY public-types /public-types
RUN rustup -v target add wasm32-wasi
RUN cargo build --release --target wasm32-wasi
@ -14,3 +15,4 @@ FROM envoyproxy/envoy:v1.30-latest
COPY --from=builder /envoyfilter/target/wasm32-wasi/release/intelligent_prompt_gateway.wasm /etc/envoy/proxy-wasm-plugins/intelligent_prompt_gateway.wasm
COPY envoyfilter/envoy.yaml /etc/envoy.yaml
CMD ["envoy", "-c", "/etc/envoy/envoy.yaml"]
# CMD ["envoy", "-c", "/etc/envoy/envoy.yaml", "--log-level", "debug"]

1
envoyfilter/docker-compose.yaml
View file

@ -15,7 +15,6 @@ services:
embeddingserver:
condition: service_healthy
embeddingserver:
build:
context: ../embedding-server

27
envoyfilter/envoy.template.yaml
View file

@ -32,6 +32,19 @@ static_resources:
domains:
- "*"
routes:
- match:
prefix: "/mistral/v1/chat/completions"
route:
auto_host_rewrite: true
cluster: mistral_7b_instruct
timeout: 60s
- match:
prefix: "/bolt_fc_1b/v1/chat/completions"
route:
prefix_rewrite: /v1/chat/completions
auto_host_rewrite: true
cluster: bolt_fc_1b
timeout: 120s
- match:
prefix: "/v1/chat/completions"
headers:
@ -180,3 +193,17 @@ static_resources:
address: mistral_7b_instruct
port_value: 10001
hostname: "mistral_7b_instruct"
- name: bolt_fc_1b
connect_timeout: 5s
type: STRICT_DNS
lb_policy: ROUND_ROBIN
load_assignment:
cluster_name: bolt_fc_1b
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: functionresolver
port_value: 80
hostname: "bolt_fc_1b"

5
envoyfilter/src/consts.rs
View file

@ -1,8 +1,9 @@
pub const DEFAULT_EMBEDDING_MODEL: &str = "BAAI/bge-large-en-v1.5";
pub const DEFAULT_COLLECTION_NAME: &str = "prompt_vector_store";
pub const DEFAULT_NER_MODEL: &str = "urchade/gliner_large-v2.1";
pub const DEFAULT_PROMPT_TARGET_THRESHOLD: f64 = 0.6;
pub const DEFAULT_NER_THRESHOLD: f64 = 0.6;
pub const RATELIMIT_SELECTOR_HEADER_KEY: &str = "x-katanemo-ratelimit-selector";
pub const SYSTEM_ROLE: &str = "system";
pub const USER_ROLE: &str = "user";
pub const GPT_35_TURBO: &str = "gpt-3.5-turbo";
pub const BOLT_FC_CLUSTER: &str = "bolt_fc_1b";
pub const BOLT_FC_REQUEST_TIMEOUT_MS: u64 = 120000; // 2 minutes

30
envoyfilter/src/filter_context.rs
View file

@ -2,7 +2,7 @@ use crate::consts::DEFAULT_EMBEDDING_MODEL;
use crate::ratelimit;
use crate::stats::{Counter, Gauge, RecordingMetric};
use crate::stream_context::StreamContext;
use log::{debug, info};
use log::debug;
use md5::Digest;
use open_message_format_embeddings::models::{
CreateEmbeddingRequest, CreateEmbeddingRequestInput, CreateEmbeddingResponse,
@ -72,6 +72,8 @@ impl FilterContext {
(":path", "/embeddings"),
(":authority", "embeddingserver"),
("content-type", "application/json"),
("x-envoy-max-retries", "3"),
("x-envoy-upstream-rq-timeout-ms", "20000"),
],
Some(json_data.as_bytes()),
vec![],
@ -87,6 +89,10 @@ impl FilterContext {
// Need to clone prompt target to leave config string intact.
prompt_target: prompt_target.clone(),
};
debug!(
"dispatched HTTP call to embedding server token_id={}",
token_id
);
if self
.callouts
.insert(token_id, {
@ -112,7 +118,16 @@ impl FilterContext {
if let Some(body) = self.get_http_call_response_body(0, body_size) {
if !body.is_empty() {
let mut embedding_response: CreateEmbeddingResponse =
serde_json::from_slice(&body).unwrap();
match serde_json::from_slice(&body) {
Ok(response) => response,
Err(e) => {
panic!(
"Error deserializing embedding response. body: {:?}: {:?}",
String::from_utf8(body).unwrap(),
e
);
}
};
let mut payload: HashMap<String, String> = HashMap::new();
payload.insert(
@ -168,13 +183,15 @@ impl FilterContext {
.active_http_calls
.record(self.callouts.len().try_into().unwrap());
}
} else {
panic!("No body in response");
}
}
fn create_vector_store_points_handler(&self, body_size: usize) {
if let Some(body) = self.get_http_call_response_body(0, body_size) {
if !body.is_empty() {
info!(
debug!(
"response body: len {:?}",
String::from_utf8(body).unwrap().len()
);
@ -225,7 +242,10 @@ impl Context for FilterContext {
body_size: usize,
_num_trailers: usize,
) {
let callout_data = self.callouts.remove(&token_id).expect("invalid token_id");
let callout_data = self
.callouts
.remove(&token_id)
.expect("invalid token_id: {}");
self.metrics
.active_http_calls
@ -250,7 +270,7 @@ impl Context for FilterContext {
http_status_code.clone_from(v);
}
});
info!("CreateVectorCollection response: {}", http_status_code);
debug!("CreateVectorCollection response: {}", http_status_code);
}
}
}

364
envoyfilter/src/stream_context.rs
View file

@ -1,6 +1,7 @@
use crate::consts::{
DEFAULT_COLLECTION_NAME, DEFAULT_EMBEDDING_MODEL, DEFAULT_NER_MODEL, DEFAULT_NER_THRESHOLD,
DEFAULT_PROMPT_TARGET_THRESHOLD, RATELIMIT_SELECTOR_HEADER_KEY, SYSTEM_ROLE, USER_ROLE,
BOLT_FC_CLUSTER, BOLT_FC_REQUEST_TIMEOUT_MS, DEFAULT_COLLECTION_NAME, DEFAULT_EMBEDDING_MODEL,
DEFAULT_PROMPT_TARGET_THRESHOLD, GPT_35_TURBO, RATELIMIT_SELECTOR_HEADER_KEY, SYSTEM_ROLE,
USER_ROLE,
};
use crate::filter_context::WasmMetrics;
use crate::ratelimit;
@ -16,9 +17,12 @@ use proxy_wasm::traits::*;
use proxy_wasm::types::*;
use public_types::common_types::{
open_ai::{ChatCompletions, Message},
NERRequest, NERResponse, SearchPointsRequest, SearchPointsResponse,
SearchPointsRequest, SearchPointsResponse,
};
use public_types::configuration::{Entity, PromptTarget};
use public_types::common_types::{
BoltFCResponse, BoltFCToolsCall, ToolParameter, ToolParameters, ToolsDefinition,
};
use public_types::configuration::{PromptTarget, PromptType};
use std::collections::HashMap;
use std::num::NonZero;
use std::rc::Rc;
@ -27,8 +31,8 @@ use std::time::Duration;
enum RequestType {
GetEmbedding,
SearchPoints,
Ner,
ContextResolver,
FunctionResolver,
FunctionCallResponse,
}
pub struct CallContext {
@ -153,8 +157,23 @@ impl StreamContext {
}
info!("similarity score: {}", search_results[0].score);
// Check to see who responded to user message. This will help us identify if control should be passed to Bolt FC or not.
// If the last message was from Bolt FC, then Bolt FC is handling the conversation (possibly for parameter collection).
let mut bolt_assistant = false;
let messages = &callout_context.request_body.messages;
if messages.len() >= 2 {
let latest_assistant_message = &messages[messages.len() - 2];
if let Some(model) = latest_assistant_message.model.as_ref() {
if model.starts_with("Bolt") {
info!("Bolt assistant message found");
bolt_assistant = true;
}
}
} else {
info!("no assistant message found, probably first interaction");
}
if search_results[0].score < DEFAULT_PROMPT_TARGET_THRESHOLD {
if search_results[0].score < DEFAULT_PROMPT_TARGET_THRESHOLD && !bolt_assistant {
info!(
"prompt target below threshold: {}",
DEFAULT_PROMPT_TARGET_THRESHOLD
@ -172,177 +191,237 @@ impl StreamContext {
return;
}
};
info!("prompt_target name: {:?}", prompt_target.name);
info!(
"prompt_target name: {:?}, type: {:?}",
prompt_target.name, prompt_target.prompt_type
);
// only extract entity names
let entity_names: Vec<String> = match prompt_target.entities {
// Clone is unavoidable here because we don't want to move the values out of the prompt target struct.
Some(ref entities) => entities.iter().map(|entity| entity.name.clone()).collect(),
None => vec![],
};
match prompt_target.prompt_type {
PromptType::FunctionResolver => {
// only extract entity names
let properties: HashMap<String, ToolParameter> = match prompt_target.parameters {
// Clone is unavoidable here because we don't want to move the values out of the prompt target struct.
Some(ref entities) => {
let mut properties: HashMap<String, ToolParameter> = HashMap::new();
for entity in entities.iter() {
let param = ToolParameter {
parameter_type: entity.parameter_type.clone(),
description: entity.description.clone(),
required: entity.required,
};
properties.insert(entity.name.clone(), param);
}
properties
}
None => HashMap::new(),
};
let tools_parameters = ToolParameters {
parameters_type: "dict".to_string(),
properties,
};
let ner_request = NERRequest {
input: callout_context.user_message.take().unwrap(),
labels: entity_names,
model: DEFAULT_NER_MODEL.to_string(),
};
let tools_defintion: ToolsDefinition = ToolsDefinition {
name: prompt_target.name.clone(),
description: prompt_target.description.clone().unwrap_or("".to_string()),
parameters: tools_parameters,
};
let json_data: String = match serde_json::to_string(&ner_request) {
Ok(json_data) => json_data,
Err(e) => {
warn!("Error serializing ner_request: {:?}", e);
self.resume_http_request();
return;
let chat_completions = ChatCompletions {
model: GPT_35_TURBO.to_string(),
messages: callout_context.request_body.messages.clone(),
tools: Some(vec![tools_defintion]),
};
let msg_body = match serde_json::to_string(&chat_completions) {
Ok(msg_body) => {
debug!("msg_body: {}", msg_body);
msg_body
}
Err(e) => {
warn!("Error serializing request_params: {:?}", e);
self.resume_http_request();
return;
}
};
let token_id = match self.dispatch_http_call(
BOLT_FC_CLUSTER,
vec![
(":method", "POST"),
(":path", "/v1/chat/completions"),
(":authority", BOLT_FC_CLUSTER),
("content-type", "application/json"),
("x-envoy-max-retries", "3"),
(
"x-envoy-upstream-rq-timeout-ms",
BOLT_FC_REQUEST_TIMEOUT_MS.to_string().as_str(),
),
],
Some(msg_body.as_bytes()),
vec![],
Duration::from_secs(5),
) {
Ok(token_id) => token_id,
Err(e) => {
panic!("Error dispatching HTTP call for function-call: {:?}", e);
}
};
debug!(
"dispatched call to function {} token_id={}",
BOLT_FC_CLUSTER, token_id
);
callout_context.request_type = RequestType::FunctionResolver;
callout_context.prompt_target = Some(prompt_target);
if self.callouts.insert(token_id, callout_context).is_some() {
panic!("duplicate token_id")
}
}
};
let token_id = match self.dispatch_http_call(
"nerhost",
vec![
(":method", "POST"),
(":path", "/ner"),
(":authority", "nerhost"),
("content-type", "application/json"),
("x-envoy-max-retries", "3"),
],
Some(json_data.as_bytes()),
vec![],
Duration::from_secs(5),
) {
Ok(token_id) => token_id,
Err(e) => {
panic!("Error dispatching HTTP call for get-embeddings: {:?}", e);
}
};
callout_context.request_type = RequestType::Ner;
callout_context.prompt_target = Some(prompt_target);
if self.callouts.insert(token_id, callout_context).is_some() {
panic!("duplicate token_id")
}
self.metrics.active_http_calls.increment(1);
}
fn ner_handler(&mut self, body: Vec<u8>, mut callout_context: CallContext) {
let ner_response: NERResponse = match serde_json::from_slice(&body) {
Ok(ner_response) => ner_response,
fn function_resolver_handler(&mut self, body: Vec<u8>, mut callout_context: CallContext) {
debug!("response received for function resolver");
let body_str = String::from_utf8(body).unwrap();
debug!("function_resolver response str: {:?}", body_str);
let mut boltfc_response: BoltFCResponse = serde_json::from_str(&body_str).unwrap();
let boltfc_response_str = boltfc_response.message.content.as_ref().unwrap();
let tools_call_response: BoltFCToolsCall = match serde_json::from_str(boltfc_response_str) {
Ok(fc_resp) => fc_resp,
Err(e) => {
warn!("Error deserializing ner_response: {:?}", e);
self.resume_http_request();
return;
}
};
info!("ner_response: {:?}", ner_response);
// This means that Bolt FC did not have enough information to resolve the function call
// Bolt FC probably responded with a message asking for more information.
// Let's send the response back to the user to initalize lightweight dialog for parameter collection
let mut request_params: HashMap<String, String> = HashMap::new();
for entity in ner_response.data.into_iter() {
if entity.score < DEFAULT_NER_THRESHOLD {
warn!(
"score of entity was too low entity name: {}, score: {}",
entity.label, entity.score
// add resolver name to the response so the client can send the response back to the correct resolver
boltfc_response.resolver_name = Some(callout_context.prompt_target.unwrap().name);
info!("some requred parameters are missing, sending response from Bolt FC back to user for parameter collection: {}", e);
let bolt_fc_dialogue_message = serde_json::to_string(&boltfc_response).unwrap();
self.send_http_response(
StatusCode::OK.as_u16().into(),
vec![("Powered-By", "Katanemo")],
Some(bolt_fc_dialogue_message.as_bytes()),
);
continue;
}
request_params.insert(entity.label, entity.text);
}
let prompt_target = callout_context.prompt_target.as_ref().unwrap();
let empty_vec: Vec<Entity> = vec![];
for entity in prompt_target.entities.as_ref().unwrap_or(&empty_vec) {
if entity.required.unwrap_or(false) && !request_params.contains_key(&entity.name) {
warn!(
"required entity missing or score of entity was too low: {}",
entity.name
);
self.resume_http_request();
return;
}
}
let req_param_bytes = match serde_json::to_string(&request_params) {
Ok(req_param_str) => req_param_str.as_bytes().to_owned(),
Err(e) => {
warn!("Error serializing request_params: {:?}", e);
self.resume_http_request();
return;
}
};
let endpoint = callout_context
// verify required parameters are present
callout_context
.prompt_target
.as_ref()
.unwrap()
.endpoint
.parameters
.as_ref()
.unwrap();
.unwrap()
.iter()
.for_each(|param| match param.required {
None => {}
Some(required) => {
if required
&& !tools_call_response.tool_calls[0]
.arguments
.contains_key(&param.name)
{
warn!("boltfc did not extract required parameter: {}", param.name);
return self.send_http_response(
StatusCode::BAD_REQUEST.as_u16().into(),
vec![],
Some("missing required parameter".as_bytes()),
);
}
}
});
let http_path = match &endpoint.path {
Some(path) => path,
None => "/",
};
debug!("tool_call_details: {:?}", tools_call_response);
let tool_name = &tools_call_response.tool_calls[0].name;
let tool_params = &tools_call_response.tool_calls[0].arguments;
debug!("tool_name: {:?}", tool_name);
debug!("tool_params: {:?}", tool_params);
let prompt_target = callout_context.prompt_target.as_ref().unwrap();
debug!("prompt_target: {:?}", prompt_target);
let http_method = match &endpoint.method {
Some(method) => method,
None => http::Method::POST.as_str(),
};
let tool_params_json_str = serde_json::to_string(&tool_params).unwrap();
let endpoint = prompt_target.endpoint.as_ref().unwrap();
let token_id = match self.dispatch_http_call(
&endpoint.cluster,
vec![
(":method", http_method),
(":path", http_path),
(":method", "POST"),
(":path", endpoint.path.as_ref().unwrap_or(&"/".to_string())),
(":authority", endpoint.cluster.as_str()),
("content-type", "application/json"),
("x-envoy-max-retries", "3"),
],
Some(&req_param_bytes),
Some(tool_params_json_str.as_bytes()),
vec![],
Duration::from_secs(5),
) {
Ok(token_id) => token_id,
Err(e) => {
panic!("Error dispatching HTTP call for context_resolver: {:?}", e);
panic!("Error dispatching HTTP call for function_resolver: {:?}", e);
}
};
callout_context.request_type = RequestType::ContextResolver;
callout_context.request_type = RequestType::FunctionCallResponse;
if self.callouts.insert(token_id, callout_context).is_some() {
panic!("duplicate token_id")
}
self.metrics.active_http_calls.increment(1);
}
fn context_resolver_handler(&mut self, body: Vec<u8>, callout_context: CallContext) {
debug!("response received for context_resolver");
let body_string = String::from_utf8(body);
let prompt_target = callout_context.prompt_target.unwrap();
let mut request_body = callout_context.request_body;
match prompt_target.system_prompt {
fn function_call_response_handler(&mut self, body: Vec<u8>, callout_context: CallContext) {
debug!("response received for function call response");
let body_str: String = String::from_utf8(body).unwrap();
debug!("function_call_response response str: {:?}", body_str);
let prompt_target = callout_context.prompt_target.as_ref().unwrap();
let mut messages: Vec<Message> = callout_context.request_body.messages.clone();
// add system prompt
match prompt_target.system_prompt.as_ref() {
None => {}
Some(system_prompt) => {
let system_prompt_message: Message = Message {
let system_prompt_message = Message {
role: SYSTEM_ROLE.to_string(),
content: Some(system_prompt),
content: Some(system_prompt.clone()),
model: None,
};
request_body.messages.push(system_prompt_message);
}
}
match body_string {
Ok(body_string) => {
info!("context_resolver response: {}", body_string);
let context_resolver_response = Message {
role: USER_ROLE.to_string(),
content: Some(body_string),
};
request_body.messages.push(context_resolver_response);
}
Err(e) => {
warn!("Error converting response to string: {:?}", e);
self.resume_http_request();
return;
messages.push(system_prompt_message);
}
}
let json_string = match serde_json::to_string(&request_body) {
// add data from function call response
messages.push({
Message {
role: USER_ROLE.to_string(),
content: Some(body_str),
model: None,
}
});
// add original user prompt
messages.push({
Message {
role: USER_ROLE.to_string(),
content: Some(callout_context.user_message.unwrap()),
model: None,
}
});
let request_message: ChatCompletions = ChatCompletions {
model: GPT_35_TURBO.to_string(),
messages,
tools: None,
};
let json_string = match serde_json::to_string(&request_message) {
Ok(json_string) => json_string,
Err(e) => {
warn!("Error serializing request_body: {:?}", e);
@ -350,6 +429,12 @@ impl StreamContext {
return;
}
};
debug!(
"function_calling sending request to openai: msg {}",
json_string
);
let request_body = callout_context.request_body;
// Tokenize and Ratelimit.
if let Some(selector) = self.ratelimit_selector.take() {
@ -405,8 +490,7 @@ impl HttpContext for StreamContext {
// Deserialize body into spec.
// Currently OpenAI API.
let mut deserialized_body: ChatCompletions = match self.get_http_request_body(0, body_size)
{
let deserialized_body: ChatCompletions = match self.get_http_request_body(0, body_size) {
Some(body_bytes) => match serde_json::from_slice(&body_bytes) {
Ok(deserialized) => deserialized,
Err(msg) => {
@ -434,12 +518,12 @@ impl HttpContext for StreamContext {
let user_message = match deserialized_body
.messages
.pop()
.and_then(|last_message| last_message.content)
.last()
.and_then(|last_message| last_message.content.clone())
{
Some(content) => content,
None => {
info!("No messages in the request body");
warn!("No messages in the request body");
return Action::Continue;
}
};
@ -468,6 +552,7 @@ impl HttpContext for StreamContext {
(":authority", "embeddingserver"),
("content-type", "application/json"),
("x-envoy-max-retries", "3"),
("x-envoy-upstream-rq-timeout-ms", "60000"),
],
Some(json_data.as_bytes()),
vec![],
@ -481,6 +566,11 @@ impl HttpContext for StreamContext {
);
}
};
debug!(
"dispatched HTTP call to embedding server token_id={}",
token_id
);
let call_context = CallContext {
request_type: RequestType::GetEmbedding,
user_message: Some(user_message),
@ -530,8 +620,10 @@ impl Context for StreamContext {
match callout_context.request_type {
RequestType::GetEmbedding => self.embeddings_handler(body, callout_context),
RequestType::SearchPoints => self.search_points_handler(body, callout_context),
RequestType::Ner => self.ner_handler(body, callout_context),
RequestType::ContextResolver => self.context_resolver_handler(body, callout_context),
RequestType::FunctionResolver => self.function_resolver_handler(body, callout_context),
RequestType::FunctionCallResponse => {
self.function_call_response_handler(body, callout_context)
}
}
}
}

151
envoyfilter/tests/integration.rs
View file

@ -8,9 +8,14 @@ use proxy_wasm_test_framework::tester::{self, Tester};
use proxy_wasm_test_framework::types::{
Action, BufferType, LogLevel, MapType, MetricType, ReturnType,
};
use public_types::configuration::{self, Endpoint, PromptTarget};
use public_types::{
common_types::{self, NERResponse, SearchPointResult, SearchPointsResponse},
common_types::{
open_ai::Message, BoltFCResponse, BoltFCToolsCall, IntOrString, ToolCallDetail,
},
configuration::{self, Endpoint, PromptTarget},
};
use public_types::{
common_types::{SearchPointResult, SearchPointsResponse},
configuration::Configuration,
};
use serial_test::serial;
@ -87,6 +92,7 @@ fn normal_flow(module: &mut Tester, filter_context: i32, http_context: i32) {
.expect_http_call(Some("embeddingserver"), None, None, None, None)
.returning(Some(1))
.expect_metric_increment("active_http_calls", 1)
.expect_log(Some(LogLevel::Debug), None)
.execute_and_expect(ReturnType::Action(Action::Pause))
.unwrap();
@ -120,12 +126,14 @@ fn normal_flow(module: &mut Tester, filter_context: i32, http_context: i32) {
let prompt_target = PromptTarget {
name: String::from("test-prompt-target"),
prompt_type: String::from("test-prompt-type"),
description: None,
prompt_type: configuration::PromptType::FunctionResolver,
few_shot_examples: vec![],
entities: Some(vec![configuration::Entity {
parameters: Some(vec![configuration::Parameter {
name: String::from("test-entity"),
parameter_type: Some(String::from("string")),
description: String::from("test-description"),
required: Some(true),
description: None,
}]),
endpoint: Some(Endpoint {
cluster: String::from("test-endpoint-cluster"),
@ -159,33 +167,13 @@ fn normal_flow(module: &mut Tester, filter_context: i32, http_context: i32) {
.returning(Some(&search_points_response_buffer))
.expect_log(Some(LogLevel::Info), None)
.expect_log(Some(LogLevel::Info), None)
.expect_http_call(Some("nerhost"), None, None, None, None)
.expect_log(Some(LogLevel::Debug), None)
.expect_http_call(Some("bolt_fc_1b"), None, None, None, None)
.returning(Some(3))
.expect_log(Some(LogLevel::Debug), None)
.expect_metric_increment("active_http_calls", 1)
.execute_and_expect(ReturnType::None)
.unwrap();
let ner_reponse = NERResponse {
model: String::from("test-model"),
data: vec![common_types::Entity {
score: 0.7,
text: String::from("test-text"),
label: String::from("test-entity"),
}],
};
let ner_response_buffer = serde_json::to_string(&ner_reponse).unwrap();
let upstream_name = prompt_target.endpoint.unwrap().cluster.leak();
module
.call_proxy_on_http_call_response(http_context, 3, 0, ner_response_buffer.len() as i32, 0)
.expect_metric_increment("active_http_calls", -1)
.expect_get_buffer_bytes(Some(BufferType::HttpCallResponseBody))
.returning(Some(&ner_response_buffer))
.expect_log(Some(LogLevel::Info), None)
.expect_http_call(Some(upstream_name), None, None, None, None)
.returning(Some(4))
.expect_metric_increment("active_http_calls", 1)
.execute_and_expect(ReturnType::None)
.unwrap()
}
fn default_config() -> Configuration {
@ -209,7 +197,7 @@ system_prompt: |
- Use miles per hour for wind speed
prompt_targets:
- type: context_resolver
- type: function_resolver
name: weather_forecast
few_shot_examples:
- what is the weather in New York?
@ -221,7 +209,7 @@ prompt_targets:
required: true
description: "The location for which the weather is requested"
- type: context_resolver
- type: function_resolver
name: weather_forecast_2
few_shot_examples:
- what is the weather in New York?
@ -327,6 +315,7 @@ fn successful_request_to_open_ai_chat_completions() {
.returning(Some(chat_completions_request_body))
// TODO: assert that the model field was added.
.expect_set_buffer_bytes(Some(BufferType::HttpRequestBody), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_metric_increment("active_http_calls", 1)
.execute_and_expect(ReturnType::Action(Action::Pause))
.unwrap();
@ -460,14 +449,57 @@ fn request_ratelimited() {
normal_flow(&mut module, filter_context, http_context);
let test_body = "test body";
let tool_call_detail = vec![ToolCallDetail {
name: String::from("test-tool"),
arguments: HashMap::from([(
String::from("test-entity"),
IntOrString::Text(String::from("test-value")),
)]),
}];
let boltfc_tools_call = BoltFCToolsCall {
tool_calls: tool_call_detail,
};
let bolt_fc_resp = BoltFCResponse {
model: String::from("test"),
message: Message {
role: String::from("system"),
content: Some(serde_json::to_string(&boltfc_tools_call).unwrap()),
model: None,
},
done_reason: String::from("test"),
done: true,
resolver_name: None,
};
let bolt_fc_resp_str = serde_json::to_string(&bolt_fc_resp).unwrap();
module
.call_proxy_on_http_call_response(http_context, 4, 0, test_body.len() as i32, 0)
.call_proxy_on_http_call_response(http_context, 3, 0, bolt_fc_resp_str.len() as i32, 0)
.expect_metric_increment("active_http_calls", -1)
.expect_get_buffer_bytes(Some(BufferType::HttpCallResponseBody))
.returning(Some(test_body))
.returning(Some(&bolt_fc_resp_str))
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_http_call(Some("test-endpoint-cluster"), None, None, None, None)
.returning(Some(4))
.expect_metric_increment("active_http_calls", 1)
.execute_and_expect(ReturnType::None)
.unwrap();
let body_text = String::from("test body");
module
.call_proxy_on_http_call_response(http_context, 4, 0, body_text.len() as i32, 0)
.expect_metric_increment("active_http_calls", -1)
.expect_get_buffer_bytes(Some(BufferType::HttpCallResponseBody))
.returning(Some(&body_text))
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Info), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_send_local_response(
@ -522,18 +554,61 @@ fn request_not_ratelimited() {
normal_flow(&mut module, filter_context, http_context);
let test_body = "test body";
let tool_call_detail = vec![ToolCallDetail {
name: String::from("test-tool"),
arguments: HashMap::from([(
String::from("test-entity"),
IntOrString::Text(String::from("test-value")),
)]),
}];
let boltfc_tools_call = BoltFCToolsCall {
tool_calls: tool_call_detail,
};
let bolt_fc_resp = BoltFCResponse {
model: String::from("test"),
message: Message {
role: String::from("system"),
content: Some(serde_json::to_string(&boltfc_tools_call).unwrap()),
model: None,
},
done_reason: String::from("test"),
done: true,
resolver_name: None,
};
let bolt_fc_resp_str = serde_json::to_string(&bolt_fc_resp).unwrap();
module
.call_proxy_on_http_call_response(http_context, 4, 0, test_body.len() as i32, 0)
.call_proxy_on_http_call_response(http_context, 3, 0, bolt_fc_resp_str.len() as i32, 0)
.expect_metric_increment("active_http_calls", -1)
.expect_get_buffer_bytes(Some(BufferType::HttpCallResponseBody))
.returning(Some(test_body))
.returning(Some(&bolt_fc_resp_str))
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_http_call(Some("test-endpoint-cluster"), None, None, None, None)
.returning(Some(4))
.expect_metric_increment("active_http_calls", 1)
.execute_and_expect(ReturnType::None)
.unwrap();
let body_text = String::from("test body");
module
.call_proxy_on_http_call_response(http_context, 4, 0, body_text.len() as i32, 0)
.expect_metric_increment("active_http_calls", -1)
.expect_get_buffer_bytes(Some(BufferType::HttpCallResponseBody))
.returning(Some(&body_text))
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Info), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_log(Some(LogLevel::Debug), None)
.expect_set_buffer_bytes(Some(BufferType::HttpRequestBody), None)
.expect_log(Some(LogLevel::Debug), None)
.execute_and_expect(ReturnType::None)
.unwrap();
}

16
function_resolver/.vscode/launch.json vendored Normal file
View file

@ -0,0 +1,16 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "function resolver server",
"cwd": "${workspaceFolder}/app",
"type": "debugpy",
"request": "launch",
"module": "uvicorn",
"args": ["main:app","--reload", "--host", "0.0.0.0", "--port", "8001", "--log-config", "logger.yaml"],
}
]
}

30
function_resolver/Dockerfile Normal file
View file

@ -0,0 +1,30 @@
FROM python:3 AS base
#
# builder
#
FROM base AS builder
WORKDIR /src
COPY requirements.txt /src/
RUN pip install --prefix=/runtime --force-reinstall -r requirements.txt
COPY . /src
#
# output
#
FROM python:3-slim AS output
COPY --from=builder /runtime /usr/local
COPY /app /app
WORKDIR /app
RUN apt-get update && apt-get install -y \
curl \
&& rm -rf /var/lib/apt/lists/*
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "80", "--log-config", "logger.yaml"]

225
function_resolver/app/bolt_handler.py Normal file
View file

@ -0,0 +1,225 @@
import json
from typing import Any, Dict, List
SYSTEM_PROMPT = """
[BEGIN OF TASK INSTRUCTION]
You are a function calling assistant with access to the following tools. You task is to assist users as best as you can.
For each user query, you may need to call one or more functions to to better generate responses.
If none of the functions are relevant, you should point it out.
If the given query lacks the parameters required by the function, you should ask users for clarification.
The users may execute functions and return results as `Observation` to you. In the case, you MUST generate responses by summarizing it.
[END OF TASK INSTRUCTION]
""".strip()
TOOL_PROMPT = """
[BEGIN OF AVAILABLE TOOLS]
{tool_text}
[END OF AVAILABLE TOOLS]
""".strip()
FORMAT_PROMPT = """
[BEGIN OF FORMAT INSTRUCTION]
You MUST use the following JSON format if using tools.
The example format is as follows. DO NOT use this format if no function call is needed.
```
{
"tool_calls": [
{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},
... (more tool calls as required)
]
}
```
[END OF FORMAT INSTRUCTION]
""".strip()
class BoltHandler:
def _format_system(self, tools: List[Dict[str, Any]]):
tool_text = self._format_tools(tools=tools)
return (
SYSTEM_PROMPT
+ "\n\n"
+ TOOL_PROMPT.format(tool_text=tool_text)
+ "\n\n"
+ FORMAT_PROMPT
+ "\n"
)
def _format_tools(self, tools: List[Dict[str, Any]]):
TOOL_DESC = "> Tool Name: {name}\nTool Description: {desc}\nTool Args:\n{args}"
tool_text = []
for tool in tools:
param_text = self.get_param_text(tool.parameters)
tool_text.append(
TOOL_DESC.format(
name=tool.name, desc=tool.description, args=param_text
)
)
return "\n".join(tool_text)
def extract_tools(self, content, executable=False):
# retrieve `tool_calls` from model responses
try:
content_json = json.loads(content)
except Exception:
fixed_content = self.fix_json_string(content)
try:
content_json = json.loads(fixed_content)
except json.JSONDecodeError:
return content
if isinstance(content_json, list):
tool_calls = content_json
elif isinstance(content_json, dict):
tool_calls = content_json.get("tool_calls", [])
else:
tool_calls = []
if not isinstance(tool_calls, list):
return content
# process and extract tools from `tool_calls`
extracted = []
for tool_call in tool_calls:
if isinstance(tool_call, dict):
try:
if not executable:
extracted.append({tool_call["name"]: tool_call["arguments"]})
else:
name, arguments = (
tool_call.get("name", ""),
tool_call.get("arguments", {}),
)
for key, value in arguments.items():
if value == "False" or value == "false":
arguments[key] = False
elif value == "True" or value == "true":
arguments[key] = True
args_str = ", ".join(
[f"{key}={repr(value)}" for key, value in arguments.items()]
)
extracted.append(f"{name}({args_str})")
except Exception:
continue
return extracted
def get_param_text(self, parameter_dict, prefix=""):
param_text = ""
for name, param in parameter_dict["properties"].items():
param_type = param.get("type", "")
required, default, param_format, properties, enum, items = (
"",
"",
"",
"",
"",
"",
)
if name in parameter_dict.get("required", []):
required = ", required"
required_param = parameter_dict.get("required", [])
if isinstance(required_param, bool):
required = ", required" if required_param else ""
elif isinstance(required_param, list) and name in required_param:
required = ", required"
else:
required = ", optional"
default_param = param.get("default", None)
if default_param:
default = f", default: {default_param}"
format_in = param.get("format", None)
if format_in:
param_format = f", format: {format_in}"
desc = param.get("description", "")
if "properties" in param:
arg_properties = self.get_param_text(param, prefix + " ")
properties += "with the properties:\n{}".format(arg_properties)
enum_param = param.get("enum", None)
if enum_param:
enum = "should be one of [{}]".format(", ".join(enum_param))
item_param = param.get("items", None)
if item_param:
item_type = item_param.get("type", None)
if item_type:
items += "each item should be the {} type ".format(item_type)
item_properties = item_param.get("properties", None)
if item_properties:
item_properties = self.get_param_text(item_param, prefix + " ")
items += "with the properties:\n{}".format(item_properties)
illustration = ", ".join(
[x for x in [desc, properties, enum, items] if len(x)]
)
param_text += (
prefix
+ "- {name} ({param_type}{required}{param_format}{default}): {illustration}\n".format(
name=name,
param_type=param_type,
required=required,
param_format=param_format,
default=default,
illustration=illustration,
)
)
return param_text
def fix_json_string(self, json_str):
# Remove any leading or trailing whitespace or newline characters
json_str = json_str.strip()
# Stack to keep track of brackets
stack = []
# Clean string to collect valid characters
fixed_str = ""
# Dictionary for matching brackets
matching_bracket = {")": "(", "}": "{", "]": "["}
# Dictionary for the opposite of matching_bracket
opening_bracket = {v: k for k, v in matching_bracket.items()}
for char in json_str:
if char in "{[(":
stack.append(char)
fixed_str += char
elif char in "}])":
if stack and stack[-1] == matching_bracket[char]:
stack.pop()
fixed_str += char
else:
# Ignore the unmatched closing brackets
continue
else:
fixed_str += char
# If there are unmatched opening brackets left in the stack, add corresponding closing brackets
while stack:
unmatched_opening = stack.pop()
fixed_str += opening_bracket[unmatched_opening]
# Attempt to parse the corrected string to ensure it’s valid JSON
return fixed_str

14
function_resolver/app/common.py Normal file
View file

@ -0,0 +1,14 @@
from pydantic import BaseModel
class Tool(BaseModel):
name: str
description: str
parameters: dict
class Message(BaseModel):
role: str
content: str
class ChatMessage(BaseModel):
messages: list[Message]
tools: list[Tool]

14
function_resolver/app/logger.yaml Normal file
View file

@ -0,0 +1,14 @@
version: 1
disable_existing_loggers: False
formatters:
timestamped:
format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
handlers:
console:
class: logging.StreamHandler
level: INFO
formatter: timestamped
stream: ext://sys.stdout
root:
level: INFO
handlers: [console]

40
function_resolver/app/main.py Normal file
View file

@ -0,0 +1,40 @@
from fastapi import FastAPI, Response
from bolt_handler import BoltHandler
from common import ChatMessage
import logging
from ollama import Client
import os
ollama_endpoint = os.getenv("OLLAMA_ENDPOINT", "localhost")
ollama_model = os.getenv("OLLAMA_MODEL", "Bolt-Function-Calling-1B:Q4_K_M")
logger = logging.getLogger('uvicorn.error')
logger.info(f"using model: {ollama_model}")
logger.info(f"using ollama endpoint: {ollama_endpoint}")
app = FastAPI()
handler = BoltHandler()
ollama_client = Client(host=ollama_endpoint)
@app.get("/healthz")
async def healthz():
return {
"status": "ok"
}
@app.post("/v1/chat/completions")
async def chat_completion(req: ChatMessage, res: Response):
logger.info("starting request")
tools_encoded = handler._format_system(req.tools)
messages = []
messages.append(
{"role": "system", "content": tools_encoded}
)
messages.append({"role": "user", "content": req.messages[-1].content})
resp = ollama_client.chat(messages=messages, model=ollama_model, stream=False)
logger.info(f"response: {resp}")
return resp

4
function_resolver/requirements.txt Normal file
View file

@ -0,0 +1,4 @@
fastapi
uvicorn
ollama
PyYAML

5
function_resolver/test/test.sh Normal file
View file

@ -0,0 +1,5 @@
PORT="${PORT:-8001}"
echo localhost:$PORT/v1/chat/completions
curl -H "content-type: application/json" -XPOST localhost:$PORT/v1/chat/completions -d @test_payload.json

5
function_resolver/test/test_envoy_cluster.sh Normal file
View file

@ -0,0 +1,5 @@
PORT="${PORT:-8001}"
echo localhost:$PORT/bolt_fc_1b/v1/chat/completions
curl -v -H "content-type: application/json" -XPOST localhost:$PORT/bolt_fc_1b/v1/chat/completions -d @test_payload.json

1
function_resolver/test/test_missing_param.sh Normal file
View file

@ -0,0 +1 @@
curl -H "content-type: application/json" -XPOST localhost:8001/v1/chat/completions -d @test_payload_missing_param.json

33
function_resolver/test/test_payload.json Normal file
View file

@ -0,0 +1,33 @@
{
"messages": [
{
"role": "user",
"content": "Find the area of a triangle with a base of 10 units and height of 5 units."
}
],
"tools": [
{
"name": "calculate_triangle_area",
"description": "Calculate the area of a triangle given its base and height.",
"parameters": {
"type": "dict",
"properties": {
"base": {
"type": "integer",
"description": "The base of the triangle.",
"required": true
},
"height": {
"type": "integer",
"description": "The height of the triangle.",
"required": true
},
"unit": {
"type": "string",
"description": "The unit of measure (defaults to 'units' if not specified)"
}
}
}
}
]
}

32
function_resolver/test/test_payload_missing_param.json Normal file
View file

@ -0,0 +1,32 @@
{
"messages": [
{
"role": "user",
"content": "Find the area of a triangle"
}
],
"tools": [
{
"name": "calculate_triangle_area",
"description": "Calculate the area of a triangle given its base and height.",
"parameters": {
"type": "dict",
"properties": {
"base": {
"type": "integer",
"description": "The base of the triangle."
},
"height": {
"type": "integer",
"description": "The height of the triangle."
},
"unit": {
"type": "string",
"description": "The unit of measure (defaults to 'units' if not specified)"
}
},
"required": ["base", "height"]
}
}
]
}

29
function_resolver/test/test_weather.json Normal file
View file

@ -0,0 +1,29 @@
{
"messages": [
{
"role": "user",
"content": "how is the weather in San Francisco for next 5 days?"
}
],
"tools": [
{
"name": "weather_forecast",
"description": "This function resolver provides weather forecast information for a given city.",
"parameters": {
"type": "dict",
"properties": {
"days": {
"description": "The number of days for which the weather forecast is requested."
},
"units": {
"description": "The units in which the weather forecast is requested."
},
"city": {
"description": "The city for which the weather forecast is requested.",
"required": true
}
}
}
}
]
}

12
gateway.code-workspace
View file

@ -12,6 +12,10 @@
"name": "embedding-server",
"path": "embedding-server"
},
{
"name": "function_resolver",
"path": "function_resolver"
},
{
"name": "chatbot-ui",
"path": "chatbot-ui"
@ -21,13 +25,9 @@
"path": "open-message-format"
},
{
"name": "demos/weather-forecast",
"path": "./demos/weather-forecast",
"name": "demos/function-calling",
"path": "./demos/function-calling",
},
{
"name": "demos/weather-forecast-local-llm",
"path": "./demos/weather-forecast-local-llm",
}
],
"settings": {}
}

58
public-types/src/common_types.rs
View file

@ -53,37 +53,75 @@ pub struct SearchPointsResponse {
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NERRequest {
pub input: String,
pub labels: Vec<String>,
pub model: String,
pub struct ToolParameter {
#[serde(rename = "type")]
#[serde(skip_serializing_if = "Option::is_none")]
pub parameter_type: Option<String>,
pub description: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub required: Option<bool>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Entity {
pub text: String,
pub label: String,
pub score: f64,
pub struct ToolParameters {
#[serde(rename = "type")]
pub parameters_type: String,
pub properties: HashMap<String, ToolParameter>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NERResponse {
pub data: Vec<Entity>,
pub struct ToolsDefinition {
pub name: String,
pub description: String,
pub parameters: ToolParameters,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BoltFCResponse {
pub model: String,
pub message: open_ai::Message,
pub done_reason: String,
pub done: bool,
pub resolver_name: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum IntOrString {
Integer(i32),
Text(String),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolCallDetail {
pub name: String,
pub arguments: HashMap<String, IntOrString>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BoltFCToolsCall {
pub tool_calls: Vec<ToolCallDetail>,
}
pub mod open_ai {
use serde::{Deserialize, Serialize};
use super::ToolsDefinition;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ChatCompletions {
#[serde(default)]
pub model: String,
pub messages: Vec<Message>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tools: Option<Vec<ToolsDefinition>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Message {
pub role: String,
pub content: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
}
}

27
public-types/src/configuration.rs
View file

@ -80,19 +80,28 @@ pub struct Endpoint {
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Entity {
pub struct Parameter {
pub name: String,
#[serde(rename = "type")]
pub parameter_type: Option<String>,
pub description: String,
pub required: Option<bool>,
pub description: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum PromptType {
#[serde(rename = "function_resolver")]
FunctionResolver,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PromptTarget {
#[serde(rename = "type")]
pub prompt_type: String,
pub prompt_type: PromptType,
pub name: String,
pub description: Option<String>,
pub few_shot_examples: Vec<String>,
pub entities: Option<Vec<Entity>>,
pub parameters: Option<Vec<Parameter>>,
pub endpoint: Option<Endpoint>,
pub system_prompt: Option<String>,
}
@ -119,27 +128,29 @@ system_prompt: |
- Use miles per hour for wind speed
prompt_targets:
- type: context_resolver
- type: function_resolver
name: weather_forecast
few_shot_examples:
- what is the weather in New York?
endpoint:
cluster: weatherhost
path: /weather
entities:
parameters:
- name: location
required: true
description: "The location for which the weather is requested"
- type: context_resolver
- type: function_resolver
name: weather_forecast_2
few_shot_examples:
- what is the weather in New York?
endpoint:
cluster: weatherhost
path: /weather
entities:
parameters:
- name: city
description: "The location for which the weather is requested"
ratelimits:
- provider: open-ai-gpt-4