diff --git a/arch/arch_config_schema.yaml b/arch/arch_config_schema.yaml index 96e14856..f2b2c8a5 100644 --- a/arch/arch_config_schema.yaml +++ b/arch/arch_config_schema.yaml @@ -135,6 +135,10 @@ properties: enum: - GET - POST + http_headers: + type: object + additionalProperties: + type: string additionalProperties: false required: - name diff --git a/arch/envoy.template.yaml b/arch/envoy.template.yaml index 5dfa60b3..0040b57b 100644 --- a/arch/envoy.template.yaml +++ b/arch/envoy.template.yaml @@ -486,14 +486,14 @@ static_resources: typed_config: "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext sni: api.mistral.ai - {% for internal_clustrer in ["arch_fc", "model_server"] %} - - name: {{ internal_clustrer }} + {% for internal_cluster in ["arch_fc", "model_server"] %} + - name: {{ internal_cluster }} connect_timeout: 5s type: STRICT_DNS dns_lookup_family: V4_ONLY lb_policy: ROUND_ROBIN load_assignment: - cluster_name: {{ internal_clustrer }} + cluster_name: {{ internal_cluster }} endpoints: - lb_endpoints: - endpoint: @@ -501,7 +501,7 @@ static_resources: socket_address: address: host.docker.internal port_value: $MODEL_SERVER_PORT - hostname: {{ internal_clustrer }} + hostname: {{ internal_cluster }} {% endfor %} - name: mistral_7b_instruct connect_timeout: 5s diff --git a/arch/tools/cli/utils.py b/arch/tools/cli/utils.py index 8e8b234d..cc6ac20b 100644 --- a/arch/tools/cli/utils.py +++ b/arch/tools/cli/utils.py @@ -89,6 +89,18 @@ def get_llm_provider_access_keys(arch_config_file): if acess_key is not None: access_key_list.append(acess_key) + for prompt_target in arch_config_yaml.get("prompt_targets", []): + for k, v in prompt_target.get("endpoint", {}).get("http_headers", {}).items(): + if k.lower() == "authorization": + print( + f"found auth header: {k} for prompt_target: {prompt_target.get('name')}/{prompt_target.get('endpoint').get('name')}" + ) + auth_tokens = v.split(" ") + if len(auth_tokens) > 1: + access_key_list.append(auth_tokens[1]) + else: + access_key_list.append(v) + return access_key_list diff --git a/crates/common/src/configuration.rs b/crates/common/src/configuration.rs index fbafe7b9..f1250499 100644 --- a/crates/common/src/configuration.rs +++ b/crates/common/src/configuration.rs @@ -242,6 +242,7 @@ pub struct EndpointDetails { pub path: Option, #[serde(rename = "http_method")] pub method: Option, + pub http_headers: Option>, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/prompt_gateway/src/stream_context.rs b/crates/prompt_gateway/src/stream_context.rs index 24e37186..2704e8d8 100644 --- a/crates/prompt_gateway/src/stream_context.rs +++ b/crates/prompt_gateway/src/stream_context.rs @@ -322,6 +322,12 @@ impl StreamContext { headers.insert(TRACE_PARENT_HEADER, self.traceparent.as_ref().unwrap()); } + // override http headers that are set in the prompt target + let http_headers = endpoint_details.http_headers.clone().unwrap_or_default(); + for (key, value) in http_headers.iter() { + headers.insert(key.as_str(), value.as_str()); + } + let call_args = CallArgs::new( ARCH_INTERNAL_CLUSTER_NAME, &path, diff --git a/demos/currency_exchange/arch_config.yaml b/demos/currency_exchange/arch_config.yaml index f51b5904..89a9e65b 100644 --- a/demos/currency_exchange/arch_config.yaml +++ b/demos/currency_exchange/arch_config.yaml @@ -12,6 +12,11 @@ llm_providers: provider_interface: openai model: gpt-4o +endpoints: + frankfurther_api: + endpoint: api.frankfurter.dev + protocol: https + system_prompt: | You are a helpful assistant. @@ -26,7 +31,7 @@ prompt_targets: description: Get currency exchange rate from USD to other currencies parameters: - name: currency_symbol - description: the currency that needs conversion + description: currency symbol to convert from USD required: true type: str in_path: true @@ -42,11 +47,6 @@ prompt_targets: name: frankfurther_api path: /v1/currencies -endpoints: - frankfurther_api: - endpoint: api.frankfurter.dev - protocol: https - tracing: random_sampling: 100 trace_arch_internal: true diff --git a/demos/stock_quote/README.md b/demos/stock_quote/README.md new file mode 100644 index 00000000..885bdd5d --- /dev/null +++ b/demos/stock_quote/README.md @@ -0,0 +1,9 @@ +This demo shows how you can use a publicly hosted rest api that is protected by an access key. + +Before you start the demo make sure you set `OPENAI_API_KEY` and `TWELVEDATA_API_KEY`. + +To get `TWELVEDATA_API_KEY` please head over to https://twelvedata.com/. + +Following screenshot shows interaction with stock quote demo, + +![alt text](stock_quote_demo.png) diff --git a/demos/stock_quote/arch_config.yaml b/demos/stock_quote/arch_config.yaml new file mode 100644 index 00000000..c763d4ca --- /dev/null +++ b/demos/stock_quote/arch_config.yaml @@ -0,0 +1,69 @@ +version: v0.1 + +listener: + address: 0.0.0.0 + port: 10000 + message_format: huggingface + connect_timeout: 0.005s + +llm_providers: + - name: gpt-4o + access_key: $OPENAI_API_KEY + provider_interface: openai + model: gpt-4o + +endpoints: + twelvedata_api: + endpoint: api.twelvedata.com + protocol: https + +system_prompt: | + You are a helpful assistant. + +prompt_guards: + input_guards: + jailbreak: + on_exception: + message: Looks like you're curious about my abilities, but I can only provide assistance for currency exchange. + +prompt_targets: + - name: stock_quote + description: get current stock exchange rate for a given symbol + parameters: + - name: symbol + description: Stock symbol + required: true + type: str + endpoint: + name: twelvedata_api + path: /quote + http_headers: + Authorization: "apikey $TWELVEDATA_API_KEY" + system_prompt: | + You are a helpful stock exchange assistant. You are given stock symbol along with its exchange rate in json format. Your task is to parse the data and present it in a human-readable format. Keep the details to highlevel and be concise. + + - name: stock_quote_time_series + description: get historical stock exchange rate for a given symbol + parameters: + - name: symbol + description: Stock symbol + required: true + type: str + - name: interval + description: Time interval + default: 1day + enum: + - 1h + - 1day + type: str + endpoint: + name: twelvedata_api + path: /time_series + http_headers: + Authorization: "apikey $TWELVEDATA_API_KEY" + system_prompt: | + You are a helpful stock exchange assistant. You are given stock symbol along with its historical data in json format. Your task is to parse the data and present it in a human-readable format. Keep the details to highlevel only and be concise. + +tracing: + random_sampling: 100 + trace_arch_internal: true diff --git a/demos/stock_quote/docker-compose.yaml b/demos/stock_quote/docker-compose.yaml new file mode 100644 index 00000000..32cfbd20 --- /dev/null +++ b/demos/stock_quote/docker-compose.yaml @@ -0,0 +1,21 @@ +services: + chatbot_ui: + build: + context: ../shared/chatbot_ui + ports: + - "18080:8080" + environment: + # this is only because we are running the sample app in the same docker container environment as archgw + - CHAT_COMPLETION_ENDPOINT=http://host.docker.internal:10000/v1 + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ./arch_config.yaml:/app/arch_config.yaml + + jaeger: + build: + context: ../shared/jaeger + ports: + - "16686:16686" + - "4317:4317" + - "4318:4318" diff --git a/demos/stock_quote/run_demo.sh b/demos/stock_quote/run_demo.sh new file mode 100644 index 00000000..eb47dce6 --- /dev/null +++ b/demos/stock_quote/run_demo.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + +# Function to start the demo +start_demo() { + # Step 1: Check if .env file exists + if [ -f ".env" ]; then + echo ".env file already exists. Skipping creation." + else + # Step 2: Create `.env` file and set OpenAI key + if [ -z "$OPENAI_API_KEY" ]; then + echo "Error: OPENAI_API_KEY environment variable is not set for the demo." + exit 1 + fi + + echo "Creating .env file..." + echo "OPENAI_API_KEY=$OPENAI_API_KEY" > .env + echo ".env file created with OPENAI_API_KEY." + fi + + # Step 3: Start Arch + echo "Starting Arch with arch_config.yaml..." + archgw up arch_config.yaml + + # Step 4: Start developer services + echo "Starting Network Agent using Docker Compose..." + docker compose up -d # Run in detached mode +} + +# Function to stop the demo +stop_demo() { + # Step 1: Stop Docker Compose services + echo "Stopping Network Agent using Docker Compose..." + docker compose down + + # Step 2: Stop Arch + echo "Stopping Arch..." + archgw down +} + +# Main script logic +if [ "$1" == "down" ]; then + stop_demo +else + # Default action is to bring the demo up + start_demo +fi diff --git a/demos/stock_quote/stock_quote_demo.png b/demos/stock_quote/stock_quote_demo.png new file mode 100644 index 00000000..abb70aa2 Binary files /dev/null and b/demos/stock_quote/stock_quote_demo.png differ