add ability to specify custom http headers in api endpoint (#386)

arch/arch_config_schema.yaml

@@ -135,6 +135,10 @@ properties:
               enum:
                 - GET
                 - POST
+            http_headers:
+              type: object
+              additionalProperties:
+                type: string
           additionalProperties: false
           required:
             - name

arch/envoy.template.yaml

@@ -486,14 +486,14 @@ static_resources:
         typed_config:
           "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
           sni: api.mistral.ai
-    {% for internal_clustrer in ["arch_fc", "model_server"] %}
-    - name: {{ internal_clustrer }}
+    {% for internal_cluster in ["arch_fc", "model_server"] %}
+    - name: {{ internal_cluster }}
       connect_timeout: 5s
       type: STRICT_DNS
       dns_lookup_family: V4_ONLY
       lb_policy: ROUND_ROBIN
       load_assignment:
-        cluster_name: {{ internal_clustrer }}
+        cluster_name: {{ internal_cluster }}
         endpoints:
           - lb_endpoints:
               - endpoint:
@@ -501,7 +501,7 @@ static_resources:
                     socket_address:
                       address: host.docker.internal
                       port_value: $MODEL_SERVER_PORT
-                  hostname: {{ internal_clustrer }}
+                  hostname: {{ internal_cluster }}
     {% endfor %}
     - name: mistral_7b_instruct
       connect_timeout: 5s

arch/tools/cli/utils.py

@@ -89,6 +89,18 @@ def get_llm_provider_access_keys(arch_config_file):
         if acess_key is not None:
             access_key_list.append(acess_key)
 
+    for prompt_target in arch_config_yaml.get("prompt_targets", []):
+        for k, v in prompt_target.get("endpoint", {}).get("http_headers", {}).items():
+            if k.lower() == "authorization":
+                print(
+                    f"found auth header: {k} for prompt_target: {prompt_target.get('name')}/{prompt_target.get('endpoint').get('name')}"
+                )
+                auth_tokens = v.split(" ")
+                if len(auth_tokens) > 1:
+                    access_key_list.append(auth_tokens[1])
+                else:
+                    access_key_list.append(v)
+
     return access_key_list
 
 

crates/common/src/configuration.rs

@@ -242,6 +242,7 @@ pub struct EndpointDetails {
     pub path: Option<String>,
     #[serde(rename = "http_method")]
     pub method: Option<HttpMethod>,
+    pub http_headers: Option<HashMap<String, String>>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

crates/prompt_gateway/src/stream_context.rs

@@ -322,6 +322,12 @@ impl StreamContext {
             headers.insert(TRACE_PARENT_HEADER, self.traceparent.as_ref().unwrap());
         }
 
+        // override http headers that are set in the prompt target
+        let http_headers = endpoint_details.http_headers.clone().unwrap_or_default();
+        for (key, value) in http_headers.iter() {
+            headers.insert(key.as_str(), value.as_str());
+        }
+
         let call_args = CallArgs::new(
             ARCH_INTERNAL_CLUSTER_NAME,
             &path,

demos/currency_exchange/arch_config.yaml

@@ -12,6 +12,11 @@ llm_providers:
     provider_interface: openai
     model: gpt-4o
 
+endpoints:
+  frankfurther_api:
+    endpoint: api.frankfurter.dev
+    protocol: https
+
 system_prompt: |
   You are a helpful assistant.
 
@@ -26,7 +31,7 @@ prompt_targets:
     description: Get currency exchange rate from USD to other currencies
     parameters:
       - name: currency_symbol
-        description: the currency that needs conversion
+        description: currency symbol to convert from USD
         required: true
         type: str
         in_path: true
@@ -42,11 +47,6 @@ prompt_targets:
       name: frankfurther_api
       path: /v1/currencies
 
-endpoints:
-  frankfurther_api:
-    endpoint: api.frankfurter.dev
-    protocol: https
-
 tracing:
   random_sampling: 100
   trace_arch_internal: true

demos/stock_quote/README.md

@@ -0,0 +1,9 @@
+This demo shows how you can use a publicly hosted rest api that is protected by an access key.
+
+Before you start the demo make sure you set `OPENAI_API_KEY` and `TWELVEDATA_API_KEY`.
+
+To get `TWELVEDATA_API_KEY` please head over to https://twelvedata.com/.
+
+Following screenshot shows interaction with stock quote demo,
+
+![alt text](stock_quote_demo.png)

demos/stock_quote/arch_config.yaml

@@ -0,0 +1,69 @@
+version: v0.1
+
+listener:
+  address: 0.0.0.0
+  port: 10000
+  message_format: huggingface
+  connect_timeout: 0.005s
+
+llm_providers:
+  - name: gpt-4o
+    access_key: $OPENAI_API_KEY
+    provider_interface: openai
+    model: gpt-4o
+
+endpoints:
+  twelvedata_api:
+    endpoint: api.twelvedata.com
+    protocol: https
+
+system_prompt: |
+  You are a helpful assistant.
+
+prompt_guards:
+  input_guards:
+    jailbreak:
+      on_exception:
+        message: Looks like you're curious about my abilities, but I can only provide assistance for currency exchange.
+
+prompt_targets:
+  - name: stock_quote
+    description: get current stock exchange rate for a given symbol
+    parameters:
+      - name: symbol
+        description: Stock symbol
+        required: true
+        type: str
+    endpoint:
+      name: twelvedata_api
+      path: /quote
+      http_headers:
+        Authorization: "apikey $TWELVEDATA_API_KEY"
+    system_prompt: |
+      You are a helpful stock exchange assistant. You are given stock symbol along with its exchange rate in json format. Your task is to parse the data and present it in a human-readable format. Keep the details to highlevel and be concise.
+
+  - name: stock_quote_time_series
+    description: get historical stock exchange rate for a given symbol
+    parameters:
+      - name: symbol
+        description: Stock symbol
+        required: true
+        type: str
+      - name: interval
+        description: Time interval
+        default: 1day
+        enum:
+          - 1h
+          - 1day
+        type: str
+    endpoint:
+      name: twelvedata_api
+      path: /time_series
+      http_headers:
+        Authorization: "apikey $TWELVEDATA_API_KEY"
+    system_prompt: |
+      You are a helpful stock exchange assistant. You are given stock symbol along with its historical data in json format. Your task is to parse the data and present it in a human-readable format. Keep the details to highlevel only and be concise.
+
+tracing:
+  random_sampling: 100
+  trace_arch_internal: true

demos/stock_quote/docker-compose.yaml

@@ -0,0 +1,21 @@
+services:
+  chatbot_ui:
+    build:
+      context: ../shared/chatbot_ui
+    ports:
+      - "18080:8080"
+    environment:
+      # this is only because we are running the sample app in the same docker container environment as archgw
+      - CHAT_COMPLETION_ENDPOINT=http://host.docker.internal:10000/v1
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ./arch_config.yaml:/app/arch_config.yaml
+
+  jaeger:
+    build:
+      context: ../shared/jaeger
+    ports:
+      - "16686:16686"
+      - "4317:4317"
+      - "4318:4318"

demos/stock_quote/run_demo.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+# Function to start the demo
+start_demo() {
+  # Step 1: Check if .env file exists
+  if [ -f ".env" ]; then
+    echo ".env file already exists. Skipping creation."
+  else
+    # Step 2: Create `.env` file and set OpenAI key
+    if [ -z "$OPENAI_API_KEY" ]; then
+      echo "Error: OPENAI_API_KEY environment variable is not set for the demo."
+      exit 1
+    fi
+
+    echo "Creating .env file..."
+    echo "OPENAI_API_KEY=$OPENAI_API_KEY" > .env
+    echo ".env file created with OPENAI_API_KEY."
+  fi
+
+  # Step 3: Start Arch
+  echo "Starting Arch with arch_config.yaml..."
+  archgw up arch_config.yaml
+
+  # Step 4: Start developer services
+  echo "Starting Network Agent using Docker Compose..."
+  docker compose up -d  # Run in detached mode
+}
+
+# Function to stop the demo
+stop_demo() {
+  # Step 1: Stop Docker Compose services
+  echo "Stopping Network Agent using Docker Compose..."
+  docker compose down
+
+  # Step 2: Stop Arch
+  echo "Stopping Arch..."
+  archgw down
+}
+
+# Main script logic
+if [ "$1" == "down" ]; then
+  stop_demo
+else
+  # Default action is to bring the demo up
+  start_demo
+fi