diff --git a/arch/arch_config_schema.yaml b/arch/arch_config_schema.yaml index 1b32b730..3c186a55 100644 --- a/arch/arch_config_schema.yaml +++ b/arch/arch_config_schema.yaml @@ -107,7 +107,10 @@ properties: required: type: boolean default: - type: string + anyOf: + - type: string + - type: integer + - type: boolean description: type: string type: @@ -120,6 +123,8 @@ properties: type: boolean format: type: string + url_encode: + type: boolean additionalProperties: false required: - name diff --git a/crates/common/src/configuration.rs b/crates/common/src/configuration.rs index 069695ba..7a82786b 100644 --- a/crates/common/src/configuration.rs +++ b/crates/common/src/configuration.rs @@ -216,6 +216,7 @@ pub struct Parameter { pub enum_values: Option>, pub default: Option, pub in_path: Option, + pub url_encode: Option, pub format: Option, } diff --git a/crates/common/src/path.rs b/crates/common/src/path.rs index f11cb7b9..bd272629 100644 --- a/crates/common/src/path.rs +++ b/crates/common/src/path.rs @@ -7,7 +7,7 @@ use crate::configuration::Parameter; pub fn replace_params_in_path( path: &str, tool_params: &HashMap, - prompt_target_params: &[Parameter], + prompt_target_params: &HashMap, ) -> Result<(String, String, HashMap), String> { let mut query_string_replaced = String::new(); let mut current_param = String::new(); @@ -22,8 +22,16 @@ pub fn replace_params_in_path( in_param = false; let param_name = current_param.clone(); if let Some(value) = tool_params.get(¶m_name) { - let value = urlencoding::encode(value); - query_string_replaced.push_str(value.into_owned().as_str()); + let should_url_encode = prompt_target_params + .get(¶m_name) + .map(|param| param.url_encode.unwrap_or_default()) + .unwrap_or_default(); + if should_url_encode { + let value = urlencoding::encode(value); + query_string_replaced.push_str(value.into_owned().as_str()); + } else { + query_string_replaced.push_str(value); + } vars_replaced.insert(param_name.clone()); } else { return Err(format!("Missing value for parameter `{}`", param_name)); @@ -51,7 +59,7 @@ pub fn replace_params_in_path( } // add default values - for param in prompt_target_params.iter() { + for param in prompt_target_params.values() { if !vars_replaced.contains(¶m.name) && param.default.is_some() { params.insert(param.name.clone(), param.default.clone().unwrap()); if query_string_replaced.contains("?") { @@ -104,7 +112,11 @@ mod test { default: Some("US".to_string()), in_path: None, format: None, - }]; + url_encode: None, + }] + .into_iter() + .map(|param| (param.name.clone(), param)) + .collect(); let out_params: HashMap = vec![ ("country".to_string(), "US".to_string()), @@ -122,7 +134,7 @@ mod test { ); let out_params = HashMap::new(); - let prompt_target_params = vec![]; + let prompt_target_params: HashMap = HashMap::new(); let path = "/cluster.open-cluster-management.io/v1/managedclusters"; let params = vec![].into_iter().collect(); assert_eq!( diff --git a/crates/llm_gateway/src/stream_context.rs b/crates/llm_gateway/src/stream_context.rs index 528358a3..f5336676 100644 --- a/crates/llm_gateway/src/stream_context.rs +++ b/crates/llm_gateway/src/stream_context.rs @@ -87,8 +87,9 @@ impl StreamContext { )); debug!( - "request received: llm provider hint: {:?}, selected llm: {}", - self.get_http_request_header(ARCH_PROVIDER_HINT_HEADER), + "request received: llm provider hint: {}, selected llm: {}", + self.get_http_request_header(ARCH_PROVIDER_HINT_HEADER) + .unwrap_or_default(), self.llm_provider.as_ref().unwrap().name ); } diff --git a/crates/prompt_gateway/src/http_context.rs b/crates/prompt_gateway/src/http_context.rs index 1ff7f91d..fcea4071 100644 --- a/crates/prompt_gateway/src/http_context.rs +++ b/crates/prompt_gateway/src/http_context.rs @@ -144,7 +144,10 @@ impl HttpContext for StreamContext { if metadata.is_none() { metadata = Some(HashMap::new()); } - metadata.as_mut().unwrap().insert("optimize_context_window".to_string(), "true".to_string()); + metadata + .as_mut() + .unwrap() + .insert("optimize_context_window".to_string(), "true".to_string()); } } diff --git a/crates/prompt_gateway/src/stream_context.rs b/crates/prompt_gateway/src/stream_context.rs index e6db7f59..be4e35cc 100644 --- a/crates/prompt_gateway/src/stream_context.rs +++ b/crates/prompt_gateway/src/stream_context.rs @@ -4,7 +4,7 @@ use common::api::open_ai::{ to_server_events, ArchState, ChatCompletionStreamResponse, ChatCompletionsRequest, ChatCompletionsResponse, Message, ModelServerResponse, ToolCall, }; -use common::configuration::{Overrides, PromptTarget, Tracing}; +use common::configuration::{Overrides, Parameter, PromptTarget, Tracing}; use common::consts::{ ARCH_FC_MODEL_NAME, ARCH_FC_REQUEST_TIMEOUT_MS, ARCH_INTERNAL_CLUSTER_NAME, ARCH_UPSTREAM_HOST_HEADER, ASSISTANT_ROLE, MESSAGES_KEY, REQUEST_ID_HEADER, SYSTEM_ROLE, @@ -89,7 +89,7 @@ impl StreamContext { streaming_response: false, user_prompt: None, is_chat_completions_request: false, - overrides: overrides, + overrides, request_id: None, traceparent: None, _tracing: tracing, @@ -191,6 +191,10 @@ impl StreamContext { callout_context.response_handler_type = ResponseHandlerType::DefaultTarget; callout_context.prompt_target_name = Some(default_prompt_target.name.clone()); + debug!( + "prompt target name: {}", + callout_context.prompt_target_name.as_ref().unwrap() + ); if let Err(e) = self.http_call(call_args, callout_context) { warn!("error dispatching default prompt target request: {}", e); @@ -267,6 +271,10 @@ impl StreamContext { // update prompt target name from the tool call response callout_context.prompt_target_name = Some(self.tool_calls.as_ref().unwrap()[0].function.name.clone()); + debug!( + "prompt target name: {}", + callout_context.prompt_target_name.as_ref().unwrap() + ); self.schedule_api_call_request(callout_context); } @@ -283,7 +291,13 @@ impl StreamContext { .to_string(); let http_method = endpoint_details.method.clone().unwrap_or_default(); - let prompt_target_params = prompt_target.parameters.clone().unwrap_or_default(); + let prompt_target_params: HashMap = prompt_target + .parameters + .clone() + .unwrap_or_default() + .into_iter() + .map(|param| (param.name.clone(), param)) + .collect(); let (path, body) = match compute_request_path_body( &endpoint_path, diff --git a/crates/prompt_gateway/src/tools.rs b/crates/prompt_gateway/src/tools.rs index b3b3d07e..6899413f 100644 --- a/crates/prompt_gateway/src/tools.rs +++ b/crates/prompt_gateway/src/tools.rs @@ -23,7 +23,7 @@ pub fn filter_tool_params(tool_params: &HashMap) -> HashMap, - prompt_target_params: &[Parameter], + prompt_target_params: &HashMap, http_method: &HttpMethod, ) -> Result<(String, Option), String> { let tool_url_params = filter_tool_params(tool_params); @@ -55,6 +55,8 @@ pub fn compute_request_path_body( #[cfg(test)] mod test { + use std::collections::HashMap; + use common::configuration::{HttpMethod, Parameter}; #[test] @@ -76,7 +78,11 @@ mod test { default: Some("US".to_string()), in_path: None, format: None, - }]; + url_encode: None, + }] + .into_iter() + .map(|param| (param.name.clone(), param)) + .collect::>(); let http_method = HttpMethod::Get; let (path, body) = super::compute_request_path_body( endpoint_path, @@ -105,7 +111,11 @@ mod test { default: Some("US".to_string()), in_path: None, format: None, - }]; + url_encode: None, + }] + .into_iter() + .map(|param| (param.name.clone(), param)) + .collect::>(); let http_method = HttpMethod::Get; let (path, body) = super::compute_request_path_body( endpoint_path, @@ -139,7 +149,11 @@ mod test { default: Some("US".to_string()), in_path: None, format: None, - }]; + url_encode: None, + }] + .into_iter() + .map(|param| (param.name.clone(), param)) + .collect::>(); let http_method = HttpMethod::Get; let (path, body) = super::compute_request_path_body( endpoint_path, diff --git a/demos/samples_python/talk_to_s3/README.md b/demos/samples_python/talk_to_s3/README.md new file mode 100644 index 00000000..5cf7cd4d --- /dev/null +++ b/demos/samples_python/talk_to_s3/README.md @@ -0,0 +1,3 @@ +This demo uses s3 public APIs to provide a chat interface to your files. + +For this demo to work make sure you have access key and secret key for your aws account. diff --git a/demos/samples_python/talk_to_s3/arch_config.yaml b/demos/samples_python/talk_to_s3/arch_config.yaml new file mode 100644 index 00000000..2d8aea9f --- /dev/null +++ b/demos/samples_python/talk_to_s3/arch_config.yaml @@ -0,0 +1,71 @@ +version: v0.1 +listener: + address: 127.0.0.1 + port: 8080 #If you configure port 443, you'll need to update the listener with tls_certificates + message_format: huggingface + +overrides: + optimize_context_window: true + +endpoints: + s3_endpoint: + endpoint: host.docker.internal:8090 + http_host: s3.us-west-1.amazonaws.com + +llm_providers: + - name: OpenAI + provider_interface: openai + access_key: $OPENAI_API_KEY + model: gpt-4o + default: true + +prompt_targets: + - name: get_all_s3_buckets + description: get all s3 buckets owned by me + parameters: + - name: max-buckets + description: maximum number of buckets to return + type: integer + default: 5 + endpoint: + name: s3_endpoint + path: / + system_prompt: | + You are given an xml response with s3 buckets. Your task is to prepare a markdown with following details, + - bucket name | creation date | region | owner + + - name: get_s3_bucket_details + description: get details of a bucket + parameters: + - name: bucket_name + description: bucket name + required: true + type: string + - name: max-keys + description: maximum number of keys to return + type: integer + default: 5 + endpoint: + name: s3_endpoint + path: /{bucket_name} + system_prompt: | + Show details of a bucket. Specifically list the objects in the bucket along with their size and last modified date in a markdown table. Take following format as reference, + - object name | size | last modified date + + - name: get_s3_object_details + description: get details of a specific object + parameters: + - name: bucket_name + description: bucket name + required: true + type: string + - name: object_name + description: object name this could be a file or a prefix + type: string + required: true + url_encode: false + endpoint: + name: s3_endpoint + path: /{bucket_name}/{object_name} + system_prompt: | + show details of an object or a prefix diff --git a/demos/samples_python/talk_to_s3/docker-compose.yaml b/demos/samples_python/talk_to_s3/docker-compose.yaml new file mode 100644 index 00000000..10866704 --- /dev/null +++ b/demos/samples_python/talk_to_s3/docker-compose.yaml @@ -0,0 +1,29 @@ +services: + chatbot_ui: + build: + context: ../../shared/chatbot_ui + ports: + - "18080:8080" + environment: + # this is only because we are running the sample app in the same docker container environment as archgw + - CHAT_COMPLETION_ENDPOINT=http://host.docker.internal:10000/v1 + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ./arch_config.yaml:/app/arch_config.yaml + + jaeger: + build: + context: ../../shared/jaeger + ports: + - "16686:16686" + - "4317:4317" + - "4318:4318" + + sigv4_proxy: + image: public.ecr.aws/aws-observability/aws-sigv4-proxy:latest + ports: + - "8090:8080" + environment: + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:?error} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:?error} diff --git a/demos/samples_python/talk_to_s3/run_demo.sh b/demos/samples_python/talk_to_s3/run_demo.sh new file mode 100644 index 00000000..edb5504f --- /dev/null +++ b/demos/samples_python/talk_to_s3/run_demo.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + +# Function to start the demo +start_demo() { + # Step 1: Check if .env file exists + if [ -f ".env" ]; then + echo ".env file already exists. Skipping creation." + else + # Step 2: Create `.env` file and set OpenAI key + if [ -z "$OPENAI_API_KEY" ]; then + echo "Error: OPENAI_API_KEY environment variable is not set for the demo." + exit 1 + fi + + echo "Creating .env file..." + echo "OPENAI_API_KEY=$OPENAI_API_KEY" > .env + echo ".env file created with OPENAI_API_KEY." + fi + + # Step 3: Start Arch + echo "Starting Arch with arch_config.yaml..." + archgw up arch_config.yaml + + # Step 4: Start developer services + echo "Starting using Docker Compose..." + docker compose up -d # Run in detached mode +} + +# Function to stop the demo +stop_demo() { + # Step 1: Stop Docker Compose services + echo "Stopping using Docker Compose..." + docker compose down + + # Step 2: Stop Arch + echo "Stopping Arch..." + archgw down +} + +# Main script logic +if [ "$1" == "down" ]; then + stop_demo +else + # Default action is to bring the demo up + start_demo +fi