From 3b672e798c4a87c0216c0a3fe1bf75b26fd77793 Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Tue, 30 Sep 2025 15:01:06 -0700 Subject: [PATCH] more changes --- arch/envoy.template.yaml | 4 ++-- arch/tools/cli/config_generator.py | 7 +++++-- arch/tools/cli/utils.py | 16 ++++++++++++++-- .../brightstaff/src/handlers/agent_selector.rs | 4 ++-- .../src/handlers/integration_tests.rs | 8 ++++---- .../src/handlers/pipeline_processor.rs | 12 ++++++------ crates/common/src/configuration.rs | 4 ++-- demos/use_cases/rag_agent/arch_config.yaml | 12 +++++++++--- demos/use_cases/rag_agent/test.rest | 15 +++++++++++++++ 9 files changed, 59 insertions(+), 23 deletions(-) diff --git a/arch/envoy.template.yaml b/arch/envoy.template.yaml index 70b864fb..16ad0f9f 100644 --- a/arch/envoy.template.yaml +++ b/arch/envoy.template.yaml @@ -124,7 +124,7 @@ static_resources: domains: - "*" routes: - {% for provider in arch_llm_providers %} + {% for provider in arch_model_providers %} # if endpoint is set then use custom cluster for upstream llm {% if provider.endpoint %} {% set llm_cluster_name = provider.cluster_name %} @@ -522,7 +522,7 @@ static_resources: domains: - "*" routes: - {% for provider in arch_llm_providers %} + {% for provider in arch_model_providers %} # if endpoint is set then use custom cluster for upstream llm {% if provider.endpoint %} {% set llm_cluster_name = provider.cluster_name %} diff --git a/arch/tools/cli/config_generator.py b/arch/tools/cli/config_generator.py index ec188e82..647f55ba 100644 --- a/arch/tools/cli/config_generator.py +++ b/arch/tools/cli/config_generator.py @@ -92,7 +92,7 @@ def validate_and_render_schema(): agents = config_yaml.get("agents", []) for agent in agents: agent_name = agent.get("name") - agent_endpoint = agent.get("endpoint") + agent_endpoint = agent.get("url") if agent_name and agent_endpoint: urlparse_result = urlparse(agent_endpoint) @@ -145,6 +145,8 @@ def validate_and_render_schema(): model_name_keys = set() model_usage_name_keys = set() + print("listeners: ", listeners) + for listener in listeners: if ( listener.get("model_providers") is None @@ -163,6 +165,7 @@ def validate_and_render_schema(): ) model_name = model_provider.get("model") + print("Processing model_provider: ", model_provider) if model_name in model_name_keys: raise Exception( f"Duplicate model name {model_name}, please provide unique model name for each model_provider" @@ -306,7 +309,7 @@ def validate_and_render_schema(): target = alias_config.get("target") if target not in model_name_keys: raise Exception( - f"Model alias '{alias_name}' targets '{target}' which is not defined as a model. Available models: {', '.join(sorted(model_name_keys))}" + f"Model alias 2 - '{alias_name}' targets '{target}' which is not defined as a model. Available models: {', '.join(sorted(model_name_keys))}" ) arch_config_string = yaml.dump(config_yaml) diff --git a/arch/tools/cli/utils.py b/arch/tools/cli/utils.py index d84322a3..f734a14b 100644 --- a/arch/tools/cli/utils.py +++ b/arch/tools/cli/utils.py @@ -106,6 +106,8 @@ def convert_legacy_listeners( listener["model_providers"] = model_providers or [] model_provider_set = True llm_gateway_listener = listener + if not model_provider_set: + listeners.append(llm_gateway_listener) return listeners, llm_gateway_listener, prompt_gateway_listener @@ -116,8 +118,18 @@ def get_llm_provider_access_keys(arch_config_file): arch_config_yaml = yaml.safe_load(arch_config) access_key_list = [] + + # Convert legacy llm_providers to model_providers + if "llm_providers" in arch_config_yaml: + if "model_providers" in arch_config_yaml: + raise Exception( + "Please provide either llm_providers or model_providers, not both. llm_providers is deprecated, please use model_providers instead" + ) + arch_config_yaml["model_providers"] = arch_config_yaml["llm_providers"] + del arch_config_yaml["llm_providers"] + listeners, _, _ = convert_legacy_listeners( - arch_config_yaml.get("listeners"), arch_config_yaml.get("llm_providers") + arch_config_yaml.get("listeners"), arch_config_yaml.get("model_providers") ) for prompt_target in arch_config_yaml.get("prompt_targets", []): @@ -133,7 +145,7 @@ def get_llm_provider_access_keys(arch_config_file): access_key_list.append(v) for listener in listeners: - for llm_provider in listener.get("llm_providers", []): + for llm_provider in listener.get("model_providers", []): access_key = llm_provider.get("access_key") if access_key is not None: access_key_list.append(access_key) diff --git a/crates/brightstaff/src/handlers/agent_selector.rs b/crates/brightstaff/src/handlers/agent_selector.rs index c0b5d3fd..75c6c1ff 100644 --- a/crates/brightstaff/src/handlers/agent_selector.rs +++ b/crates/brightstaff/src/handlers/agent_selector.rs @@ -184,8 +184,8 @@ mod tests { fn create_test_agent_struct(name: &str) -> Agent { Agent { name: name.to_string(), - kind: "test".to_string(), - endpoint: "http://localhost:8080".to_string(), + kind: Some("test".to_string()), + url: "http://localhost:8080".to_string(), } } diff --git a/crates/brightstaff/src/handlers/integration_tests.rs b/crates/brightstaff/src/handlers/integration_tests.rs index 27dc7ae1..d5a21555 100644 --- a/crates/brightstaff/src/handlers/integration_tests.rs +++ b/crates/brightstaff/src/handlers/integration_tests.rs @@ -48,13 +48,13 @@ mod integration_tests { let agents = vec![ Agent { name: "filter-agent".to_string(), - kind: "filter".to_string(), - endpoint: "http://localhost:8081".to_string(), + kind: Some("filter".to_string()), + url: "http://localhost:8081".to_string(), }, Agent { name: "terminal-agent".to_string(), - kind: "terminal".to_string(), - endpoint: "http://localhost:8082".to_string(), + kind: Some("terminal".to_string()), + url: "http://localhost:8082".to_string(), }, ]; diff --git a/crates/brightstaff/src/handlers/pipeline_processor.rs b/crates/brightstaff/src/handlers/pipeline_processor.rs index d6f70e27..8c9063ac 100644 --- a/crates/brightstaff/src/handlers/pipeline_processor.rs +++ b/crates/brightstaff/src/handlers/pipeline_processor.rs @@ -24,23 +24,23 @@ pub enum PipelineError { /// Service for processing agent pipelines pub struct PipelineProcessor { client: reqwest::Client, - llm_endpoint: String, + url: String, } impl Default for PipelineProcessor { fn default() -> Self { Self { client: reqwest::Client::new(), - llm_endpoint: "http://localhost:11000/v1/chat/completions".to_string(), + url: "http://localhost:11000/v1/chat/completions".to_string(), } } } impl PipelineProcessor { - pub fn new(llm_endpoint: String) -> Self { + pub fn new(url: String) -> Self { Self { client: reqwest::Client::new(), - llm_endpoint, + url, } } @@ -116,7 +116,7 @@ impl PipelineProcessor { let response = self .client - .post(&self.llm_endpoint) + .post(&self.url) .headers(agent_headers) .body(request_body) .send() @@ -169,7 +169,7 @@ impl PipelineProcessor { let response = self .client - .post(&self.llm_endpoint) + .post(&self.url) .headers(agent_headers) .body(request_body) .send() diff --git a/crates/common/src/configuration.rs b/crates/common/src/configuration.rs index 1f8a8e24..d800834f 100644 --- a/crates/common/src/configuration.rs +++ b/crates/common/src/configuration.rs @@ -21,8 +21,8 @@ pub struct ModelAlias { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Agent { pub name: String, - pub kind: String, - pub endpoint: String, + pub kind: Option, + pub url: String, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/demos/use_cases/rag_agent/arch_config.yaml b/demos/use_cases/rag_agent/arch_config.yaml index 9be14463..4f52324f 100644 --- a/demos/use_cases/rag_agent/arch_config.yaml +++ b/demos/use_cases/rag_agent/arch_config.yaml @@ -1,6 +1,10 @@ version: v0.3.0 agents: + - name: query_rewriter + url: http://host.docker.internal:10500/v1/chat/completions + - name: context_builder + url: http://host.docker.internal:10501/v1/chat/completions - name: rag_agent url: http://host.docker.internal:10502/v1/chat/completions - name: research_agent @@ -15,7 +19,7 @@ model_providers: - model: openai/gpt-4o access_key: $OPENAI_API_KEY - model: ollama/llama3.1 - url: http://host.docker.internal:11434 + base_url: http://host.docker.internal:11434 model_aliases: fast-llm: @@ -24,11 +28,13 @@ model_aliases: target: gpt-4o listeners: - - type: agent + - type: agent_listener + name: agent_listener_1 port: 8001 router: arch_agent_router agents: - - agent: rag_agent + - name: rag_agent + agent: rag_agent description: virtual assistant for device contracts for simple queries filter_chain: - query_rewriter diff --git a/demos/use_cases/rag_agent/test.rest b/demos/use_cases/rag_agent/test.rest index 730ad9b2..df8e4f7b 100644 --- a/demos/use_cases/rag_agent/test.rest +++ b/demos/use_cases/rag_agent/test.rest @@ -43,6 +43,21 @@ Content-Type: application/json { "model": "{{model}}", + "messages": [ + { + "role": "user", + "content": "what is the effective date of the master agreement for arcadyan" + } + ], + "stream": true +} + +### models_listeners test +POST http://localhost:12000/v1/chat/completions +Content-Type: application/json + +{ + "model": "gpt-4o-mini", "messages": [ { "role": "user",