diff --git a/arch/arch_config_schema.yaml b/arch/arch_config_schema.yaml index 67e8d73e..feaaa90f 100644 --- a/arch/arch_config_schema.yaml +++ b/arch/arch_config_schema.yaml @@ -52,6 +52,8 @@ properties: - https http_host: type: string + agent_orchestrator: + type: boolean additionalProperties: false required: - endpoint @@ -93,7 +95,6 @@ properties: additionalProperties: false required: - name - - model overrides: type: object properties: diff --git a/arch/envoy.template.yaml b/arch/envoy.template.yaml index ca722b7c..cac17187 100644 --- a/arch/envoy.template.yaml +++ b/arch/envoy.template.yaml @@ -142,6 +142,19 @@ static_resources: cluster: {{ llm_cluster_name }} timeout: 60s {% endfor %} + + {% if agent_orchestrator %} + - match: + prefix: "/" + headers: + - name: "x-arch-llm-provider" + string_match: + exact: {{ agent_orchestrator }} + route: + auto_host_rewrite: true + cluster: {{ agent_orchestrator }} + timeout: 60s + {% endif %} http_filters: - name: envoy.filters.http.compressor typed_config: diff --git a/arch/tools/cli/config_generator.py b/arch/tools/cli/config_generator.py index f0aae28a..615a3df5 100644 --- a/arch/tools/cli/config_generator.py +++ b/arch/tools/cli/config_generator.py @@ -150,6 +150,12 @@ def validate_and_render_schema(): if llm_gateway_listener.get("timeout") == None: llm_gateway_listener["timeout"] = "10s" + agent_orchestrator = None + for name, endpoint_details in endpoints.items(): + if endpoint_details.get("agent_orchestrator", False): + agent_orchestrator = name + break + data = { "prompt_gateway_listener": prompt_gateway_listener, "llm_gateway_listener": llm_gateway_listener, @@ -159,6 +165,7 @@ def validate_and_render_schema(): "arch_llm_providers": config_yaml["llm_providers"], "arch_tracing": arch_tracing, "local_llms": llms_with_endpoint, + "agent_orchestrator": agent_orchestrator, } rendered = template.render(data) diff --git a/crates/common/src/configuration.rs b/crates/common/src/configuration.rs index 0c51810e..616536a4 100644 --- a/crates/common/src/configuration.rs +++ b/crates/common/src/configuration.rs @@ -160,7 +160,7 @@ pub struct LlmProvider { pub name: String, pub provider_interface: LlmProviderType, pub access_key: Option, - pub model: String, + pub model: Option, pub default: Option, pub stream: Option, pub endpoint: Option, @@ -177,6 +177,7 @@ impl Display for LlmProvider { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Endpoint { pub endpoint: Option, + pub agent_orchestrator: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/llm_gateway/src/filter_context.rs b/crates/llm_gateway/src/filter_context.rs index 0edba456..fc31355a 100644 --- a/crates/llm_gateway/src/filter_context.rs +++ b/crates/llm_gateway/src/filter_context.rs @@ -1,6 +1,7 @@ use crate::metrics::Metrics; use crate::stream_context::StreamContext; use common::configuration::Configuration; +use common::configuration::Overrides; use common::consts::OTEL_COLLECTOR_HTTP; use common::consts::OTEL_POST_PATH; use common::http::CallArgs; @@ -31,6 +32,7 @@ pub struct FilterContext { callouts: RefCell>, llm_providers: Option>, traces_queue: Arc>>, + overrides: Rc>, } impl FilterContext { @@ -40,6 +42,7 @@ impl FilterContext { metrics: Rc::new(Metrics::new()), llm_providers: None, traces_queue: Arc::new(Mutex::new(VecDeque::new())), + overrides: Rc::new(None), } } } @@ -69,6 +72,7 @@ impl RootContext for FilterContext { }; ratelimit::ratelimits(Some(config.ratelimits.unwrap_or_default())); + self.overrides = Rc::new(config.overrides); match config.llm_providers.try_into() { Ok(llm_providers) => self.llm_providers = Some(Rc::new(llm_providers)), @@ -93,6 +97,7 @@ impl RootContext for FilterContext { .expect("LLM Providers must exist when Streams are being created"), ), Arc::clone(&self.traces_queue), + Rc::clone(&self.overrides), ))) } diff --git a/crates/llm_gateway/src/stream_context.rs b/crates/llm_gateway/src/stream_context.rs index 81b2d14b..6a104784 100644 --- a/crates/llm_gateway/src/stream_context.rs +++ b/crates/llm_gateway/src/stream_context.rs @@ -3,7 +3,7 @@ use common::api::open_ai::{ ChatCompletionStreamResponseServerEvents, ChatCompletionsRequest, ChatCompletionsResponse, Message, StreamOptions, }; -use common::configuration::LlmProvider; +use common::configuration::{LlmProvider, LlmProviderType, Overrides}; use common::consts::{ ARCH_PROVIDER_HINT_HEADER, ARCH_ROUTING_HEADER, CHAT_COMPLETIONS_PATH, HEALTHZ_PATH, RATELIMIT_SELECTOR_HEADER_KEY, REQUEST_ID_HEADER, TRACE_PARENT_HEADER, @@ -42,6 +42,7 @@ pub struct StreamContext { request_body_sent_time: Option, user_message: Option, traces_queue: Arc>>, + overrides: Rc>, } impl StreamContext { @@ -50,10 +51,12 @@ impl StreamContext { metrics: Rc, llm_providers: Rc, traces_queue: Arc>>, + overrides: Rc>, ) -> Self { StreamContext { context_id, metrics, + overrides, ratelimit_selector: None, streaming_response: false, response_tokens: 0, @@ -91,7 +94,12 @@ impl StreamContext { self.get_http_request_header(ARCH_PROVIDER_HINT_HEADER) .unwrap_or_default(), self.llm_provider.as_ref().unwrap().name, - self.llm_provider.as_ref().unwrap().model + self.llm_provider + .as_ref() + .unwrap() + .model + .as_ref() + .unwrap_or(&String::new()) ); } @@ -184,24 +192,42 @@ impl HttpContext for StreamContext { return Action::Continue; } - self.select_llm_provider(); + let routing_header_value = self.get_http_request_header(ARCH_ROUTING_HEADER); - // if endpoint is not set then use provider name as routing header so envoy can resolve the cluster name - if self.llm_provider().endpoint.is_none() { + let use_agent_orchestrator = match self.overrides.as_ref() { + Some(overrides) => overrides.use_agent_orchestrator.unwrap_or_default(), + None => false, + }; + + if let Some(routing_header_value) = routing_header_value.as_ref() { + debug!("routing header already set: {}", routing_header_value); + self.llm_provider = Some(Rc::new(LlmProvider { + name: routing_header_value.to_string(), + provider_interface: LlmProviderType::OpenAI, + access_key: None, + endpoint: None, + model: None, + default: None, + stream: None, + port: None, + rate_limits: None, + })); + } else { + self.select_llm_provider(); + debug!("setting routing header to: {}", self.llm_provider().name); self.add_http_request_header( ARCH_ROUTING_HEADER, &self.llm_provider().provider_interface.to_string(), ); - } else { - self.add_http_request_header(ARCH_ROUTING_HEADER, &self.llm_provider().name); - } - - if let Err(error) = self.modify_auth_headers() { - // ensure that the provider has an endpoint if the access key is missing else return a bad request - if self.llm_provider.as_ref().unwrap().endpoint.is_none() { - self.send_server_error(error, Some(StatusCode::BAD_REQUEST)); + if let Err(error) = self.modify_auth_headers() { + // ensure that the provider has an endpoint if the access key is missing else return a bad request + if self.llm_provider.as_ref().unwrap().endpoint.is_none() && !use_agent_orchestrator + { + self.send_server_error(error, Some(StatusCode::BAD_REQUEST)); + } } } + self.delete_content_length_header(); self.save_ratelimit_header(); @@ -267,7 +293,7 @@ impl HttpContext for StreamContext { // remove metadata from the request body //TODO: move this to prompt gateway - deserialized_body.metadata = None; + // deserialized_body.metadata = None; // delete model key from message array for message in deserialized_body.messages.iter_mut() { message.model = None; @@ -280,10 +306,24 @@ impl HttpContext for StreamContext { .last() .cloned(); - // override model name from the llm provider - deserialized_body - .model - .clone_from(&self.llm_provider.as_ref().unwrap().model); + let model_name = match self.llm_provider.as_ref() { + Some(llm_provider) => match llm_provider.model.as_ref() { + Some(model) => model, + None => "--", + }, + None => "--", + }; + + deserialized_body.model = model_name.to_string(); + + // if use_agent_orchestrator || self.llm_provider.as_ref().unwrap().model.is_none() { + // deserialized_body.model = "None".to_string() + // } else { + // // override model name from the llm provider + // deserialized_body + // .model + // .clone_from(&self.llm_provider.as_ref().unwrap().model.as_ref().unwrap()); + // } let chat_completion_request_str = serde_json::to_string(&deserialized_body).unwrap(); trace!( diff --git a/crates/prompt_gateway/src/filter_context.rs b/crates/prompt_gateway/src/filter_context.rs index f782dea2..d5368686 100644 --- a/crates/prompt_gateway/src/filter_context.rs +++ b/crates/prompt_gateway/src/filter_context.rs @@ -1,6 +1,8 @@ use crate::metrics::Metrics; use crate::stream_context::StreamContext; -use common::configuration::{Configuration, Overrides, PromptGuards, PromptTarget, Tracing}; +use common::configuration::{ + Configuration, Endpoint, Overrides, PromptGuards, PromptTarget, Tracing, +}; use common::http::Client; use common::stats::Gauge; use log::trace; @@ -21,6 +23,7 @@ pub struct FilterContext { overrides: Rc>, system_prompt: Rc>, prompt_targets: Rc>, + endpoints: Rc>>, prompt_guards: Rc, tracing: Rc>, } @@ -34,6 +37,7 @@ impl FilterContext { prompt_targets: Rc::new(HashMap::new()), overrides: Rc::new(None), prompt_guards: Rc::new(PromptGuards::default()), + endpoints: Rc::new(None), tracing: Rc::new(None), } } @@ -73,6 +77,7 @@ impl RootContext for FilterContext { } self.system_prompt = Rc::new(config.system_prompt); self.prompt_targets = Rc::new(prompt_targets); + self.endpoints = Rc::new(config.endpoints); if let Some(prompt_guards) = config.prompt_guards { self.prompt_guards = Rc::new(prompt_guards) @@ -94,6 +99,7 @@ impl RootContext for FilterContext { Rc::clone(&self.metrics), Rc::clone(&self.system_prompt), Rc::clone(&self.prompt_targets), + Rc::clone(&self.endpoints), Rc::clone(&self.overrides), Rc::clone(&self.tracing), ))) diff --git a/crates/prompt_gateway/src/http_context.rs b/crates/prompt_gateway/src/http_context.rs index 4f9b5cfd..ba3e71bc 100644 --- a/crates/prompt_gateway/src/http_context.rs +++ b/crates/prompt_gateway/src/http_context.rs @@ -4,7 +4,7 @@ use common::{ self, ArchState, ChatCompletionStreamResponse, ChatCompletionTool, ChatCompletionsRequest, }, consts::{ - ARCH_FC_MODEL_NAME, ARCH_INTERNAL_CLUSTER_NAME, ARCH_STATE_HEADER, + ARCH_FC_MODEL_NAME, ARCH_INTERNAL_CLUSTER_NAME, ARCH_ROUTING_HEADER, ARCH_STATE_HEADER, ARCH_UPSTREAM_HOST_HEADER, ASSISTANT_ROLE, CHAT_COMPLETIONS_PATH, HEALTHZ_PATH, MODEL_SERVER_NAME, MODEL_SERVER_REQUEST_TIMEOUT_MS, REQUEST_ID_HEADER, TOOL_ROLE, TRACE_PARENT_HEADER, USER_ROLE, @@ -33,6 +33,28 @@ impl HttpContext for StreamContext { // manipulate the body in benign ways e.g., compression. self.set_http_request_header("content-length", None); + if let Some(overrides) = self.overrides.as_ref() { + if overrides.use_agent_orchestrator.unwrap_or_default() { + // get endpoint that has agent_orchestrator set to true + if let Some(endpoints) = self.endpoints.as_ref() { + let agent_orchestrator = endpoints + .iter() + .find(|(_, endpoint)| endpoint.agent_orchestrator.unwrap_or_default()) + .map(|(name, _)| name.clone()); + if let Some(agent_orchestrator_name) = agent_orchestrator { + debug!( + "Setting ARCH_PROVIDER_HINT_HEADER to {}", + agent_orchestrator_name + ); + self.set_http_request_header( + ARCH_ROUTING_HEADER, + Some(&agent_orchestrator_name), + ); + }; + } + } + } + let request_path = self.get_http_request_header(":path").unwrap_or_default(); if request_path == HEALTHZ_PATH { self.send_http_response(200, vec![], None); @@ -49,6 +71,7 @@ impl HttpContext for StreamContext { self.request_id = self.get_http_request_header(REQUEST_ID_HEADER); self.traceparent = self.get_http_request_header(TRACE_PARENT_HEADER); + Action::Continue } diff --git a/crates/prompt_gateway/src/stream_context.rs b/crates/prompt_gateway/src/stream_context.rs index 8ec7ee01..e0a319ef 100644 --- a/crates/prompt_gateway/src/stream_context.rs +++ b/crates/prompt_gateway/src/stream_context.rs @@ -4,7 +4,7 @@ use common::api::open_ai::{ to_server_events, ArchState, ChatCompletionStreamResponse, ChatCompletionsRequest, ChatCompletionsResponse, Message, ToolCall, }; -use common::configuration::{Overrides, PromptTarget, Tracing}; +use common::configuration::{Endpoint, Overrides, PromptTarget, Tracing}; use common::consts::{ API_REQUEST_TIMEOUT_MS, ARCH_FC_MODEL_NAME, ARCH_INTERNAL_CLUSTER_NAME, ARCH_UPSTREAM_HOST_HEADER, ASSISTANT_ROLE, DEFAULT_TARGET_REQUEST_TIMEOUT_MS, MESSAGES_KEY, @@ -46,6 +46,7 @@ pub struct StreamCallContext { pub struct StreamContext { system_prompt: Rc>, pub prompt_targets: Rc>, + pub endpoints: Rc>>, pub overrides: Rc>, pub metrics: Rc, pub callouts: RefCell>, @@ -72,6 +73,7 @@ impl StreamContext { metrics: Rc, system_prompt: Rc>, prompt_targets: Rc>, + endpoints: Rc>>, overrides: Rc>, tracing: Rc>, ) -> Self { @@ -80,6 +82,7 @@ impl StreamContext { metrics, system_prompt, prompt_targets, + endpoints, callouts: RefCell::new(HashMap::new()), chat_completions_request: None, tool_calls: None, @@ -312,6 +315,51 @@ impl StreamContext { callout_context.prompt_target_name = Some(self.tool_calls.as_ref().unwrap()[0].function.name.clone()); + if let Some(overrides) = self.overrides.as_ref() { + if overrides.use_agent_orchestrator.unwrap_or_default() { + let mut metadata = HashMap::new(); + metadata.insert("use_agent_orchestrator".to_string(), "true".to_string()); + + metadata.insert( + "Agent-Name".to_string(), + callout_context + .prompt_target_name + .as_ref() + .unwrap() + .to_string(), + ); + + if let Some(overrides) = self.overrides.as_ref() { + if overrides.optimize_context_window.unwrap_or_default() { + metadata.insert("optimize_context_window".to_string(), "true".to_string()); + } + } + + if let Some(overrides) = self.overrides.as_ref() { + if overrides.use_agent_orchestrator.unwrap_or_default() { + metadata.insert("use_agent_orchestrator".to_string(), "true".to_string()); + } + } + + let messages = self.construct_llm_messages(&callout_context); + + let chat_completion_request = ChatCompletionsRequest { + model: callout_context.request_body.model.clone(), + messages, + tools: None, + stream: callout_context.request_body.stream, + stream_options: callout_context.request_body.stream_options.clone(), + metadata: Some(metadata), + }; + + let body_str = serde_json::to_string(&chat_completion_request).unwrap(); + debug!("sending request to llm agent: {}", body_str); + self.set_http_request_body(0, self.request_body_size, body_str.as_bytes()); + self.resume_http_request(); + return; + } + } + self.schedule_api_call_request(callout_context); } diff --git a/crates/prompt_gateway/src/tools.rs b/crates/prompt_gateway/src/tools.rs index e1c432e2..c909a2dd 100644 --- a/crates/prompt_gateway/src/tools.rs +++ b/crates/prompt_gateway/src/tools.rs @@ -8,7 +8,9 @@ pub fn filter_tool_params(tool_params: &Option>) -> HashM if tool_params.is_none() { return HashMap::new(); } - tool_params.as_ref().unwrap() + tool_params + .as_ref() + .unwrap() .iter() .filter(|(_, value)| value.is_number() || value.is_string() || value.is_bool()) .map(|(key, value)| match value { diff --git a/demos/use_cases/orchestrating_agents/arch_config.yaml b/demos/use_cases/orchestrating_agents/arch_config.yaml index df867aa9..9e90f158 100644 --- a/demos/use_cases/orchestrating_agents/arch_config.yaml +++ b/demos/use_cases/orchestrating_agents/arch_config.yaml @@ -7,12 +7,20 @@ listeners: message_format: openai timeout: 30s + egress_traffic: + address: 0.0.0.0 + port: 12000 + message_format: openai + timeout: 30s + overrides: use_agent_orchestrator: true endpoints: - triage_service: + agent_gateway: + agent_orchestrator: true endpoint: host.docker.internal:18083 + connect_timeout: 0.005s llm_providers: - name: gpt-4o-mini @@ -27,30 +35,12 @@ system_prompt: | prompt_targets: - name: sales_agent description: handles queries related to sales and purchases - endpoint: - name: triage_service - path: /sales - http_method: POST - pass_context: true - auto_llm_dispatch_on_response: false - name: issues_and_repairs description: handles issues, repairs, or refunds - endpoint: - name: triage_service - path: /issues - http_method: POST - pass_context: true - auto_llm_dispatch_on_response: false - name: escalate_to_human description: escalates to human agent - endpoint: - name: triage_service - path: /escalate - http_method: POST - pass_context: true - auto_llm_dispatch_on_response: false tracing: random_sampling: 100 diff --git a/demos/use_cases/orchestrating_agents/main.py b/demos/use_cases/orchestrating_agents/main.py index cccdbbd4..db178db0 100644 --- a/demos/use_cases/orchestrating_agents/main.py +++ b/demos/use_cases/orchestrating_agents/main.py @@ -1,12 +1,15 @@ import json import os import random +import time from typing import Any, Dict, List from fastapi import FastAPI, Response from datetime import datetime, date, timedelta, timezone import logging +import openai from pydantic import BaseModel, Field from opentelemetry import trace +from fastapi.responses import StreamingResponse from opentelemetry.sdk.trace import TracerProvider from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter @@ -57,19 +60,103 @@ class Choice(BaseModel): message: Message -@app.post("/sales") -async def sales_agent(req: ChatCompletionsRequest, res: Response): - logger.info(f"sales: received messages: {req.messages}") - return "I am a sales agent, how can I help you?" +class ChatCompletionResponse(BaseModel): + choices: List[Choice] + metadata: Dict[str, Any] = None -@app.post("/issues") -async def issues_agent(req: ChatCompletionsRequest, res: Response): - logger.info(f"issues: received messages: {req.messages}") - return "I am a issues agent, how can I help you?" +class ChunkChoice(BaseModel): + delta: Message -@app.post("/escalate") -async def escalate_agent(req: ChatCompletionsRequest, res: Response): - logger.info(f"escalates: received messages: {req.messages}") - return "You're talking to a human, how can I help you?" +class ChatCompletionStreamResponse(BaseModel): + model: str + choices: List[ChunkChoice] + + +client = openai.OpenAI(base_url="http://host.docker.internal:12000/v1", api_key="--") + +agent_map = { + "sales_agent": { + "role": "sales agent", + "instructions": "You are a sales agent for ACME Inc." + "Always answer in a sentence or less." + "Follow the following routine with the user:" + "1. Ask them about any problems in their life related to catching roadrunners.\n" + "2. Casually mention one of ACME's crazy made-up products can help.\n" + " - Don't mention price.\n" + "3. Once the user is bought in, drop a ridiculous price.\n" + "4. Only after everything, and if the user says yes, " + "tell them a crazy caveat and execute their order.\n" + "", + }, + "issues_and_repairs": { + "role": "issues and repairs agent", + "instructions": "You are a customer support agent for ACME Inc." + "Always answer in a sentence or less." + "Follow the following routine with the user:" + "1. First, ask probing questions and understand the user's problem deeper.\n" + " - unless the user has already provided a reason.\n" + "2. Propose a fix (make one up).\n" + "3. ONLY if not satisfied, offer a refund.\n" + "4. If accepted, search for the ID and then execute refund." + "", + }, + "escalate_to_human": { + "role": "human agent", + "instructions": """Pretend you are a human trying to address the user's problem.""", + }, + "unknown agent": { + "role": "llm agent", + "instructions": "You are an LLM agent. You can do anything you want.", + }, +} + + +@app.post("/v1/chat/completions") +async def completion_api(req: ChatCompletionsRequest): + if req.metadata is None: + req.metadata = {} + agent_name = req.metadata.get("Agent-Name", "unknown agent") + logger.info(f"agent: {agent_name}") + + def stream(): + agent_role = agent_map.get(agent_name)["role"] + agent_instructions = agent_map.get(agent_name)["instructions"] + system_prompt = "You are a " + agent_role + ". " + agent_instructions + messages = [{"role": "system", "content": system_prompt}] + for message in req.messages: + messages.append({"role": message.role, "content": message.content}) + completion = client.chat.completions.create( + model="--", + messages=messages, + stream=True, + ) + for line in completion: + if line.choices and len(line.choices) > 0 and line.choices[0].delta: + chunk_response_str = json.dumps(line.model_dump()) + yield "data: " + chunk_response_str + "\n\n" + yield "data: [DONE]" + "\n\n" + + # content = agent_map.get(agent_name) + + # for c in content: + # resp = ChatCompletionStreamResponse( + # model="--", + # choices=[ + # ChunkChoice( + # delta=Message( + # role="assistant", + # content=c, + # ) + # ) + # ], + # ) + # # random sleep between 10m and 50ms + # time.sleep(random.randint(10, 50) / 1000) + + # yield "data: " + json.dumps(resp.model_dump()) + "\n\n" + + # yield "data: [DONE]" + "\n\n" + + return StreamingResponse(stream(), media_type="text/event-stream") diff --git a/demos/use_cases/orchestrating_agents/poetry.lock b/demos/use_cases/orchestrating_agents/poetry.lock index 863beca3..3cf3ce5b 100644 --- a/demos/use_cases/orchestrating_agents/poetry.lock +++ b/demos/use_cases/orchestrating_agents/poetry.lock @@ -204,6 +204,17 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools", "tox"] +[[package]] +name = "distro" +version = "1.9.0" +description = "Distro - an OS platform information API" +optional = false +python-versions = ">=3.6" +files = [ + {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, + {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, +] + [[package]] name = "exceptiongroup" version = "1.2.2" @@ -329,6 +340,51 @@ files = [ {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, ] +[[package]] +name = "httpcore" +version = "1.0.7" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"}, + {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "idna" version = "3.10" @@ -366,6 +422,116 @@ perf = ["ipython"] test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] type = ["pytest-mypy"] +[[package]] +name = "jiter" +version = "0.9.0" +description = "Fast iterable JSON parser." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jiter-0.9.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:816ec9b60fdfd1fec87da1d7ed46c66c44ffec37ab2ef7de5b147b2fce3fd5ad"}, + {file = "jiter-0.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b1d3086f8a3ee0194ecf2008cf81286a5c3e540d977fa038ff23576c023c0ea"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1339f839b91ae30b37c409bf16ccd3dc453e8b8c3ed4bd1d6a567193651a4a51"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ffba79584b3b670fefae66ceb3a28822365d25b7bf811e030609a3d5b876f538"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cfc7d0a8e899089d11f065e289cb5b2daf3d82fbe028f49b20d7b809193958d"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e00a1a2bbfaaf237e13c3d1592356eab3e9015d7efd59359ac8b51eb56390a12"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1d9870561eb26b11448854dce0ff27a9a27cb616b632468cafc938de25e9e51"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9872aeff3f21e437651df378cb75aeb7043e5297261222b6441a620218b58708"}, + {file = "jiter-0.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1fd19112d1049bdd47f17bfbb44a2c0001061312dcf0e72765bfa8abd4aa30e5"}, + {file = "jiter-0.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6ef5da104664e526836070e4a23b5f68dec1cc673b60bf1edb1bfbe8a55d0678"}, + {file = "jiter-0.9.0-cp310-cp310-win32.whl", hash = "sha256:cb12e6d65ebbefe5518de819f3eda53b73187b7089040b2d17f5b39001ff31c4"}, + {file = "jiter-0.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:c43ca669493626d8672be3b645dbb406ef25af3f4b6384cfd306da7eb2e70322"}, + {file = "jiter-0.9.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6c4d99c71508912a7e556d631768dcdef43648a93660670986916b297f1c54af"}, + {file = "jiter-0.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8f60fb8ce7df529812bf6c625635a19d27f30806885139e367af93f6e734ef58"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51c4e1a4f8ea84d98b7b98912aa4290ac3d1eabfde8e3c34541fae30e9d1f08b"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f4c677c424dc76684fea3e7285a7a2a7493424bea89ac441045e6a1fb1d7b3b"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2221176dfec87f3470b21e6abca056e6b04ce9bff72315cb0b243ca9e835a4b5"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3c7adb66f899ffa25e3c92bfcb593391ee1947dbdd6a9a970e0d7e713237d572"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c98d27330fdfb77913c1097a7aab07f38ff2259048949f499c9901700789ac15"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eda3f8cc74df66892b1d06b5d41a71670c22d95a1ca2cbab73654745ce9d0419"}, + {file = "jiter-0.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:dd5ab5ddc11418dce28343123644a100f487eaccf1de27a459ab36d6cca31043"}, + {file = "jiter-0.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:42f8a68a69f047b310319ef8e2f52fdb2e7976fb3313ef27df495cf77bcad965"}, + {file = "jiter-0.9.0-cp311-cp311-win32.whl", hash = "sha256:a25519efb78a42254d59326ee417d6f5161b06f5da827d94cf521fed961b1ff2"}, + {file = "jiter-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:923b54afdd697dfd00d368b7ccad008cccfeb1efb4e621f32860c75e9f25edbd"}, + {file = "jiter-0.9.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:7b46249cfd6c48da28f89eb0be3f52d6fdb40ab88e2c66804f546674e539ec11"}, + {file = "jiter-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:609cf3c78852f1189894383cf0b0b977665f54cb38788e3e6b941fa6d982c00e"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d726a3890a54561e55a9c5faea1f7655eda7f105bd165067575ace6e65f80bb2"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2e89dc075c1fef8fa9be219e249f14040270dbc507df4215c324a1839522ea75"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04e8ffa3c353b1bc4134f96f167a2082494351e42888dfcf06e944f2729cbe1d"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:203f28a72a05ae0e129b3ed1f75f56bc419d5f91dfacd057519a8bd137b00c42"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fca1a02ad60ec30bb230f65bc01f611c8608b02d269f998bc29cca8619a919dc"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:237e5cee4d5d2659aaf91bbf8ec45052cc217d9446070699441a91b386ae27dc"}, + {file = "jiter-0.9.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:528b6b71745e7326eed73c53d4aa57e2a522242320b6f7d65b9c5af83cf49b6e"}, + {file = "jiter-0.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9f48e86b57bc711eb5acdfd12b6cb580a59cc9a993f6e7dcb6d8b50522dcd50d"}, + {file = "jiter-0.9.0-cp312-cp312-win32.whl", hash = "sha256:699edfde481e191d81f9cf6d2211debbfe4bd92f06410e7637dffb8dd5dfde06"}, + {file = "jiter-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:099500d07b43f61d8bd780466d429c45a7b25411b334c60ca875fa775f68ccb0"}, + {file = "jiter-0.9.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:2764891d3f3e8b18dce2cff24949153ee30c9239da7c00f032511091ba688ff7"}, + {file = "jiter-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:387b22fbfd7a62418d5212b4638026d01723761c75c1c8232a8b8c37c2f1003b"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d8da8629ccae3606c61d9184970423655fb4e33d03330bcdfe52d234d32f69"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1be73d8982bdc278b7b9377426a4b44ceb5c7952073dd7488e4ae96b88e1103"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2228eaaaa111ec54b9e89f7481bffb3972e9059301a878d085b2b449fbbde635"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:11509bfecbc319459647d4ac3fd391d26fdf530dad00c13c4dadabf5b81f01a4"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f22238da568be8bbd8e0650e12feeb2cfea15eda4f9fc271d3b362a4fa0604d"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:17f5d55eb856597607562257c8e36c42bc87f16bef52ef7129b7da11afc779f3"}, + {file = "jiter-0.9.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6a99bed9fbb02f5bed416d137944419a69aa4c423e44189bc49718859ea83bc5"}, + {file = "jiter-0.9.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e057adb0cd1bd39606100be0eafe742de2de88c79df632955b9ab53a086b3c8d"}, + {file = "jiter-0.9.0-cp313-cp313-win32.whl", hash = "sha256:f7e6850991f3940f62d387ccfa54d1a92bd4bb9f89690b53aea36b4364bcab53"}, + {file = "jiter-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:c8ae3bf27cd1ac5e6e8b7a27487bf3ab5f82318211ec2e1346a5b058756361f7"}, + {file = "jiter-0.9.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f0b2827fb88dda2cbecbbc3e596ef08d69bda06c6f57930aec8e79505dc17001"}, + {file = "jiter-0.9.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:062b756ceb1d40b0b28f326cba26cfd575a4918415b036464a52f08632731e5a"}, + {file = "jiter-0.9.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6f7838bc467ab7e8ef9f387bd6de195c43bad82a569c1699cb822f6609dd4cdf"}, + {file = "jiter-0.9.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4a2d16360d0642cd68236f931b85fe50288834c383492e4279d9f1792e309571"}, + {file = "jiter-0.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e84ed1c9c9ec10bbb8c37f450077cbe3c0d4e8c2b19f0a49a60ac7ace73c7452"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f3c848209ccd1bfa344a1240763975ca917de753c7875c77ec3034f4151d06c"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7825f46e50646bee937e0f849d14ef3a417910966136f59cd1eb848b8b5bb3e4"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d82a811928b26d1a6311a886b2566f68ccf2b23cf3bfed042e18686f1f22c2d7"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c058ecb51763a67f019ae423b1cbe3fa90f7ee6280c31a1baa6ccc0c0e2d06e"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9897115ad716c48f0120c1f0c4efae348ec47037319a6c63b2d7838bb53aaef4"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:351f4c90a24c4fb8c87c6a73af2944c440494ed2bea2094feecacb75c50398ae"}, + {file = "jiter-0.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d45807b0f236c485e1e525e2ce3a854807dfe28ccf0d013dd4a563395e28008a"}, + {file = "jiter-0.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1537a890724ba00fdba21787010ac6f24dad47f763410e9e1093277913592784"}, + {file = "jiter-0.9.0-cp38-cp38-win32.whl", hash = "sha256:e3630ec20cbeaddd4b65513fa3857e1b7c4190d4481ef07fb63d0fad59033321"}, + {file = "jiter-0.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:2685f44bf80e95f8910553bf2d33b9c87bf25fceae6e9f0c1355f75d2922b0ee"}, + {file = "jiter-0.9.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:9ef340fae98065071ccd5805fe81c99c8f80484e820e40043689cf97fb66b3e2"}, + {file = "jiter-0.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:efb767d92c63b2cd9ec9f24feeb48f49574a713870ec87e9ba0c2c6e9329c3e2"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:113f30f87fb1f412510c6d7ed13e91422cfd329436364a690c34c8b8bd880c42"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8793b6df019b988526f5a633fdc7456ea75e4a79bd8396a3373c371fc59f5c9b"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7a9aaa5102dba4e079bb728076fadd5a2dca94c05c04ce68004cfd96f128ea34"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d838650f6ebaf4ccadfb04522463e74a4c378d7e667e0eb1865cfe3990bfac49"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0194f813efdf4b8865ad5f5c5f50f8566df7d770a82c51ef593d09e0b347020"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a7954a401d0a8a0b8bc669199db78af435aae1e3569187c2939c477c53cb6a0a"}, + {file = "jiter-0.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4feafe787eb8a8d98168ab15637ca2577f6ddf77ac6c8c66242c2d028aa5420e"}, + {file = "jiter-0.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:27cd1f2e8bb377f31d3190b34e4328d280325ad7ef55c6ac9abde72f79e84d2e"}, + {file = "jiter-0.9.0-cp39-cp39-win32.whl", hash = "sha256:161d461dcbe658cf0bd0aa375b30a968b087cdddc624fc585f3867c63c6eca95"}, + {file = "jiter-0.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:e8b36d8a16a61993be33e75126ad3d8aa29cf450b09576f3c427d27647fcb4aa"}, + {file = "jiter-0.9.0.tar.gz", hash = "sha256:aadba0964deb424daa24492abc3d229c60c4a31bfee205aedbf1acc7639d7893"}, +] + +[[package]] +name = "openai" +version = "1.66.3" +description = "The official Python library for the openai API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "openai-1.66.3-py3-none-any.whl", hash = "sha256:a427c920f727711877ab17c11b95f1230b27767ba7a01e5b66102945141ceca9"}, + {file = "openai-1.66.3.tar.gz", hash = "sha256:8dde3aebe2d081258d4159c4cb27bdc13b5bb3f7ea2201d9bd940b9a89faf0c9"}, +] + +[package.dependencies] +anyio = ">=3.5.0,<5" +distro = ">=1.7.0,<2" +httpx = ">=0.23.0,<1" +jiter = ">=0.4.0,<1" +pydantic = ">=1.9.0,<3" +sniffio = "*" +tqdm = ">4" +typing-extensions = ">=4.11,<5" + +[package.extras] +datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +realtime = ["websockets (>=13,<15)"] + [[package]] name = "opentelemetry-api" version = "1.28.2" @@ -839,6 +1005,27 @@ anyio = ">=3.6.2,<5" [package.extras] full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] +[[package]] +name = "tqdm" +version = "4.67.1" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, + {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"] +discord = ["requests"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "typing-extensions" version = "4.12.2" @@ -996,4 +1183,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "4f3c897c33f2fe2c1631228aa41d7cfb119779df02089ec40ee6e9c7cbe03172" +content-hash = "bdbc16ae3e5c6cf7910f7471bec4dd47a64d2876bc71ea2580ca16f82c42fe37" diff --git a/demos/use_cases/orchestrating_agents/pyproject.toml b/demos/use_cases/orchestrating_agents/pyproject.toml index af3b5e14..642ee236 100644 --- a/demos/use_cases/orchestrating_agents/pyproject.toml +++ b/demos/use_cases/orchestrating_agents/pyproject.toml @@ -14,6 +14,7 @@ uvicorn = "^0.32.0" opentelemetry-api = "^1.28.0" opentelemetry-sdk = "^1.28.0" opentelemetry-exporter-otlp = "^1.28.0" +openai = "^1.66.3" [build-system] requires = ["poetry-core"]