From 25692bbbfc39e806011ff97e0b9d4a64bc858490 Mon Sep 17 00:00:00 2001 From: obinnascale3 <109410793+obinnascale3@users.noreply.github.com> Date: Fri, 31 Jan 2025 20:16:30 +0100 Subject: [PATCH 1/2] Add Langtrace as a supported observability tool (#376) * add langtrace as a tracing tool * add setup step for Arch installation --------- Co-authored-by: Obinna Okafor --- docs/source/guides/observability/tracing.rst | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/source/guides/observability/tracing.rst b/docs/source/guides/observability/tracing.rst index 23bf80d2..8dda14fd 100644 --- a/docs/source/guides/observability/tracing.rst +++ b/docs/source/guides/observability/tracing.rst @@ -289,6 +289,53 @@ To send tracing data to `Datadog `_: + +1. **Configure Arch**: Make sure Arch is installed and setup correctly. For more information, refer to the `installation guide `_. + +2. **Install Langtrace**: Install the Langtrace SDK.: + + .. code-block:: console + + $ pip install langtrace-python-sdk + +3. **Set Environment Variables**: Provide your Langtrace API key. + + .. code-block:: console + + $ export LANGTRACE_API_KEY= + +4. **Trace Requests**: Once you have Langtrace set up, you can start tracing requests. + + Here's an example of how to trace a request using the Langtrace Python SDK: + + .. code-block:: python + + import os + from langtrace_python_sdk import langtrace # Must precede any llm module imports + from openai import OpenAI + + langtrace.init(api_key=os.environ['LANGTRACE_API_KEY']) + + client = OpenAI(api_key=os.environ['OPENAI_API_KEY'], base_url="http://localhost:12000/v1") + + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "Hello"}, + ] + ) + + print(chat_completion.choices[0].message.content) + +5. **Verify Traces**: Access the Langtrace dashboard to view your traces. + Best Practices -------------- @@ -312,6 +359,7 @@ Additional Resources - `W3C Trace Context Specification `_ - `AWS X-Ray Exporter `_ - `Datadog Exporter `_ +- `Langtrace Documentation `_ .. Note:: Replace placeholders such as ```` and ```` with your actual configurations. From 962727f244c638fcb6d84794943b59e16414df16 Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Mon, 3 Feb 2025 14:51:59 -0800 Subject: [PATCH 2/2] Infer port from protocol if port is not specified and add ability to override hostname in clusters def (#389) --- arch/arch_config_schema.yaml | 4 ++++ arch/envoy.template.yaml | 8 +++++++ arch/tools/cli/config_generator.py | 29 +++++++++++++++++++----- demos/currency_exchange/arch_config.yaml | 2 +- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/arch/arch_config_schema.yaml b/arch/arch_config_schema.yaml index b532117c..96e14856 100644 --- a/arch/arch_config_schema.yaml +++ b/arch/arch_config_schema.yaml @@ -33,6 +33,8 @@ properties: enum: - http - https + http_host: + type: string additionalProperties: false required: - endpoint @@ -66,6 +68,8 @@ properties: enum: - http - https + http_host: + type: string additionalProperties: false required: - name diff --git a/arch/envoy.template.yaml b/arch/envoy.template.yaml index 588c6f66..5dfa60b3 100644 --- a/arch/envoy.template.yaml +++ b/arch/envoy.template.yaml @@ -537,7 +537,11 @@ static_resources: socket_address: address: {{ cluster.endpoint }} port_value: {{ cluster.port }} + {% if cluster.http_host %} + hostname: {{ cluster.http_host }} + {% else %} hostname: {{ cluster.endpoint }} + {% endif %} {% if cluster.protocol == "https" %} transport_socket: name: envoy.transport_sockets.tls @@ -566,7 +570,11 @@ static_resources: socket_address: address: {{ local_llm_provider.endpoint }} port_value: {{ local_llm_provider.port }} + {% if local_llm_provider.http_host %} + hostname: {{ local_llm_provider.http_host }} + {% else %} hostname: {{ local_llm_provider.endpoint }} + {% endif %} {% if local_llm_provider.protocol == "https" %} transport_socket: name: envoy.transport_sockets.tls diff --git a/arch/tools/cli/config_generator.py b/arch/tools/cli/config_generator.py index e535894b..447585fb 100644 --- a/arch/tools/cli/config_generator.py +++ b/arch/tools/cli/config_generator.py @@ -16,6 +16,20 @@ ARCH_CONFIG_SCHEMA_FILE = os.getenv( ) +def get_endpoint_and_port(endpoint, protocol): + endpoint_tokens = endpoint.split(":") + if len(endpoint_tokens) > 1: + endpoint = endpoint_tokens[0] + port = int(endpoint_tokens[1]) + return endpoint, port + else: + if protocol == "http": + port = 80 + else: + port = 443 + return endpoint, port + + def validate_and_render_schema(): env = Environment(loader=FileSystemLoader("./")) template = env.get_template("envoy.template.yaml") @@ -42,9 +56,11 @@ def validate_and_render_schema(): for name, endpoint_details in endpoints.items(): inferred_clusters[name] = endpoint_details endpoint = inferred_clusters[name]["endpoint"] - if len(endpoint.split(":")) > 1: - inferred_clusters[name]["endpoint"] = endpoint.split(":")[0] - inferred_clusters[name]["port"] = int(endpoint.split(":")[1]) + protocol = inferred_clusters[name].get("protocol", "http") + ( + inferred_clusters[name]["endpoint"], + inferred_clusters[name]["port"], + ) = get_endpoint_and_port(endpoint, protocol) print("defined clusters from arch_config.yaml: ", json.dumps(inferred_clusters)) @@ -77,9 +93,10 @@ def validate_and_render_schema(): if llm_provider.get("endpoint", None): endpoint = llm_provider["endpoint"] - if len(endpoint.split(":")) > 1: - llm_provider["endpoint"] = endpoint.split(":")[0] - llm_provider["port"] = int(endpoint.split(":")[1]) + protocol = llm_provider.get("protocol", "http") + llm_provider["endpoint"], llm_provider["port"] = get_endpoint_and_port( + endpoint, protocol + ) llms_with_endpoint.append(llm_provider) config_yaml["llm_providers"] = updated_llm_providers diff --git a/demos/currency_exchange/arch_config.yaml b/demos/currency_exchange/arch_config.yaml index f8776c48..f51b5904 100644 --- a/demos/currency_exchange/arch_config.yaml +++ b/demos/currency_exchange/arch_config.yaml @@ -44,7 +44,7 @@ prompt_targets: endpoints: frankfurther_api: - endpoint: api.frankfurter.dev:443 + endpoint: api.frankfurter.dev protocol: https tracing: