From 38f7691163253c90befa441f84ba7ac0e6d4c10e Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Fri, 24 Jan 2025 17:14:24 -0800 Subject: [PATCH] add support for custom llm with ssl support (#380) * add support for custom llm with ssl support Add support for using custom llm that are served through https protocol. * add instructions on how to add custom inference endpoint * fix formatting * add more details * Apply suggestions from code review Co-authored-by: Salman Paracha * Apply suggestions from code review * fix precommit --------- Co-authored-by: Salman Paracha --- arch/arch_config_schema.yaml | 5 ++++ arch/envoy.template.yaml | 12 +++++++++ .../currency_exchange_ollama/arch_config.yaml | 1 + docs/source/concepts/llm_provider.rst | 25 +++++++++++++++++++ 4 files changed, 43 insertions(+) diff --git a/arch/arch_config_schema.yaml b/arch/arch_config_schema.yaml index e8c7e348..b532117c 100644 --- a/arch/arch_config_schema.yaml +++ b/arch/arch_config_schema.yaml @@ -61,6 +61,11 @@ properties: type: boolean endpoint: type: string + protocol: + type: string + enum: + - http + - https additionalProperties: false required: - name diff --git a/arch/envoy.template.yaml b/arch/envoy.template.yaml index 17147cc7..588c6f66 100644 --- a/arch/envoy.template.yaml +++ b/arch/envoy.template.yaml @@ -567,6 +567,18 @@ static_resources: address: {{ local_llm_provider.endpoint }} port_value: {{ local_llm_provider.port }} hostname: {{ local_llm_provider.endpoint }} + {% if local_llm_provider.protocol == "https" %} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + sni: {{ local_llm_provider.endpoint }} + common_tls_context: + tls_params: + tls_minimum_protocol_version: TLSv1_2 + tls_maximum_protocol_version: TLSv1_3 + {% endif %} + {% endfor %} - name: arch_internal connect_timeout: 5s diff --git a/demos/currency_exchange_ollama/arch_config.yaml b/demos/currency_exchange_ollama/arch_config.yaml index 5936ff17..5cb77750 100644 --- a/demos/currency_exchange_ollama/arch_config.yaml +++ b/demos/currency_exchange_ollama/arch_config.yaml @@ -7,6 +7,7 @@ listener: connect_timeout: 0.005s llm_providers: + - name: local-llama provider_interface: openai model: llama3.2 diff --git a/docs/source/concepts/llm_provider.rst b/docs/source/concepts/llm_provider.rst index 498ee875..d1731e64 100644 --- a/docs/source/concepts/llm_provider.rst +++ b/docs/source/concepts/llm_provider.rst @@ -32,6 +32,31 @@ calls, handling retries, managing rate limits, and ensuring seamless integration LLMs. Simply configure the details of the LLMs your application will use, and Arch offers a unified interface to make outbound LLM calls. +Adding custom LLM Provider +-------------------------- + +We support any OpenAI compliant LLM for example mistral, openai, ollama etc. We offer first class support for openai and ollama. You can easily configure an LLM that communicates over the OpenAI API interface, by following the below guide. + +For example following code block shows you how to add an ollama-supported LLM in the `arch_config.yaml` file. +.. code-block:: yaml + + - name: local-llama + provider_interface: openai + model: llama3.2 + endpoint: host.docker.internal:11434 + + +For example following code block shows you how to add mistral llm provider in the `arch_config.yaml` file. + +.. code-block:: yaml + + - name: mistral-ai + provider_interface: openai + model: ministral-3b-latest + endpoint: api.mistral.ai:443 + protocol: https + + Example: Using the OpenAI Python SDK ------------------------------------