mirror of
https://github.com/katanemo/plano.git
synced 2026-05-07 23:02:43 +02:00
79b1c5415f
* add toxic/jailbreak model * fix path loading model * fix syntax * fix bug,lint, format * fix bug * formatting * add parallel + chunking * fix bug * working version * fix onnnx name erorr * device * fix jailbreak config * fix syntax error * format * add requirement + cli download for dockerfile * add task * add skeleton change for envoy filter for prompt guard * fix hardware config * fix bug * add config changes * add gitignore * merge main * integrate arch-guard with filter * add hardware config * nothing * add hardware config feature * fix requirement * fix chat ui * fix onnx * fix lint * remove non intel cpu * remove onnx * working version * modify docker * fix guard time * add nvidia support * remove nvidia * add gpu * add gpu * add gpu support * add gpu support for compose * add gpu support for compose * add gpu support for compose * add gpu support for compose * add gpu support for compose * fix docker file * fix int test * correct gpu docker * upgrad python 10 * fix logits to be gpu compatible * default to cpu dockerfile * resolve comments * fix lint + unused parameters * fix * remove eetq install for cpu * remove deploy gpu --------- Co-authored-by: Adil Hafeez <adil@katanemo.com>
93 lines
2.3 KiB
YAML
93 lines
2.3 KiB
YAML
|
|
services:
|
|
|
|
config_generator:
|
|
build:
|
|
context: ../../
|
|
dockerfile: config_generator/Dockerfile
|
|
volumes:
|
|
- ../../envoyfilter/envoy.template.yaml:/usr/src/app/envoy.template.yaml
|
|
- ./bolt_config.yaml:/usr/src/app/bolt_config.yaml
|
|
- ./generated:/usr/src/app/out
|
|
|
|
bolt:
|
|
build:
|
|
context: ../../
|
|
dockerfile: envoyfilter/Dockerfile
|
|
hostname: bolt
|
|
ports:
|
|
- "10000:10000"
|
|
- "19901:9901"
|
|
volumes:
|
|
- ./generated/envoy.yaml:/etc/envoy/envoy.yaml
|
|
- /etc/ssl/cert.pem:/etc/ssl/cert.pem
|
|
depends_on:
|
|
config_generator:
|
|
condition: service_completed_successfully
|
|
model_server:
|
|
condition: service_healthy
|
|
environment:
|
|
- LOG_LEVEL=debug
|
|
|
|
model_server:
|
|
build:
|
|
context: ../../model_server
|
|
dockerfile: Dockerfile
|
|
ports:
|
|
- "18081:80"
|
|
healthcheck:
|
|
test: ["CMD", "curl" ,"http://localhost:80/healthz"]
|
|
interval: 5s
|
|
retries: 20
|
|
volumes:
|
|
- ~/.cache/huggingface:/root/.cache/huggingface
|
|
- ./bolt_config.yaml:/root/bolt_config.yaml
|
|
# Uncomment following lines to enable GPU support
|
|
# deploy:
|
|
# resources:
|
|
# reservations:
|
|
# devices:
|
|
# - capabilities: [gpu]
|
|
# runtime: nvidia # Enables GPU support
|
|
# environment:
|
|
# - NVIDIA_VISIBLE_DEVICES=all # Use all available GPUs
|
|
|
|
|
|
function_resolver:
|
|
build:
|
|
context: ../../function_resolver
|
|
dockerfile: Dockerfile
|
|
ports:
|
|
- "18082:80"
|
|
healthcheck:
|
|
test: ["CMD", "curl" ,"http://localhost:80/healthz"]
|
|
interval: 5s
|
|
retries: 20
|
|
volumes:
|
|
- ~/.cache/huggingface:/root/.cache/huggingface
|
|
environment:
|
|
# use ollama endpoint that is hosted by host machine (no virtualization)
|
|
- OLLAMA_ENDPOINT=host.docker.internal
|
|
# uncomment following line to use ollama endpoint that is hosted by docker
|
|
# - OLLAMA_ENDPOINT=ollama
|
|
|
|
ollama:
|
|
image: ollama/ollama
|
|
container_name: ollama
|
|
volumes:
|
|
- ./ollama:/root/.ollama
|
|
restart: unless-stopped
|
|
ports:
|
|
- '11434:11434'
|
|
profiles:
|
|
- manual
|
|
|
|
chatbot_ui:
|
|
build:
|
|
context: ../../chatbot_ui
|
|
dockerfile: Dockerfile
|
|
ports:
|
|
- "18080:8080"
|
|
environment:
|
|
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
|
- CHAT_COMPLETION_ENDPOINT=http://bolt:10000/v1
|