diff --git a/.github/workflows/e2e_test_demos.yml b/.github/workflows/e2e_test_demos.yml index be74ffd5..f7a434d1 100644 --- a/.github/workflows/e2e_test_demos.yml +++ b/.github/workflows/e2e_test_demos.yml @@ -22,7 +22,6 @@ jobs: - name: build arch docker image run: | docker build -f arch/Dockerfile . -t katanemo/archgw -t katanemo/archgw:0.2.8 - docker build -f arch/Dockerfile.brightstaff . -t katanemo/archgw:brightstaff -t katanemo/archgw:brightstaff_0.2.8 - name: install poetry run: | diff --git a/arch/Dockerfile b/arch/Dockerfile index 98a113ad..7eab78b2 100644 --- a/arch/Dockerfile +++ b/arch/Dockerfile @@ -4,8 +4,8 @@ RUN rustup -v target add wasm32-wasip1 WORKDIR /arch COPY crates . -RUN cd prompt_gateway && cargo build --release --target wasm32-wasip1 -RUN cd llm_gateway && cargo build --release --target wasm32-wasip1 +RUN cargo build --release --target wasm32-wasip1 -p prompt_gateway -p llm_gateway +RUN cargo build --release -p brightstaff # copy built filter into envoy image FROM docker.io/envoyproxy/envoy:v1.32-latest as envoy @@ -13,20 +13,27 @@ FROM docker.io/envoyproxy/envoy:v1.32-latest as envoy #Build config generator, so that we have a single build image for both Rust and Python FROM python:3.12-slim as arch -RUN apt-get update && apt-get install -y gettext-base curl && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y supervisor gettext-base curl && apt-get clean && rm -rf /var/lib/apt/lists/* COPY --from=builder /arch/target/wasm32-wasip1/release/prompt_gateway.wasm /etc/envoy/proxy-wasm-plugins/prompt_gateway.wasm COPY --from=builder /arch/target/wasm32-wasip1/release/llm_gateway.wasm /etc/envoy/proxy-wasm-plugins/llm_gateway.wasm +COPY --from=builder /arch/target/release/brightstaff /app/brightstaff COPY --from=envoy /usr/local/bin/envoy /usr/local/bin/envoy + WORKDIR /app COPY arch/requirements.txt . RUN pip install -r requirements.txt COPY arch/tools/cli/config_generator.py . COPY arch/envoy.template.yaml . COPY arch/arch_config_schema.yaml . +COPY arch/supervisord.conf /etc/supervisor/conf.d/supervisord.conf RUN pip install requests RUN touch /var/log/envoy.log +RUN mkdir -p /var/log/supervisor/ +RUN touch /var/log/supervisor/supervisord.log + +ENTRYPOINT ["sh","-c", "/usr/bin/supervisord"] # ENTRYPOINT ["sh","-c", "python config_generator.py && envsubst < /etc/envoy/envoy.yaml > /etc/envoy.env_sub.yaml && envoy -c /etc/envoy.env_sub.yaml --log-level trace 2>&1 | tee /var/log/envoy.log"] -ENTRYPOINT ["sh","-c", "python config_generator.py && envsubst < /etc/envoy/envoy.yaml > /etc/envoy.env_sub.yaml && envoy -c /etc/envoy.env_sub.yaml --component-log-level wasm:debug 2>&1 | tee /var/log/envoy.log"] +# ENTRYPOINT ["sh","-c", "python config_generator.py && envsubst < /etc/envoy/envoy.yaml > /etc/envoy.env_sub.yaml && envoy -c /etc/envoy.env_sub.yaml --component-log-level wasm:info 2>&1 | tee /var/log/envoy.log"] diff --git a/arch/envoy.template.yaml b/arch/envoy.template.yaml index 42f9b17c..f9dd6d4a 100644 --- a/arch/envoy.template.yaml +++ b/arch/envoy.template.yaml @@ -677,9 +677,9 @@ static_resources: - endpoint: address: socket_address: - address: host.docker.internal + address: 0.0.0.0 port_value: 9091 - hostname: bright_staff + hostname: localhost - name: router_model_host connect_timeout: 0.5s diff --git a/arch/supervisord.conf b/arch/supervisord.conf new file mode 100644 index 00000000..b20a510a --- /dev/null +++ b/arch/supervisord.conf @@ -0,0 +1,16 @@ +[supervisord] +nodaemon=true + +[program:brightstaff] +command=sh -c "/app/brightstaff 2>&1 | tee /var/log/brightstaff.log" +stdout_logfile=/dev/stdout +redirect_stderr=true +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 + +[program:envoy] +command=/bin/sh -c "python /app/config_generator.py && envsubst < /etc/envoy/envoy.yaml > /etc/envoy.env_sub.yaml && envoy -c /etc/envoy.env_sub.yaml 2>&1 | tee /var/log//envoy.log" +stdout_logfile=/dev/stdout +redirect_stderr=true +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 diff --git a/arch/tools/cli/consts.py b/arch/tools/cli/consts.py index 47d638c2..23694fe4 100644 --- a/arch/tools/cli/consts.py +++ b/arch/tools/cli/consts.py @@ -6,14 +6,9 @@ KATANEMO_LOCAL_MODEL_LIST = [ "katanemo/Arch-Guard", ] SERVICE_NAME_ARCHGW = "archgw" -SERVICE_NAME_BRIGHTSTAFF = "brightstaff" SERVICE_NAME_MODEL_SERVER = "model_server" SERVICE_ALL = "all" MODEL_SERVER_LOG_FILE = "~/archgw_logs/modelserver.log" ACCESS_LOG_FILES = "~/archgw_logs/access*" ARCHGW_DOCKER_NAME = "archgw" -BRIGHTSTAFF_DOCKER_NAME = "brightstaff" ARCHGW_DOCKER_IMAGE = os.getenv("ARCHGW_DOCKER_IMAGE", "katanemo/archgw:0.2.8") -BRIGHTSTAFF_DOCKER_IMAGE = os.getenv( - "BRIGHTSTAFF_DOCKER_IMAGE", "katanemo/archgw:brightstaff_0.2.8" -) diff --git a/arch/tools/cli/core.py b/arch/tools/cli/core.py index 84c160c1..47590f2f 100644 --- a/arch/tools/cli/core.py +++ b/arch/tools/cli/core.py @@ -8,7 +8,6 @@ from cli.utils import getLogger from cli.consts import ( ARCHGW_DOCKER_IMAGE, ARCHGW_DOCKER_NAME, - BRIGHTSTAFF_DOCKER_NAME, KATANEMO_LOCAL_MODEL_LIST, ) from huggingface_hub import snapshot_download @@ -17,7 +16,6 @@ from cli.docker_cli import ( docker_container_status, docker_remove_container, docker_start_archgw_detached, - docker_start_brightstaff_detached, docker_stop_container, health_check_endpoint, stream_gateway_logs, @@ -122,42 +120,6 @@ def start_arch(arch_config_file, env, log_timeout=120, foreground=False): stop_docker_container() -def start_brightstaff(arch_config_file, env, log_timeout=120, foreground=False): - """ - Start Docker Compose in detached mode and stream logs until services are healthy. - - Args: - path (str): The path where the prompt_config.yml file is located. - log_timeout (int): Time in seconds to show logs before checking for healthy state. - """ - log.info("Starting brightstaff") - - try: - brightstaff_container_status = docker_container_status(BRIGHTSTAFF_DOCKER_NAME) - if brightstaff_container_status != "not found": - log.info( - f"brightstaff found in docker, stopping and removing it: status: {brightstaff_container_status}" - ) - docker_stop_container(BRIGHTSTAFF_DOCKER_NAME) - docker_remove_container(BRIGHTSTAFF_DOCKER_NAME) - - return_code, _, brightstaff_stderr = docker_start_brightstaff_detached( - arch_config_file, - env, - ) - if return_code != 0: - log.info("Failed to start brightstaff: " + str(return_code)) - log.info("stderr: " + brightstaff_stderr) - sys.exit(1) - - if foreground: - stream_gateway_logs(follow=True, service="brightstaff") - - except KeyboardInterrupt: - log.info("Keyboard interrupt received, stopping arch gateway service.") - stop_docker_container(service=BRIGHTSTAFF_DOCKER_NAME) - - def stop_docker_container(service=ARCHGW_DOCKER_NAME): """ Shutdown all Docker Compose services by running `docker-compose down`. diff --git a/arch/tools/cli/docker_cli.py b/arch/tools/cli/docker_cli.py index d6b34af5..0dd7ecc7 100644 --- a/arch/tools/cli/docker_cli.py +++ b/arch/tools/cli/docker_cli.py @@ -6,8 +6,6 @@ import requests from cli.consts import ( ARCHGW_DOCKER_IMAGE, ARCHGW_DOCKER_NAME, - BRIGHTSTAFF_DOCKER_IMAGE, - BRIGHTSTAFF_DOCKER_NAME, ) from cli.utils import getLogger @@ -60,7 +58,8 @@ def docker_start_archgw_detached( port_mappings_args = [item for port in port_mappings for item in ("-p", port)] volume_mappings = [ - f"{logs_path_abs}:/var/log:rw", + # TODO: fix log path + # f"{logs_path_abs}:/var/log:rw", f"{arch_config_file}:/app/arch_config.yaml:ro", # "/Users/adilhafeez/src/intelligent-prompt-gateway/crates/target/wasm32-wasip1/release:/etc/envoy/proxy-wasm-plugins:ro", ] @@ -86,48 +85,6 @@ def docker_start_archgw_detached( return result.returncode, result.stdout, result.stderr -def docker_start_brightstaff_detached( - arch_config_file: str, - env: dict, -) -> str: - env_args = [item for key, value in env.items() for item in ["-e", f"{key}={value}"]] - - port_mappings = [ - "9091:9091", - ] - port_mappings_args = [item for port in port_mappings for item in ("-p", port)] - - volume_mappings = [ - f"{arch_config_file}:/app/arch_config.yaml:ro", - ] - volume_mappings_args = [ - item for volume in volume_mappings for item in ("-v", volume) - ] - - llm_provider_endpoint = env.get( - "LLM_PROVIDER_ENDPOINT", "http://host.docker.internal:12000/v1/chat/completions" - ) - - options = [ - "docker", - "run", - "-d", - "--name", - BRIGHTSTAFF_DOCKER_NAME, - *port_mappings_args, - *volume_mappings_args, - *env_args, - "-e", - f"LLM_PROVIDER_ENDPOINT={llm_provider_endpoint}", - "--add-host", - "host.docker.internal:host-gateway", - BRIGHTSTAFF_DOCKER_IMAGE, - ] - - result = subprocess.run(options, capture_output=True, text=True, check=False) - return result.returncode, result.stdout, result.stderr - - def health_check_endpoint(endpoint: str) -> bool: try: response = requests.get(endpoint) diff --git a/arch/tools/cli/main.py b/arch/tools/cli/main.py index 2ac5cd20..001f3d9c 100644 --- a/arch/tools/cli/main.py +++ b/arch/tools/cli/main.py @@ -14,7 +14,6 @@ from cli.utils import ( ) from cli.core import ( start_arch_modelserver, - start_brightstaff, stop_arch_modelserver, start_arch, stop_docker_container, @@ -22,10 +21,8 @@ from cli.core import ( ) from cli.consts import ( ARCHGW_DOCKER_IMAGE, - BRIGHTSTAFF_DOCKER_IMAGE, KATANEMO_DOCKERHUB_REPO, SERVICE_NAME_ARCHGW, - SERVICE_NAME_BRIGHTSTAFF, SERVICE_NAME_MODEL_SERVER, SERVICE_ALL, ) @@ -43,7 +40,6 @@ logo = r""" # Command to build archgw and model_server Docker images ARCHGW_DOCKERFILE = "./arch/Dockerfile" -BRIGHTSTAFF_DOCKERFILE = "./arch/Dockerfile.brightstaff" MODEL_SERVER_BUILD_FILE = "./model_server/pyproject.toml" @@ -60,7 +56,6 @@ def verify_service_name(service): if service not in [ SERVICE_NAME_ARCHGW, SERVICE_NAME_MODEL_SERVER, - SERVICE_NAME_BRIGHTSTAFF, SERVICE_ALL, ]: print(f"Error: Invalid service {service}. Exiting") @@ -124,35 +119,6 @@ def build(service): click.echo("archgw image built successfully.") - if service == SERVICE_NAME_BRIGHTSTAFF or service == SERVICE_ALL: - if os.path.exists(BRIGHTSTAFF_DOCKERFILE): - click.echo("Building brightstaff image...") - try: - subprocess.run( - [ - "docker", - "build", - "-f", - BRIGHTSTAFF_DOCKERFILE, - "-t", - f"{KATANEMO_DOCKERHUB_REPO}:brightstaff_latest", - "-t", - f"{BRIGHTSTAFF_DOCKER_IMAGE}", - ".", - "--add-host=host.docker.internal:host-gateway", - ], - check=True, - ) - click.echo("brightstaff image built successfully.") - except subprocess.CalledProcessError as e: - click.echo(f"Error building brightstaff image: {e}") - sys.exit(1) - else: - click.echo("Error: Dockerfile not found in /arch") - sys.exit(1) - - click.echo("brightstaff image built successfully.") - """Install the model server dependencies using Poetry.""" if service == SERVICE_NAME_MODEL_SERVER or service == SERVICE_ALL: # Check if pyproject.toml exists @@ -199,7 +165,7 @@ def up(file, path, service, foreground): sys.exit(1) if service == SERVICE_NAME_MODEL_SERVER: - log.info("Download archgw models from HuggingFace...") + log.info("Download models from HuggingFace...") download_models_from_hf() start_arch_modelserver(foreground) return @@ -229,8 +195,6 @@ def up(file, path, service, foreground): log.info(f"Validation stderr: {validation_stderr}") sys.exit(1) - log.info("Starting arch model server and arch gateway") - # Set the ARCH_CONFIG_FILE environment variable env_stage = { "OTEL_TRACING_HTTP_ENDPOINT": "http://host.docker.internal:4318/v1/traces", @@ -253,7 +217,6 @@ def up(file, path, service, foreground): else: app_env_file = os.path.abspath(os.path.join(path, ".env")) - print(f"app_env_file: {app_env_file}") if not os.path.exists( app_env_file ): # check to see if the environment variables in the current environment or not @@ -276,13 +239,10 @@ def up(file, path, service, foreground): if service == SERVICE_NAME_ARCHGW: start_arch(arch_config_file, env, foreground=foreground) - if service == SERVICE_NAME_BRIGHTSTAFF: - start_brightstaff(arch_config_file, env, foreground=foreground) else: download_models_from_hf() start_arch_modelserver(foreground) start_arch(arch_config_file, env, foreground=foreground) - start_brightstaff(arch_config_file, env, foreground=foreground) @click.command() @@ -300,12 +260,9 @@ def down(service): stop_arch_modelserver() elif service == SERVICE_NAME_ARCHGW: stop_docker_container() - elif service == SERVICE_NAME_BRIGHTSTAFF: - stop_docker_container(SERVICE_NAME_BRIGHTSTAFF) else: stop_arch_modelserver() stop_docker_container(SERVICE_NAME_ARCHGW) - stop_docker_container(SERVICE_NAME_BRIGHTSTAFF) @click.command() diff --git a/build_filter_image.sh b/build_filter_image.sh index 8e00331a..04b0789f 100644 --- a/build_filter_image.sh +++ b/build_filter_image.sh @@ -1,2 +1 @@ docker build -f arch/Dockerfile . -t katanemo/archgw -t katanemo/archgw:0.2.8 -docker build -f arch/Dockerfile.brightstaff . -t katanemo/archgw:brightstaff -t katanemo/archgw:brightstaff_0.2.8