diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..9e6adb2 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,102 @@ +# ============================================================================= +# Stage 1: venv-builder +# Minimal image whose only job is to populate the venv. Uses the same Python +# source as the runtime stage (deadsnakes) so the symlinks inside the venv +# (e.g. venv/bin/python -> /usr/bin/python3.13) stay valid after COPY --from. +# Everything in this stage except the venv itself is discarded. +# ============================================================================= +FROM ubuntu:24.04 AS venv-builder + +RUN apt-get update \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + ca-certificates \ + git \ + libpq-dev \ + pkg-config \ + software-properties-common \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get install -y --no-install-recommends \ + python3.13 \ + python3.13-venv \ + python3.13-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ + +# Build the venv at the path it will live at in the final image, so shebangs +# and console-scripts inside the venv reference the correct runtime location +# once the seed step rsyncs them into the named volume. +ENV VIRTUAL_ENV=/workspaces/dograh/venv \ + PATH=/workspaces/dograh/venv/bin:$PATH +RUN mkdir -p /workspaces/dograh && python3.13 -m venv "$VIRTUAL_ENV" + +# Layer 1: API deps. Cache invalidates only when these two files change. +RUN --mount=type=bind,source=api/requirements.txt,target=/tmp/req.txt \ + --mount=type=bind,source=api/requirements.dev.txt,target=/tmp/req.dev.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip install -r /tmp/req.txt -r /tmp/req.dev.txt + +# Layer 2: pipecat deps. Cache invalidates when pipecat source changes. +# After installing pipecat, two hardening tweaks (mirrored from api/Dockerfile): +# 1. Swap opencv-python (pulled by pipecat[webrtc]) for opencv-python-headless. +# The non-headless build links against X11/Qt (libxcb*); without those +# shared libs in the image, `import cv2` fails at runtime. +# 2. Pre-download NLTK's punkt_tab tokenizer so pipecat's text processing +# doesn't hit the network on first agent run. NLTK auto-finds it under +# sys.prefix/nltk_data, so it travels with the venv on COPY/rsync. +RUN --mount=type=bind,source=pipecat,target=/tmp/pipecat,rw \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip install '/tmp/pipecat[cartesia,deepgram,openai,elevenlabs,groq,google,azure,sarvam,soundfile,silero,webrtc,speechmatics,openrouter,camb,mcp]' \ + && uv pip install --group /tmp/pipecat/pyproject.toml:dev \ + && uv pip uninstall opencv-python \ + && uv pip install opencv-python-headless \ + && python -c "import nltk; nltk.download('punkt_tab', download_dir='/workspaces/dograh/venv/nltk_data', quiet=True)" + + +# ============================================================================= +# Stage 2: runtime devcontainer image +# Inherits the devcontainer base (vscode user, sudo, etc.) and brings only the +# populated venv across from the builder stage. +# ============================================================================= +FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04 + +RUN apt-get update \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + ffmpeg \ + git \ + jq \ + libpq-dev \ + pkg-config \ + postgresql-client \ + procps \ + redis-tools \ + rsync \ + software-properties-common \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get install -y --no-install-recommends \ + python3.13 \ + python3.13-venv \ + python3.13-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# uv is still needed at runtime so post-create.sh can do the editable +# pipecat install (and any ad-hoc `uv pip install` users might run). +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ + +# Bring the populated venv across. At runtime, the named volume in +# docker-compose.yml shadows /workspaces/dograh/venv; post-create.sh +# rsyncs from /opt/venv-template into the (initially empty) volume, +# comparing build-stamps so an image rebuild that changed deps re-seeds. +COPY --from=venv-builder --chown=vscode:vscode /workspaces/dograh/venv /opt/venv-template +RUN date -u +%s > /opt/venv-template/.build-stamp \ + && chown vscode:vscode /opt/venv-template/.build-stamp + +ENV VIRTUAL_ENV=/workspaces/dograh/venv \ + PATH=/workspaces/dograh/venv/bin:$PATH diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json new file mode 100644 index 0000000..96d8688 --- /dev/null +++ b/.devcontainer/devcontainer-lock.json @@ -0,0 +1,9 @@ +{ + "features": { + "ghcr.io/devcontainers/features/node:1": { + "version": "1.7.1", + "resolved": "ghcr.io/devcontainers/features/node@sha256:8c0de46939b61958041700ee89e3493f3b2e4131a06dc46b4d9423427d06e5f6", + "integrity": "sha256:8c0de46939b61958041700ee89e3493f3b2e4131a06dc46b4d9423427d06e5f6" + } + } +} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..f9edc8e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,70 @@ +{ + "name": "Dograh", + "dockerComposeFile": [ + "../docker-compose-local.yaml", + "docker-compose.yml" + ], + "service": "workspace", + "runServices": [ + "workspace", + "postgres", + "redis", + "minio" + ], + "workspaceFolder": "/workspaces/dograh", + "shutdownAction": "stopCompose", + "overrideCommand": false, + "remoteUser": "vscode", + "features": { + "ghcr.io/devcontainers/features/node:1": { + "version": "24" + } + }, + "initializeCommand": "git submodule update --init --recursive", + "postCreateCommand": "bash .devcontainer/scripts/post-create.sh", + "postStartCommand": "bash .devcontainer/scripts/post-start.sh", + "forwardPorts": [ + 5432, + 6379, + 9000, + 9001 + ], + "portsAttributes": { + "3000": { + "label": "Dograh UI", + "onAutoForward": "ignore" + }, + "8000": { + "label": "Dograh API", + "onAutoForward": "ignore" + }, + "5432": { + "label": "Postgres" + }, + "6379": { + "label": "Redis" + }, + "9000": { + "label": "MinIO API" + }, + "9001": { + "label": "MinIO Console" + } + }, + "customizations": { + "vscode": { + "settings": { + "python.defaultInterpreterPath": "/workspaces/dograh/venv/bin/python", + "terminal.integrated.defaultProfile.linux": "bash" + }, + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.debugpy", + "ms-azuretools.vscode-docker", + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode" + ] + } + } +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 0000000..bf4d756 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,31 @@ +services: + workspace: + build: + context: . + dockerfile: .devcontainer/Dockerfile + command: sleep infinity + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_healthy + environment: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PYTHONUNBUFFERED: "1" + extra_hosts: + - "host.docker.internal:host-gateway" + init: true + networks: + - app-network + volumes: + - .:/workspaces/dograh:cached + - dograh-venv:/workspaces/dograh/venv + - dograh-ui-node_modules:/workspaces/dograh/ui/node_modules + - dograh-ts-validator-node_modules:/workspaces/dograh/api/mcp_server/ts_validator/node_modules + +volumes: + dograh-venv: + dograh-ui-node_modules: + dograh-ts-validator-node_modules: diff --git a/.devcontainer/scripts/post-create.sh b/.devcontainer/scripts/post-create.sh new file mode 100755 index 0000000..00d02bc --- /dev/null +++ b/.devcontainer/scripts/post-create.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="/workspaces/dograh" +UI_ENV_EXAMPLE="$ROOT_DIR/ui/.env.example" +UI_ENV_FILE="$ROOT_DIR/ui/.env" +VENV_PATH="$ROOT_DIR/venv" +VENV_TEMPLATE="/opt/venv-template" + +TOTAL_STEPS=5 +STEP=0 +STEP_START=$SECONDS +SCRIPT_START=$SECONDS + +step() { + STEP=$((STEP + 1)) + STEP_START=$SECONDS + printf '\n==> [%d/%d] %s\n' "$STEP" "$TOTAL_STEPS" "$1" +} + +step_done() { + printf ' done in %ds\n' "$((SECONDS - STEP_START))" +} + +fail() { + printf '\n!! FAILED at step %d/%d (%s) after %ds\n' \ + "$STEP" "$TOTAL_STEPS" "${1:-unknown}" "$((SECONDS - SCRIPT_START))" >&2 + exit 1 +} +trap 'fail "exit $?"' ERR + +copy_if_missing() { + local src=$1 + local dst=$2 + if [[ -f "$dst" ]]; then + echo "Keeping existing $dst" + return + fi + cp "$src" "$dst" + echo "Created $dst from $src" +} + +# Copy an api/.env*.example template to its target, rewriting infra hostnames +# from `localhost` to the docker service names defined in +# docker-compose-local.yaml. MINIO_PUBLIC_ENDPOINT stays on localhost — that +# URL ends up in UI responses and is loaded by the host browser via the +# forwarded port. No-op if the target already exists. +copy_env_with_docker_hostnames() { + local src=$1 + local dst=$2 + if [[ -f "$dst" ]]; then + echo "Keeping existing $dst" + return + fi + cp "$src" "$dst" + sed -i \ + -e 's|@localhost:5432|@postgres:5432|g' \ + -e 's|@localhost:6379|@redis:6379|g' \ + -e 's|^MINIO_ENDPOINT=localhost:9000|MINIO_ENDPOINT=minio:9000|' \ + "$dst" + echo "Created $dst from $src (rewrote service hostnames for docker network)" +} + +# Seed the venv named volume from the image-baked template, but only when +# the template's build-stamp differs from what's currently in the volume +# (first start, or any rebuild that changed requirements.txt / pipecat). +seed_venv() { + local image_stamp venv_stamp + image_stamp=$(cat "$VENV_TEMPLATE/.build-stamp" 2>/dev/null || echo missing) + venv_stamp=$(cat "$VENV_PATH/.build-stamp" 2>/dev/null || echo none) + + if [[ "$image_stamp" == "$venv_stamp" ]]; then + echo "Venv already in sync with image template (stamp=$venv_stamp)" + return + fi + + echo "Re-seeding venv: image=$image_stamp, volume=$venv_stamp" + rsync -a --delete "$VENV_TEMPLATE/" "$VENV_PATH/" +} + +cd "$ROOT_DIR" + +step "Fixing ownership of named volume mountpoints" +# Named volumes are created owned by root; postCreateCommand runs as the +# remote user. Chown the mountpoint roots so the steps below can write. +sudo chown "$(id -u):$(id -g)" \ + "$VENV_PATH" \ + "$ROOT_DIR/ui/node_modules" \ + "$ROOT_DIR/api/mcp_server/ts_validator/node_modules" +step_done + +step "Seeding venv from image template" +seed_venv +step_done + +step "Copying example env files into place" +copy_env_with_docker_hostnames "$ROOT_DIR/api/.env.example" "$ROOT_DIR/api/.env" +copy_env_with_docker_hostnames "$ROOT_DIR/api/.env.test.example" "$ROOT_DIR/api/.env.test" +copy_if_missing "$UI_ENV_EXAMPLE" "$UI_ENV_FILE" +step_done + +step "Switching pipecat to editable install from workspace" +# pipecat's deps are already in the seeded venv as a frozen snapshot from +# the build context. Re-register editable from the bind-mounted workspace +# so source edits take effect. --no-deps skips re-resolving transitive +# dependencies (already present from the seeded image template). +uv pip install -e "$ROOT_DIR/pipecat" --no-deps +step_done + +step "Installing npm dependencies (ui + ts_validator in parallel)" +npm ci --prefix ui & +ui_pid=$! +npm ci --prefix api/mcp_server/ts_validator & +ts_pid=$! +wait "$ui_pid" || fail "npm ci ui" +wait "$ts_pid" || fail "npm ci ts_validator" +step_done + +# Optional personal hook: gitignored script for per-developer tools (e.g. +# claude, codex, etc.). Runs only if present; safe to omit. +LOCAL_HOOK="$ROOT_DIR/.devcontainer/install.local.sh" +if [[ -f "$LOCAL_HOOK" ]]; then + printf '\n==> Running local install hook (%s)\n' "$LOCAL_HOOK" + bash "$LOCAL_HOOK" +fi + +printf '\nDevcontainer bootstrap complete in %ds.\n' "$((SECONDS - SCRIPT_START))" diff --git a/.devcontainer/scripts/post-start.sh b/.devcontainer/scripts/post-start.sh new file mode 100755 index 0000000..90602bb --- /dev/null +++ b/.devcontainer/scripts/post-start.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Intentionally no `http://localhost:PORT` URLs below — VS Code's terminal +# URL detector adds any printed URL to its auto-forwarded-ports list and +# then polls it, which produces ECONNREFUSED log spam every ~20s for ports +# that aren't bound yet. The Ports panel auto-detects bound ports anyway. +cat <<'EOF' +Dograh devcontainer ready. + +Start the backend: + bash scripts/start_services_dev.sh + +Start the UI in another terminal: + cd ui && npm run dev -- --hostname 0.0.0.0 + +URLs and other workflow notes: docs/contribution/setup.mdx +EOF diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 21d22ff..1ef77d3 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -65,21 +65,21 @@ jobs: with: submodules: recursive - - name: Set up Python 3.12 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: "3.12" - cache: pip - cache-dependency-path: | - api/requirements.txt - api/requirements.dev.txt - pipecat/pyproject.toml + python-version: "3.13" - name: Set up Node 22 (test_ts_bridge.py shells out to node) uses: actions/setup-node@v4 with: node-version: "22" + - name: Create Python virtual environment + run: | + python -m venv .venv + echo "${{ github.workspace }}/.venv/bin" >> "$GITHUB_PATH" + - name: Install api and pipecat dependencies run: ./scripts/setup_requirements.sh --dev diff --git a/.github/workflows/pre-pr-drift-check.yml b/.github/workflows/pre-pr-drift-check.yml index d3b2ad0..e32337e 100644 --- a/.github/workflows/pre-pr-drift-check.yml +++ b/.github/workflows/pre-pr-drift-check.yml @@ -27,15 +27,10 @@ jobs: with: submodules: recursive - - name: Set up Python 3.12 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: "3.12" - cache: pip - cache-dependency-path: | - api/requirements.txt - api/requirements.dev.txt - pipecat/pyproject.toml + python-version: "3.13" - name: Set up Node 22 uses: actions/setup-node@v4 @@ -44,6 +39,11 @@ jobs: cache: npm cache-dependency-path: ui/package-lock.json + - name: Create Python virtual environment + run: | + python -m venv .venv + echo "${{ github.workspace }}/.venv/bin" >> "$GITHUB_PATH" + - name: Install api and pipecat dependencies run: ./scripts/setup_requirements.sh --dev diff --git a/.gitignore b/.gitignore index d45b824..e4ccaf3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ infrastructure/ prd/ .vercel +.devcontainer/install.local.sh venv/ .venv/ .playwright-mcp @@ -18,4 +19,3 @@ coturn/ *.wav dograh_pcm_cache/ node_modules/ -.vscode \ No newline at end of file diff --git a/.python-version b/.python-version index 7acdc73..976544c 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.13.7 \ No newline at end of file +3.13.7 diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..9826cbf --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,128 @@ +// Debug configurations for Dograh contributors. +// +// Prerequisites: +// - Python interpreter selected in VS Code (devcontainer sets this +// automatically; otherwise pick `./venv/bin/python` via the +// "Python: Select Interpreter" command). +// - api/.env exists (copy from api/.env.example +// is created automatically by the devcontainer post-create script). +// - api/.env.test exists for the test configurations (copy from +// api/.env.example and point at a throwaway database). +// +// All Python configs set justMyCode=false so the debugger steps into +// library code (useful for tracing through pipecat/fastapi/etc.). +{ + "version": "0.2.0", + "configurations": [ + { + "name": "API: Uvicorn (reload)", + "type": "debugpy", + "request": "launch", + "module": "uvicorn", + "args": [ + "api.app:app", + "--reload", + "--host", "0.0.0.0", + "--port", "8000" + ], + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/api/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}" + }, + "justMyCode": false + }, + { + "name": "API: Arq worker (watch)", + "type": "debugpy", + "request": "launch", + "module": "arq", + "args": [ + "api.tasks.arq.WorkerSettings", + "--watch", "${workspaceFolder}/api", + "--custom-log-dict", "api.tasks.arq.LOG_CONFIG" + ], + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/api/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}" + }, + "justMyCode": false + }, + { + "name": "API: Campaign orchestrator", + "type": "debugpy", + "request": "launch", + "module": "api.services.campaign.campaign_orchestrator", + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/api/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}" + }, + "justMyCode": false + }, + { + "name": "API: ARI manager", + "type": "debugpy", + "request": "launch", + "module": "api.services.telephony.ari_manager", + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/api/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}" + }, + "justMyCode": false + }, + { + "name": "Tests: API (pytest, full suite)", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": ["tests", "-xvs"], + "cwd": "${workspaceFolder}/api", + "envFile": "${workspaceFolder}/api/.env.test", + "env": { + "PYTHONPATH": "${workspaceFolder}" + }, + "justMyCode": false + }, + { + "name": "Tests: API (pytest, current file)", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": ["${file}", "-xvs"], + "cwd": "${workspaceFolder}/api", + "envFile": "${workspaceFolder}/api/.env.test", + "env": { + "PYTHONPATH": "${workspaceFolder}" + }, + "justMyCode": false + }, + { + "name": "Tests: Pipecat (pytest, current file)", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": ["${file}", "-xvs"], + "cwd": "${workspaceFolder}/pipecat", + "envFile": "${workspaceFolder}/api/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}/pipecat/src" + }, + "justMyCode": false + }, + { + "name": "Python: Current file", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/api/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}" + }, + "justMyCode": false + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..f1daa35 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.defaultInterpreterPath": "${workspaceFolder}/venv/bin/python" +} diff --git a/api/.env.test.example b/api/.env.test.example new file mode 100644 index 0000000..3953e65 --- /dev/null +++ b/api/.env.test.example @@ -0,0 +1,17 @@ +# Test environment. Read by pytest runs and the "Tests: API" launch +# configurations in .vscode/launch.json. +# +# Tests target a separate database (`test_db`) so they don't clobber dev +# data. Create it once after the postgres container is up: +# docker compose -f docker-compose-local.yaml exec postgres \ +# createdb -U postgres test_db + +ENVIRONMENT="test" +LOG_LEVEL="DEBUG" + +UI_APP_URL=http://localhost:3000 + +DATABASE_URL="postgresql+asyncpg://postgres:postgres@localhost:5432/test_db" +REDIS_URL="redis://:redissecret@localhost:6379/0" + +MINIO_PUBLIC_ENDPOINT=http://localhost:9000 diff --git a/api/Dockerfile b/api/Dockerfile index b871000..f764d86 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,41 +1,52 @@ # Multi-stage Dockerfile -# Stage 1: Builder - Install Python dependencies -FROM python:3.12-slim AS builder +# Stage 1: Builder - Install Python dependencies into a venv via uv +# (mirrors .devcontainer/Dockerfile's venv-builder stage). +FROM python:3.13-slim AS builder WORKDIR /app -# Install git in builder stage (needed for pip install from git) +# Install git in builder stage (needed for any pip install from git URLs) RUN apt-get update && apt-get install -y \ git \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Copy and install requirements -COPY api/requirements.txt . +# uv (https://github.com/astral-sh/uv) for ~5-10x faster installs than pip. +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ -# Install dependencies to user directory for easy copying -RUN pip install --user --no-cache-dir -r requirements.txt && \ - # Clean up pip cache after installation - rm -rf /root/.cache/pip +# Build the venv at the path it will live at in the final image, so shebangs +# and console-scripts inside the venv reference the correct runtime location +# after COPY --from. +ENV VIRTUAL_ENV=/opt/venv \ + PATH=/opt/venv/bin:$PATH +RUN python -m venv "$VIRTUAL_ENV" -# Copy and install pipecat from local submodule -COPY pipecat /tmp/pipecat -RUN pip install --user --no-cache-dir '/tmp/pipecat[cartesia,deepgram,openai,elevenlabs,groq,google,azure,sarvam,soundfile,silero,webrtc,speechmatics,openrouter,camb,mcp]' && \ - # Swap opencv-python (pulled by pipecat[webrtc]) for opencv-python-headless - # to drop X11/Qt dependencies that otherwise require libxcb etc. in runner. - pip uninstall -y opencv-python && \ - pip install --user --no-cache-dir opencv-python-headless && \ - # Pre-download NLTK punkt_tab tokenizer data (required by pipecat at runtime) - python -c "import nltk; nltk.download('punkt_tab', quiet=True)" && \ - # Clean up pip cache and temporary pipecat directory - rm -rf /root/.cache/pip /tmp/pipecat +# Layer 1: API deps. Cache invalidates only when requirements.txt changes. +RUN --mount=type=bind,source=api/requirements.txt,target=/tmp/req.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip install -r /tmp/req.txt -# Strip cache files, test/example dirs, and type stubs from installed packages -RUN find /root/.local -type f -name '*.pyc' -delete && \ - find /root/.local -type d -name '__pycache__' -prune -exec rm -rf {} + && \ - find /root/.local -type f -name '*.pyo' -delete && \ - find /root/.local -type d \( -name tests -o -name test -o -name examples \) -prune -exec rm -rf {} + && \ - find /root/.local -name '*.pyi' -delete +# Layer 2: pipecat deps. Cache invalidates when pipecat source changes. +# After installing pipecat, two hardening tweaks: +# 1. Swap opencv-python (pulled by pipecat[webrtc]) for opencv-python-headless. +# The non-headless build links against X11/Qt (libxcb*); without those +# shared libs in the image, `import cv2` fails at runtime. +# 2. Pre-download NLTK's punkt_tab tokenizer so pipecat's text processing +# doesn't hit the network on first agent run. NLTK auto-finds it under +# sys.prefix/nltk_data, so it travels with the venv on COPY. +RUN --mount=type=bind,source=pipecat,target=/tmp/pipecat,rw \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip install '/tmp/pipecat[cartesia,deepgram,openai,elevenlabs,groq,google,azure,sarvam,soundfile,silero,webrtc,speechmatics,openrouter,camb,mcp]' \ + && uv pip uninstall opencv-python \ + && uv pip install opencv-python-headless \ + && python -c "import nltk; nltk.download('punkt_tab', download_dir='/opt/venv/nltk_data', quiet=True)" + +# Strip cache files, test/example dirs, and type stubs from the venv +RUN find /opt/venv -type f -name '*.pyc' -delete && \ + find /opt/venv -type d -name '__pycache__' -prune -exec rm -rf {} + && \ + find /opt/venv -type f -name '*.pyo' -delete && \ + find /opt/venv -type d \( -name tests -o -name test -o -name examples \) -prune -exec rm -rf {} + && \ + find /opt/venv -name '*.pyi' -delete # Stage 2: Node deps for ts_validator (built with full node:22-slim, only # node_modules is copied into the runner). @@ -56,7 +67,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && chmod +x /usr/local/bin/ffmpeg /usr/local/bin/ffprobe # Stage 4: Runtime - Minimal image with only runtime dependencies -FROM python:3.12-slim AS runner +FROM python:3.13-slim AS runner WORKDIR /app @@ -65,18 +76,17 @@ COPY --from=ffmpeg-static /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg COPY --from=ffmpeg-static /usr/local/bin/ffprobe /usr/local/bin/ffprobe # Node.js 22 binary only (ts_validator subprocess needs node >=22.6 for -# native TypeScript stripping; see api/mcp_server/ts_bridge.py). python:3.12-slim +# native TypeScript stripping; see api/mcp_server/ts_bridge.py). python:3.13-slim # already provides libstdc++6, libgcc-s1, and ca-certificates that node needs. COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node -# Copy Python packages from builder stage -COPY --from=builder /root/.local /root/.local +# Copy the populated venv from the builder stage. NLTK data lives at +# /opt/venv/nltk_data and is auto-discovered via sys.prefix. +COPY --from=builder /opt/venv /opt/venv -# Copy NLTK data (punkt_tab tokenizer) from builder stage -COPY --from=builder /root/nltk_data /root/nltk_data - -# Make sure scripts in .local are available -ENV PATH=/root/.local/bin:$PATH +# Activate the venv for subsequent RUN/CMD layers. +ENV VIRTUAL_ENV=/opt/venv \ + PATH=/opt/venv/bin:$PATH # Set Python to not generate .pyc files in runtime ENV PYTHONDONTWRITEBYTECODE=1 @@ -104,4 +114,4 @@ ENV LOG_TO_FILE=false EXPOSE 8000 # Run the FastAPI app with uvicorn -CMD ["./scripts/start_services_docker.sh"] \ No newline at end of file +CMD ["./scripts/start_services_docker.sh"] diff --git a/api/conftest.py b/api/conftest.py index ab47760..68a8aef 100644 --- a/api/conftest.py +++ b/api/conftest.py @@ -30,6 +30,11 @@ import sys import loguru import pytest +REPO_ROOT = Path(__file__).resolve().parents[1] +SDK_PY_SRC = REPO_ROOT / "sdk" / "python" / "src" +if str(SDK_PY_SRC) not in sys.path: + sys.path.insert(0, str(SDK_PY_SRC)) + from api.constants import APP_ROOT_DIR # noqa: E402 diff --git a/api/pyproject.toml b/api/pyproject.toml index b937d41..71d886e 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -2,4 +2,4 @@ name = "dograh-api" version = "1.31.0" description = "Backend API for Dograh voice AI platform" -requires-python = ">=3.12" +requires-python = ">=3.13,<3.14" diff --git a/api/requirements.dev.txt b/api/requirements.dev.txt index b5bd106..2ea8a2e 100644 --- a/api/requirements.dev.txt +++ b/api/requirements.dev.txt @@ -2,4 +2,3 @@ mypy==2.0.0 watchfiles==1.1.1 datamodel-code-generator==0.56.1 twine==6.2.0 --e ./sdk/python diff --git a/docs/contribution/devcontainer.mdx b/docs/contribution/devcontainer.mdx new file mode 100644 index 0000000..a0ae211 --- /dev/null +++ b/docs/contribution/devcontainer.mdx @@ -0,0 +1,101 @@ +--- +title: Devcontainer Workflow +description: Details for running and maintaining the Dograh devcontainer. +--- + +The checked-in `.devcontainer/` follows the Dev Containers specification, so it should also work in Cursor, JetBrains IDEs, and other Dev Container-compatible editors. Only the VS Code path is actively tested. If something breaks elsewhere, please open a [GitHub issue](https://github.com/dograh-hq/dograh/issues). + +### What the Bootstrap Does + +While the container starts, the devcontainer will: + +- Initialize the `pipecat` git submodule on the host through `initializeCommand` +- Bring up `postgres`, `redis`, and `minio` through Docker Compose and wait for them to be healthy +- Seed the `venv` named volume from the image-baked Python 3.13 venv +- Reinstall `pipecat` as editable from the bind-mounted submodule so source edits take effect +- Create `api/.env`, `api/.env.test`, and `ui/.env` from their `*.example` templates if they do not already exist +- Run `npm ci` for `ui/` and `api/mcp_server/ts_validator/` + +In the API env files, `localhost` for Postgres, Redis, and MinIO is rewritten to the docker network aliases `postgres`, `redis`, and `minio`. `MINIO_PUBLIC_ENDPOINT` deliberately stays on `localhost` because the browser on your host loads from it. + + +If you already had an `api/.env` or `api/.env.test` from host-managed development with `localhost` hosts for Postgres, Redis, or MinIO, the bootstrap leaves it untouched. Edit those URLs to the docker network aliases before starting the backend inside the devcontainer, or delete the file and let the bootstrap recreate it on the next rebuild. + + +### Dev Container CLI + +Install the CLI once on your host: + +```bash +npm install -g @devcontainers/cli +``` + +Start or rebuild the container from the repository root: + +```bash +devcontainer up --workspace-folder . +``` + +Open a shell in the running workspace container: + +```bash +devcontainer exec --workspace-folder . bash +``` + +Run a one-off command from the host: + +```bash +devcontainer exec --workspace-folder . bash scripts/start_services_dev.sh +``` + +### Backend Logs + +Tail backend logs from inside the container: + +```bash +tail -f logs/latest/*.log +``` + +### Restarting the Backend + +Re-run the same start script to restart. It reads the PID files under `run/`, terminates the previous services along with their descendants, and starts fresh ones. + +```bash +bash scripts/start_services_dev.sh +``` + + +`uvicorn` runs with `--reload --reload-dir api`, so edits under `api/` are picked up automatically. The other services (`ari_manager`, `campaign_orchestrator`, `arq`) do not auto-reload; re-run the start script after changing code they execute. + + +### When to Rebuild the Container + +The workspace bind mount and the `venv` / `node_modules` named volumes persist across container restarts, so you rarely need to rebuild. Rebuild only when one of these changes: + +- `.devcontainer/Dockerfile` or `.devcontainer/devcontainer.json` +- `api/requirements.txt` or `api/requirements.dev.txt` +- The `pipecat/` submodule + +Plain source edits, `ui/package.json`, and the `.env.example` templates do **not** require a rebuild. After a `git pull`, a quick check: + +```bash +git diff HEAD@{1} HEAD -- .devcontainer api/requirements.txt api/requirements.dev.txt pipecat +``` + +If the diff is empty, you can keep your current container. + +### Personal Install Hook + +Anything you install inside the container outside the named volumes, notably under `/home/vscode`, is wiped on rebuild. This includes `npm i -g` packages and tools like the Claude or Codex CLIs. + +To reinstall personal tooling automatically on every rebuild, drop an executable script at `.devcontainer/install.local.sh`. It is gitignored and runs at the tail of the post-create bootstrap. Example: + +```bash +#!/usr/bin/env bash +set -euo pipefail + +command -v claude >/dev/null 2>&1 || curl -fsSL https://claude.ai/install.sh | bash +command -v codex >/dev/null 2>&1 || npm i -g @openai/codex +``` + +Keep entries idempotent with `command -v` or a marker file so re-runs are cheap and safe. diff --git a/docs/contribution/fork-workflow.mdx b/docs/contribution/fork-workflow.mdx new file mode 100644 index 0000000..22e7fc5 --- /dev/null +++ b/docs/contribution/fork-workflow.mdx @@ -0,0 +1,41 @@ +--- +title: Fork Workflow +description: Keep your Dograh fork connected to upstream. +--- + +The contributor bootstrap script configures two remotes: + +- `origin`: your fork, where you push +- `upstream`: `dograh-hq/dograh`, where new commits land + +If you cloned `dograh-hq/dograh` directly instead of your fork, run this once inside the devcontainer after it boots: + +```bash +bash scripts/setup_fork.sh +``` + +To pull in upstream changes: + +```bash +git fetch upstream +git checkout main +git merge upstream/main +git push origin main +``` + +Check your remotes any time with: + +```bash +git remote -v +``` + +You should see: + +```bash +origin https://github.com//dograh.git (fetch/push) +upstream https://github.com/dograh-hq/dograh.git (fetch/push) +``` + + +Always push feature branches to **`origin`** (your fork), then open a pull request against `dograh-hq/dograh:main`. Never push directly to `upstream`. + diff --git a/docs/contribution/host-managed-setup.mdx b/docs/contribution/host-managed-setup.mdx new file mode 100644 index 0000000..25dd5d9 --- /dev/null +++ b/docs/contribution/host-managed-setup.mdx @@ -0,0 +1,67 @@ +--- +title: Host-managed Setup +description: Set up Dograh directly on your host without the devcontainer. +--- + +Use this only if you do not want to use the devcontainer. + +### System Requirements + +- Git +- Node.js 24 to run the UI +- Python 3.13 to run the backend +- Docker to run Postgres, Redis, and MinIO locally + +1. Run the contributor bootstrap. It configures `origin` as your fork and `upstream` as `dograh-hq/dograh`, initializes the pipecat submodule, creates the Python venv, and copies the `.env` templates. + +```bash macOS/Linux +bash scripts/setup_fork.sh +``` +```powershell Windows +.\scripts\setup_fork.ps1 +``` + +2. Activate the virtual environment: + +```bash macOS/Linux +source venv/bin/activate +``` +```powershell Windows +.\venv\Scripts\Activate.ps1 +``` + +3. Ensure your local Node version is 24: +```bash +nvm use 24 +``` +4. Install UI dependencies: +```bash +cd ui && npm install && cd .. +``` +5. Start the local Docker services: +```bash +docker compose -f docker-compose-local.yaml up -d +``` +6. Install Python requirements: + +```bash macOS/Linux +bash scripts/setup_requirements.sh --dev +``` +```powershell Windows +.\scripts\setup_requirements.ps1 -Dev +``` + +7. Start the backend services: + +```bash macOS/Linux +bash scripts/start_services_dev.sh +``` +```powershell Windows +.\scripts\start_services_dev.ps1 +``` + +8. Start the UI: +```bash +cd ui && npm run dev +``` +9. Open the application on `http://localhost:3000`. diff --git a/docs/contribution/introduction.mdx b/docs/contribution/introduction.mdx index 0c8b45b..023f239 100644 --- a/docs/contribution/introduction.mdx +++ b/docs/contribution/introduction.mdx @@ -1,4 +1,4 @@ --- title: Contribution -description: If you would like to set up the development environment and use a coding agent like Claude to make changes to the codebase, you can follow this document to help setup the right development environment for yourself. +description: If you would like to set up Dograh development environment in your IDE and use a coding agent like Claude/ Codex to make changes to the codebase, you can follow this document to help setup the right development environment for yourself. --- \ No newline at end of file diff --git a/docs/contribution/setup.mdx b/docs/contribution/setup.mdx index b2fc9b7..a2ed249 100644 --- a/docs/contribution/setup.mdx +++ b/docs/contribution/setup.mdx @@ -1,151 +1,63 @@ --- title: Setup -description: You can use this document to setup the dev environment for yourself. +description: Set up the Dograh contributor environment with the devcontainer-first workflow. --- -If the below steps do not work out for you, it would be great if you can open an issue on [Github](https://github.com/dograh-hq/dograh/issues). +If the steps below do not work for you, please open an issue on [GitHub](https://github.com/dograh-hq/dograh/issues). -### System Requirements -- git to clone the forked repository -- Node.js 24 to run the UI (we recommend using [NVM](https://github.com/nvm-sh/nvm) on macOS/Linux or [NVM for Windows](https://github.com/coreybutler/nvm-windows) on Windows to manage your node versions locally) -- Python 3.13 to run the backend -- Docker to run the database and redis cache locally +### Recommended: Devcontainer Setup - -All commands below are shown for **macOS / Linux**. Expand the **Windows** tab for the PowerShell equivalent where it differs. - +#### System Requirements +- Git +- Docker Desktop or another local Docker engine +- For the IDE path: VS Code with the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) +- For the terminal-only path: Node.js on your host so you can install the Dev Container CLI -### Steps -1. Fork the Dograh repository by going to https://github.com/dograh-hq/dograh -2. Clone **your fork** on your machine. You can skip `--recurse-submodules` here — the bootstrap script in the next step will initialize submodules for you. -``` +1. Fork the Dograh repository at https://github.com/dograh-hq/dograh +2. Clone **your fork**: +```bash git clone https://github.com//dograh cd dograh ``` -3. Run the contributor bootstrap. It configures `origin` (your fork) and `upstream` (`dograh-hq/dograh`), initializes the pipecat submodule, creates the Python venv, and copies the `.env` templates. Re-running it is safe — already-configured pieces are skipped. - -```bash macOS/Linux -bash scripts/setup_fork.sh -``` -```powershell Windows -.\scripts\setup_fork.ps1 -``` - -Activate the venv (the bootstrap script created it but won't activate it for you): - -```bash macOS/Linux -source venv/bin/activate -``` -```powershell Windows -.\venv\Scripts\Activate.ps1 -``` - -4. Ensure you are on right version of Node.js using `node --version` -``` -nvm use 24 -``` -5. Install UI dependencies -``` -cd ui && npm install && cd .. -``` -6. Start local docker services -Please ensure you dont have any other instance of conflicting services running by checking `docker ps` -``` -docker compose -f docker-compose-local.yaml up -d -``` -Verify that the processes have started by running `docker ps` -``` -abhishek$ docker ps -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -9066b7244b2f postgres:17 "docker-entrypoint.s…" 18 seconds ago Up 18 seconds (healthy) 0.0.0.0:5432->5432/tcp, [::]:5432->5432/tcp dograh-postgres-1 -6c7cb8afdf18 redis:7 "docker-entrypoint.s…" 18 seconds ago Up 18 seconds (healthy) 0.0.0.0:6379->6379/tcp, [::]:6379->6379/tcp dograh-redis-1 -a57e3e92b02c minio/minio "/usr/bin/docker-ent…" 18 seconds ago Up 18 seconds (healthy) 127.0.0.1:9000-9001->9000-9001/tcp dograh-minio-1 -``` -7. Install Python requirements. The script installs `api/requirements.txt` and pipecat with the required extras. Add the dev flag if you also want the pipecat dev dependency group (pytest, ruff, pre-commit, etc.). - -```bash macOS/Linux -# Default (runtime only) -bash scripts/setup_requirements.sh +3. Start the devcontainer. -# Include pipecat dev dependencies -bash scripts/setup_requirements.sh --dev -``` -```powershell Windows -# Default (runtime only) -.\scripts\setup_requirements.ps1 + In VS Code, open the repository and run **Dev Containers: Reopen in Container**. -# Include pipecat dev dependencies -.\scripts\setup_requirements.ps1 -Dev + Without an IDE, use the Dev Container CLI: +```bash +npm install -g @devcontainers/cli +devcontainer up --workspace-folder . +devcontainer exec --workspace-folder . bash ``` - -8. Start backend services - -```bash macOS/Linux +4. Wait for the first build to finish. The first build takes several minutes; subsequent opens are much faster. +5. Start the backend from a terminal inside the container: +```bash bash scripts/start_services_dev.sh ``` -```powershell Windows -.\scripts\start_services_dev.ps1 + + Without an IDE, run the same command from your host: +```bash +devcontainer exec --workspace-folder . bash scripts/start_services_dev.sh ``` - -Verify that your backend server is running - -```bash macOS/Linux +6. Start the UI from another terminal inside the container: +```bash +cd ui +npm run dev -- --hostname 0.0.0.0 +``` + + Without an IDE, use another host terminal: +```bash +devcontainer exec --workspace-folder . bash -lc 'cd ui && npm run dev -- --hostname 0.0.0.0' +``` +7. Verify that the backend is healthy: +```bash curl -X GET localhost:8000/api/v1/health ``` -```powershell Windows -curl.exe http://localhost:8000/api/v1/health -``` - -You would be able to see the logs in logs/ directory. - -```bash macOS/Linux -tail -f logs/latest/*.log -``` -```powershell Windows -Get-Content logs/latest/*.log -Wait -``` - +8. Open the app at `http://localhost:3000`. -#### Restarting the backend -Re-run the same start script to restart. It reads the PID files under `run/`, terminates the previous services along with their descendants, and starts fresh ones. - -```bash macOS/Linux -bash scripts/start_services_dev.sh -``` -```powershell Windows -.\scripts\start_services_dev.ps1 -``` - +### More Setup Options - -`uvicorn` runs with `--reload --reload-dir api`, so edits under `api/` are picked up automatically — no restart needed. The other services (`ari_manager`, `campaign_orchestrator`, `arq`) do **not** auto-reload; re-run the start script after changing code they execute. - -9. Start the UI -``` -cd ui && npm run dev -``` -10. You should be able to open the application on `localhost:3000` now - -### Keeping your fork in sync with upstream -The bootstrap script configures two remotes: `origin` (your fork, where you push) and `upstream` (`dograh-hq/dograh`, where new commits land). To pull in upstream changes: - -```bash -git fetch upstream -git checkout main -git merge upstream/main # or: git rebase upstream/main -git push origin main -``` - -Check your remotes any time with `git remote -v`. You should see: -``` -origin https://github.com//dograh.git (fetch/push) -upstream https://github.com/dograh-hq/dograh.git (fetch/push) -``` - - -Always push feature branches to **`origin`** (your fork), then open a pull request against `dograh-hq/dograh:main`. Never push directly to `upstream`. - - -### Next Steps -We ship with AGENTS.md and CLAUDE.md which will help the Coding Agents get started quickly with the codebase. This should help your favourite coding agents to be able to navigate the codebase quickly and you can make changes to it and suit your specification better. +- For what the devcontainer bootstrap does, rebuild guidance, logs, and personal install hooks, see [Devcontainer Workflow](/contribution/devcontainer). +- If you cloned `dograh-hq/dograh` directly instead of your fork, see [Fork Workflow](/contribution/fork-workflow) to reset `origin` and `upstream`. +- If you do not want to use the devcontainer, see [Host-managed Setup](/contribution/host-managed-setup). diff --git a/docs/docs.json b/docs/docs.json index e5711fb..717e90c 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -125,6 +125,16 @@ "tab": "Developer", "icon": "code", "groups": [ + { + "group": "Contribution", + "pages": [ + "contribution/introduction", + "contribution/setup", + "contribution/devcontainer", + "contribution/host-managed-setup", + "contribution/fork-workflow" + ] + }, { "group": "Guides", "pages": [ @@ -152,13 +162,6 @@ "deployment/update", "deployment/heroku" ] - }, - { - "group": "Contribution", - "pages": [ - "contribution/introduction", - "contribution/setup" - ] } ] }, @@ -292,7 +295,6 @@ "search": { "prompt": "Search for Tools, Webhook, Deployment, etc..." }, - "openapi": "/api-reference/openapi.json", "customCSS": "/custom.css", "contextual": { "options": [ @@ -311,5 +313,8 @@ "github": "https://github.com/dograh-hq", "linkedin": "https://linkedin.com/company/dograh" } + }, + "api": { + "openapi": "api-reference/openapi.json" } } diff --git a/scripts/setup_fork.ps1 b/scripts/setup_fork.ps1 index 5844acf..5a5d0e8 100644 --- a/scripts/setup_fork.ps1 +++ b/scripts/setup_fork.ps1 @@ -102,18 +102,33 @@ Write-Host '[3/4] Python virtual environment' -ForegroundColor Blue $VenvPath = Join-Path $BaseDir 'venv' $VenvActivate = Join-Path $VenvPath 'Scripts/Activate.ps1' -if (Test-Path $VenvActivate) { - Write-Host "OK venv already exists at $VenvPath" -ForegroundColor Green -} else { - $py = $null - foreach ($candidate in @('python3.13', 'python', 'python3')) { - if (Get-Command $candidate -ErrorAction SilentlyContinue) { - $py = $candidate - break +function Get-Python313Command { + foreach ($candidate in @('python3.13', 'python3', 'python')) { + if (-not (Get-Command $candidate -ErrorAction SilentlyContinue)) { + continue + } + + $version = & $candidate -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" 2>$null + if ($LASTEXITCODE -eq 0 -and $version -eq '3.13') { + return $candidate } } + + return $null +} + +if (Test-Path $VenvActivate) { + $venvPython = Join-Path $VenvPath 'Scripts/python.exe' + $venvVersion = & $venvPython -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" 2>$null + if ($LASTEXITCODE -ne 0 -or $venvVersion -ne '3.13') { + Write-Host "Error: existing venv uses Python $venvVersion. Remove $VenvPath and re-run with Python 3.13." -ForegroundColor Red + exit 1 + } + Write-Host "OK venv already exists at $VenvPath (Python $venvVersion)" -ForegroundColor Green +} else { + $py = Get-Python313Command if (-not $py) { - Write-Host 'Error: no python interpreter found on PATH. Install Python 3.13.' -ForegroundColor Red + Write-Host 'Error: no Python 3.13 interpreter found on PATH. Install Python 3.13.' -ForegroundColor Red exit 1 } & $py -m venv $VenvPath @@ -128,8 +143,9 @@ Write-Host '' Write-Host '[4/4] Environment files' -ForegroundColor Blue $pairs = @( - @{ Src = 'api/.env.example'; Dst = 'api/.env' }, - @{ Src = 'ui/.env.example'; Dst = 'ui/.env' } + @{ Src = 'api/.env.example'; Dst = 'api/.env' }, + @{ Src = 'api/.env.test.example'; Dst = 'api/.env.test' }, + @{ Src = 'ui/.env.example'; Dst = 'ui/.env' } ) foreach ($p in $pairs) { if (Test-Path $p.Dst) { diff --git a/scripts/setup_fork.sh b/scripts/setup_fork.sh index f7cfe47..502bac5 100755 --- a/scripts/setup_fork.sh +++ b/scripts/setup_fork.sh @@ -102,18 +102,36 @@ echo "" echo -e "${BLUE}[3/4] Python virtual environment${NC}" VENV_PATH="$BASE_DIR/venv" -if [[ -d "$VENV_PATH" && -f "$VENV_PATH/bin/activate" ]]; then - echo -e "${GREEN}✓ venv already exists at $VENV_PATH${NC}" -else - PY="" +find_python_313() { + local candidate="" + local version="" + for candidate in python3.13 python3 python; do - if command -v "$candidate" >/dev/null 2>&1; then - PY="$candidate" - break + if ! command -v "$candidate" >/dev/null 2>&1; then + continue + fi + + version=$("$candidate" -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" 2>/dev/null || true) + if [[ "$version" == "3.13" ]]; then + echo "$candidate" + return 0 fi done + + return 1 +} + +if [[ -d "$VENV_PATH" && -f "$VENV_PATH/bin/activate" ]]; then + VENV_VERSION=$("$VENV_PATH/bin/python" -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" 2>/dev/null || true) + if [[ "$VENV_VERSION" != "3.13" ]]; then + echo -e "${RED}Error: existing venv uses Python ${VENV_VERSION:-unknown}. Remove $VENV_PATH and re-run with Python 3.13.${NC}" + exit 1 + fi + echo -e "${GREEN}✓ venv already exists at $VENV_PATH (Python $VENV_VERSION)${NC}" +else + PY="$(find_python_313 || true)" if [[ -z "$PY" ]]; then - echo -e "${RED}Error: no python interpreter found on PATH. Install Python 3.13.${NC}" + echo -e "${RED}Error: no Python 3.13 interpreter found on PATH. Install Python 3.13.${NC}" exit 1 fi "$PY" -m venv "$VENV_PATH" @@ -126,7 +144,7 @@ echo "" ############################################################################### echo -e "${BLUE}[4/4] Environment files${NC}" -for pair in "api/.env.example|api/.env" "ui/.env.example|ui/.env"; do +for pair in "api/.env.example|api/.env" "api/.env.test.example|api/.env.test" "ui/.env.example|ui/.env"; do src="${pair%|*}" dst="${pair#*|}" if [[ -f "$dst" ]]; then diff --git a/scripts/setup_local.devcontainer.md b/scripts/setup_local.devcontainer.md new file mode 100644 index 0000000..0f7af1f --- /dev/null +++ b/scripts/setup_local.devcontainer.md @@ -0,0 +1,13 @@ +# Devcontainer contributor setup + +`setup_local.sh` and `setup_local.ps1` provision the OSS Docker stack for local +deployments. They are not the recommended contributor workflow for this +repository. + +For day-to-day development, use the checked-in devcontainer under +`.devcontainer/`. The full contributor instructions live in +`../docs/contribution/setup.mdx`. + +The devcontainer flow pins Python 3.13, installs backend and frontend +dependencies in-container, creates a container-specific API env file, and +starts Postgres, Redis, and MinIO automatically. diff --git a/scripts/setup_requirements.ps1 b/scripts/setup_requirements.ps1 index c71dba6..21f5359 100644 --- a/scripts/setup_requirements.ps1 +++ b/scripts/setup_requirements.ps1 @@ -18,6 +18,22 @@ $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $BaseDir = Split-Path -Parent $ScriptDir Set-Location $BaseDir +# Fail early if the active Python is not 3.12 or 3.13. uv pip installs into +# whichever interpreter resolves here (the active venv, or PATH python), so a +# mismatch surfaces as confusing wheel/build errors much later. +$PythonBin = if ($env:PYTHON) { $env:PYTHON } else { 'python' } +if (-not (Get-Command $PythonBin -ErrorAction SilentlyContinue)) { + Write-Error "'$PythonBin' not found on PATH. Activate the project venv (or set `$env:PYTHON) and retry." + exit 1 +} + +$PyMajMin = & $PythonBin -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' +if ($PyMajMin -ne '3.12' -and $PyMajMin -ne '3.13') { + $PyPath = (Get-Command $PythonBin).Source + Write-Error "Python 3.12 or 3.13 required, found $PyMajMin at $PyPath. Activate a venv built with python3.12 or python3.13 and retry." + exit 1 +} + Write-Host "Setting up pipecat as a git submodule..." if (-not $Dev) { @@ -25,24 +41,30 @@ if (-not $Dev) { git submodule update --init --recursive } +# Use uv (https://github.com/astral-sh/uv) for ~5-10x faster installs. +if (-not (Get-Command uv -ErrorAction SilentlyContinue)) { + Write-Host "Installing uv..." + Invoke-RestMethod https://astral.sh/uv/install.ps1 | Invoke-Expression + $env:Path = "$env:USERPROFILE\.local\bin;$env:Path" +} + # Install dograh API requirements first so pipecat's extras win on any # shared transitive dependencies (matches api/Dockerfile and CI workflow). Write-Host "Installing dograh API requirements..." -pip install -r api/requirements.txt +uv pip install -r api/requirements.txt if ($Dev) { Write-Host "Installing dograh API dev requirements..." - pip install -r api/requirements.dev.txt + uv pip install -r api/requirements.dev.txt } # Install pipecat in editable mode with all extras Write-Host "Installing pipecat dependencies..." -pip install -e './pipecat[cartesia,deepgram,openai,elevenlabs,groq,google,azure,sarvam,soundfile,silero,webrtc,speechmatics,openrouter,camb]' +uv pip install -e './pipecat[cartesia,deepgram,openai,elevenlabs,groq,google,azure,sarvam,soundfile,silero,webrtc,speechmatics,openrouter,camb]' if ($Dev) { Write-Host "Installing pipecat dev dependencies..." - pip install --upgrade pip - pip install --group pipecat/pyproject.toml:dev + uv pip install --group pipecat/pyproject.toml:dev } Write-Host "Setup complete! Requirements are installed." diff --git a/scripts/setup_requirements.sh b/scripts/setup_requirements.sh index 201b952..f744a2f 100755 --- a/scripts/setup_requirements.sh +++ b/scripts/setup_requirements.sh @@ -32,6 +32,26 @@ DOGRAH_DIR="$(dirname "$SCRIPT_DIR")" cd "$DOGRAH_DIR" +# Fail early if the active Python is not 3.12 or 3.13. uv pip installs into +# whichever interpreter resolves here (the active venv, or PATH python3), so a +# mismatch surfaces as confusing wheel/build errors much later. +PYTHON_BIN="${PYTHON:-python3}" +if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + echo "Error: '$PYTHON_BIN' not found on PATH." >&2 + echo "Activate the project venv (or set PYTHON=/path/to/python) and retry." >&2 + exit 1 +fi + +PY_MAJ_MIN=$("$PYTHON_BIN" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') +case "$PY_MAJ_MIN" in + 3.12|3.13) ;; + *) + echo "Error: Python 3.12 or 3.13 required, found $PY_MAJ_MIN at $(command -v "$PYTHON_BIN")." >&2 + echo "Activate a venv built with python3.12 or python3.13 and retry." >&2 + exit 1 + ;; +esac + echo "Setting up pipecat as a git submodule..." if [ "$DEV_MODE" -eq 0 ]; then @@ -39,24 +59,32 @@ if [ "$DEV_MODE" -eq 0 ]; then git submodule update --init --recursive fi +# Use uv (https://github.com/astral-sh/uv) for ~5-10x faster installs. +# The devcontainer Dockerfile pre-installs uv; this fallback handles CI runners +# and contributor laptops that don't have it yet. +if ! command -v uv >/dev/null 2>&1; then + echo "Installing uv..." + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="$HOME/.local/bin:$PATH" +fi + # Install dograh API requirements first so pipecat's extras win on any # shared transitive dependencies (matches api/Dockerfile and CI workflow). echo "Installing dograh API requirements..." -pip install -r api/requirements.txt +uv pip install -r api/requirements.txt if [ "$DEV_MODE" -eq 1 ]; then echo "Installing dograh API dev requirements..." - pip install -r api/requirements.dev.txt + uv pip install -r api/requirements.dev.txt fi # Install pipecat in editable mode with all extras echo "Installing pipecat dependencies..." -pip install -e ./pipecat[cartesia,deepgram,openai,elevenlabs,groq,google,azure,sarvam,soundfile,silero,webrtc,speechmatics,openrouter,camb,mcp] +uv pip install -e ./pipecat[cartesia,deepgram,openai,elevenlabs,groq,google,azure,sarvam,soundfile,silero,webrtc,speechmatics,openrouter,camb,mcp] if [ "$DEV_MODE" -eq 1 ]; then echo "Installing pipecat dev dependencies..." - pip install --upgrade pip - pip install --group pipecat/pyproject.toml:dev + uv pip install --group pipecat/pyproject.toml:dev fi echo "Setup complete! Requirements are installed." diff --git a/scripts/start_services_dev.ps1 b/scripts/start_services_dev.ps1 index 27967e5..6a6c78b 100644 --- a/scripts/start_services_dev.ps1 +++ b/scripts/start_services_dev.ps1 @@ -21,7 +21,7 @@ $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $BaseDir = Split-Path -Parent $ScriptDir Set-Location $BaseDir -$EnvFile = Join-Path $BaseDir 'api/.env' +$EnvFile = if ($env:DOGRAH_ENV_FILE) { $env:DOGRAH_ENV_FILE } else { Join-Path $BaseDir 'api/.env' } $RunDir = Join-Path $BaseDir 'run' $LogsRoot = Join-Path $BaseDir 'logs' $LatestDir = Join-Path $LogsRoot 'latest' @@ -29,6 +29,7 @@ $VenvPath = Join-Path $BaseDir 'venv' Write-Host "Starting Dograh Services (DEV MODE) in BASE_DIR: $BaseDir" Write-Host "Auto-reload enabled for api/ directory changes" +Write-Host "Environment file: $EnvFile" ############################################################################### ### 1) Load environment variables diff --git a/scripts/start_services_dev.sh b/scripts/start_services_dev.sh index 7fe8de2..2a608e8 100755 --- a/scripts/start_services_dev.sh +++ b/scripts/start_services_dev.sh @@ -8,7 +8,7 @@ set -e # Exit on error # Determine BASE_DIR as parent of the scripts directory BASE_DIR="$(cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" && pwd)" -ENV_FILE="$BASE_DIR/api/.env" +ENV_FILE="${DOGRAH_ENV_FILE:-$BASE_DIR/api/.env}" RUN_DIR="$BASE_DIR/run" # Where we keep *.pid BASE_LOG_DIR="$BASE_DIR/logs" # Base logs directory @@ -26,6 +26,7 @@ HEALTH_INTERVAL=${HEALTH_INTERVAL:-2} cd "$BASE_DIR" echo "Starting Dograh Services (DEV MODE) at $(date) in BASE_DIR: ${BASE_DIR}" echo "Auto-reload enabled for api/ directory changes" +echo "Environment file: $ENV_FILE" ############################################################################### ### 1) Load environment variables diff --git a/sdk/python/src/dograh_sdk/_generated_models.py b/sdk/python/src/dograh_sdk/_generated_models.py index 95a3a28..d4dbeea 100644 --- a/sdk/python/src/dograh_sdk/_generated_models.py +++ b/sdk/python/src/dograh_sdk/_generated_models.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: -# filename: dograh-openapi-XXXXXX.json.N8gRI5v3bD -# timestamp: 2026-05-23T09:14:22+00:00 +# filename: dograh-openapi-T200ed.json +# timestamp: 2026-05-25T12:42:12+00:00 from __future__ import annotations