diff --git a/docker-compose-local.yaml b/docker-compose-local.yaml index 28ba69b..7c3bae3 100644 --- a/docker-compose-local.yaml +++ b/docker-compose-local.yaml @@ -58,23 +58,6 @@ services: networks: - app-network - coturn: - image: coturn/coturn:4.8.0 - container_name: coturn - restart: unless-stopped - ports: - - "3478:3478/udp" # TURN/STUN UDP - - "3478:3478/tcp" # TURN/STUN TCP - - "5349:5349/tcp" # TURNS (TLS) - - "49152-49200:49152-49200/udp" # Relay ports - volumes: - - ./config/coturn/turnserver.conf:/etc/coturn/turnserver.conf:ro - command: - - "-c" - - "/etc/coturn/turnserver.conf" - networks: - - app-network - volumes: postgres_data: redis_data: diff --git a/docs/contribution/setup.mdx b/docs/contribution/setup.mdx index a2629ab..b2fc9b7 100644 --- a/docs/contribution/setup.mdx +++ b/docs/contribution/setup.mdx @@ -18,19 +18,26 @@ All commands below are shown for **macOS / Linux**. Expand the **Windows** tab f ### Steps 1. Fork the Dograh repository by going to https://github.com/dograh-hq/dograh -2. Clone the forked repository on your machine (use `--recurse-submodules` so the pipecat submodule is pulled in too) +2. Clone **your fork** on your machine. You can skip `--recurse-submodules` here — the bootstrap script in the next step will initialize submodules for you. ``` -git clone --recurse-submodules https://github.com//dograh +git clone https://github.com//dograh cd dograh ``` -3. Create a python virtual environment +3. Run the contributor bootstrap. It configures `origin` (your fork) and `upstream` (`dograh-hq/dograh`), initializes the pipecat submodule, creates the Python venv, and copies the `.env` templates. Re-running it is safe — already-configured pieces are skipped. + +```bash macOS/Linux +bash scripts/setup_fork.sh +``` +```powershell Windows +.\scripts\setup_fork.ps1 +``` + +Activate the venv (the bootstrap script created it but won't activate it for you): ```bash macOS/Linux -python3 -m venv venv source venv/bin/activate ``` ```powershell Windows -python -m venv venv .\venv\Scripts\Activate.ps1 ``` @@ -55,17 +62,7 @@ CONTAINER ID IMAGE COMMAND CREATED STATUS 6c7cb8afdf18 redis:7 "docker-entrypoint.s…" 18 seconds ago Up 18 seconds (healthy) 0.0.0.0:6379->6379/tcp, [::]:6379->6379/tcp dograh-redis-1 a57e3e92b02c minio/minio "/usr/bin/docker-ent…" 18 seconds ago Up 18 seconds (healthy) 127.0.0.1:9000-9001->9000-9001/tcp dograh-minio-1 ``` -7. Setup environment variables - -```bash macOS/Linux -cp api/.env.example api/.env && cp ui/.env.example ui/.env -``` -```powershell Windows -Copy-Item api/.env.example api/.env -Copy-Item ui/.env.example ui/.env -``` - -8. Install Python requirements. The script initializes the pipecat submodule, installs `api/requirements.txt`, and installs pipecat with the required extras. Add the dev flag if you also want the pipecat dev dependency group (pytest, ruff, pre-commit, etc.). +7. Install Python requirements. The script installs `api/requirements.txt` and pipecat with the required extras. Add the dev flag if you also want the pipecat dev dependency group (pytest, ruff, pre-commit, etc.). ```bash macOS/Linux # Default (runtime only) @@ -82,7 +79,7 @@ bash scripts/setup_requirements.sh --dev .\scripts\setup_requirements.ps1 -Dev ``` -9. Start backend services +8. Start backend services ```bash macOS/Linux bash scripts/start_services_dev.sh @@ -124,11 +121,31 @@ bash scripts/start_services_dev.sh `uvicorn` runs with `--reload --reload-dir api`, so edits under `api/` are picked up automatically — no restart needed. The other services (`ari_manager`, `campaign_orchestrator`, `arq`) do **not** auto-reload; re-run the start script after changing code they execute. -10. Start the UI +9. Start the UI ``` cd ui && npm run dev ``` -11. You should be able to open the application on `localhost:3000` now +10. You should be able to open the application on `localhost:3000` now + +### Keeping your fork in sync with upstream +The bootstrap script configures two remotes: `origin` (your fork, where you push) and `upstream` (`dograh-hq/dograh`, where new commits land). To pull in upstream changes: + +```bash +git fetch upstream +git checkout main +git merge upstream/main # or: git rebase upstream/main +git push origin main +``` + +Check your remotes any time with `git remote -v`. You should see: +``` +origin https://github.com//dograh.git (fetch/push) +upstream https://github.com/dograh-hq/dograh.git (fetch/push) +``` + + +Always push feature branches to **`origin`** (your fork), then open a pull request against `dograh-hq/dograh:main`. Never push directly to `upstream`. + ### Next Steps We ship with AGENTS.md and CLAUDE.md which will help the Coding Agents get started quickly with the codebase. This should help your favourite coding agents to be able to navigate the codebase quickly and you can make changes to it and suit your specification better. diff --git a/scripts/CLAUDE.md b/scripts/CLAUDE.md new file mode 100644 index 0000000..1c3e942 --- /dev/null +++ b/scripts/CLAUDE.md @@ -0,0 +1,30 @@ +# scripts/ + +## Bash ↔ PowerShell parity — keep them in sync + +Most contributor-facing scripts ship as a `.sh` + `.ps1` pair so macOS/Linux and Windows users get the same workflow. **When you edit one, edit the other in the same change.** Env-var names, defaults, flags, and behavior should match — if `start_services_dev.sh` reads `HEALTH_MAX_ATTEMPTS`, so should `start_services_dev.ps1`. + +Current pairs: +- `setup_fork.{sh,ps1}` — contributor bootstrap (git remotes, submodule, venv, env files) +- `setup_requirements.{sh,ps1}` — Python + pipecat dependency install +- `start_services_dev.{sh,ps1}` — local backend launcher (auto-reload + health-check wait) +- `stop_services.{sh,ps1}` +- `makemigrate.{sh,ps1}` / `migrate.{sh,ps1}` — Alembic helpers + +Bash-only (deployment / CI / OSS-user setup — not intended for Windows contributors): +- `start_services.sh` — VM production +- `start_services_docker.sh` — Docker image CMD +- `rolling_update.sh` — zero-downtime VM redeploy +- `setup_local.sh` / `setup_remote.sh` — OSS Docker-compose setup +- `format.sh` / `lint.sh` / `pre_commit.sh` +- `generate_sdk.sh` / `release_sdks.sh` / `dump_docs_openapi.py` + +## The three "start" scripts — pick the right one + +| Script | Where it runs | Key behavior | +| ----------------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `start_services_dev.sh` | Local dev shell | `uvicorn --reload`, exits after launching, restart by re-running, single arq worker, waits for `/api/v1/health` before exiting. | +| `start_services.sh` | VM production | Multi-port uvicorn behind nginx, `sudo nginx -t && systemctl reload`, writes `run/active_band` for `rolling_update.sh`. | +| `start_services_docker.sh` | Docker image `CMD` | PID 1: traps SIGTERM, uvicorn `--workers $FASTAPI_WORKERS`, `wait -n` so a dying child tears the container down. | + +If you find yourself adding nginx/sudo logic to the dev script, or `--reload` to the production/Docker scripts, stop — you probably want a different file. diff --git a/scripts/setup_fork.ps1 b/scripts/setup_fork.ps1 new file mode 100644 index 0000000..5844acf --- /dev/null +++ b/scripts/setup_fork.ps1 @@ -0,0 +1,165 @@ +#!/usr/bin/env pwsh +# Contributor bootstrap (Windows). Run this once after cloning your fork. +# Configures git remotes (origin = your fork, upstream = dograh-hq/dograh), +# initializes the pipecat submodule, creates the Python venv, and copies +# the .env templates. + +$ErrorActionPreference = 'Stop' + +$UpstreamUrl = 'https://github.com/dograh-hq/dograh.git' +$CanonicalHttps = $UpstreamUrl +$CanonicalSsh = 'git@github.com:dograh-hq/dograh.git' + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$BaseDir = Split-Path -Parent $ScriptDir +Set-Location $BaseDir + +# Must be inside a git repo +try { + git rev-parse --git-dir | Out-Null +} catch { + Write-Host 'Error: not a git repository. Run this from inside your cloned fork.' -ForegroundColor Red + exit 1 +} + +Write-Host '+==============================================================+' -ForegroundColor Blue +Write-Host '| Dograh Contributor Bootstrap |' -ForegroundColor Blue +Write-Host '+==============================================================+' -ForegroundColor Blue +Write-Host '' + +function Get-RemoteUrl([string]$Name) { + try { return (git remote get-url $Name 2>$null) } catch { return $null } +} + +############################################################################### +### 1) Configure git remotes +############################################################################### + +Write-Host '[1/4] Configuring git remotes' -ForegroundColor Blue + +$currentOrigin = Get-RemoteUrl 'origin' + +$needsForkPrompt = $false +if (-not $currentOrigin) { + $needsForkPrompt = $true +} elseif ($currentOrigin -eq $CanonicalHttps -or $currentOrigin -eq $CanonicalSsh) { + Write-Host "origin currently points at the canonical repo ($currentOrigin)." -ForegroundColor Yellow + Write-Host 'You should push to your own fork, not the canonical repo.' -ForegroundColor Yellow + $needsForkPrompt = $true +} + +if ($needsForkPrompt) { + Write-Host 'Enter your fork URL (e.g. https://github.com//dograh.git):' -ForegroundColor Yellow + $forkUrl = (Read-Host '>').Trim() + if (-not $forkUrl) { + Write-Host 'Fork URL is required.' -ForegroundColor Red + exit 1 + } + if ($currentOrigin) { + git remote remove origin | Out-Null + } + git remote add origin $forkUrl + Write-Host "OK origin set to $forkUrl" -ForegroundColor Green +} else { + Write-Host "OK origin already set: $currentOrigin" -ForegroundColor Green +} + +$existingUpstream = Get-RemoteUrl 'upstream' +if (-not $existingUpstream) { + git remote add upstream $UpstreamUrl + Write-Host "OK upstream set to $UpstreamUrl" -ForegroundColor Green +} elseif ($existingUpstream -ne $UpstreamUrl -and $existingUpstream -ne $CanonicalSsh) { + Write-Host "upstream currently points at $existingUpstream (expected $UpstreamUrl)." -ForegroundColor Yellow + $reset = (Read-Host 'Reset upstream to dograh-hq/dograh? [y/N]').Trim() + if ($reset -match '^[Yy]') { + git remote set-url upstream $UpstreamUrl + Write-Host "OK upstream reset to $UpstreamUrl" -ForegroundColor Green + } else { + Write-Host 'Leaving upstream alone.' -ForegroundColor Yellow + } +} else { + Write-Host 'OK upstream already set' -ForegroundColor Green +} + +Write-Host '' +git remote -v +Write-Host '' + +############################################################################### +### 2) Initialize submodules +############################################################################### + +Write-Host '[2/4] Initializing pipecat submodule' -ForegroundColor Blue +git submodule update --init --recursive +Write-Host 'OK submodules initialized' -ForegroundColor Green +Write-Host '' + +############################################################################### +### 3) Python venv +############################################################################### + +Write-Host '[3/4] Python virtual environment' -ForegroundColor Blue +$VenvPath = Join-Path $BaseDir 'venv' +$VenvActivate = Join-Path $VenvPath 'Scripts/Activate.ps1' + +if (Test-Path $VenvActivate) { + Write-Host "OK venv already exists at $VenvPath" -ForegroundColor Green +} else { + $py = $null + foreach ($candidate in @('python3.13', 'python', 'python3')) { + if (Get-Command $candidate -ErrorAction SilentlyContinue) { + $py = $candidate + break + } + } + if (-not $py) { + Write-Host 'Error: no python interpreter found on PATH. Install Python 3.13.' -ForegroundColor Red + exit 1 + } + & $py -m venv $VenvPath + $ver = (& $py --version) + Write-Host "OK venv created at $VenvPath using $py ($ver)" -ForegroundColor Green +} +Write-Host '' + +############################################################################### +### 4) .env files +############################################################################### + +Write-Host '[4/4] Environment files' -ForegroundColor Blue +$pairs = @( + @{ Src = 'api/.env.example'; Dst = 'api/.env' }, + @{ Src = 'ui/.env.example'; Dst = 'ui/.env' } +) +foreach ($p in $pairs) { + if (Test-Path $p.Dst) { + Write-Host "OK $($p.Dst) already exists" -ForegroundColor Green + } elseif (Test-Path $p.Src) { + Copy-Item $p.Src $p.Dst + Write-Host "OK created $($p.Dst) from $($p.Src)" -ForegroundColor Green + } else { + Write-Host "WARN $($p.Src) not found, skipping" -ForegroundColor Yellow + } +} +Write-Host '' + +############################################################################### +### Done +############################################################################### + +Write-Host '+==============================================================+' -ForegroundColor Green +Write-Host '| Bootstrap complete |' -ForegroundColor Green +Write-Host '+==============================================================+' -ForegroundColor Green +Write-Host '' +Write-Host 'Next steps:' -ForegroundColor Yellow +Write-Host ' 1. .\venv\Scripts\Activate.ps1' +Write-Host ' 2. .\scripts\setup_requirements.ps1' +Write-Host ' 3. cd ui; npm install; cd ..' +Write-Host ' 4. docker compose -f docker-compose-local.yaml up -d' +Write-Host ' 5. .\scripts\start_services_dev.ps1' +Write-Host '' +Write-Host 'To sync your fork with upstream later:' -ForegroundColor Yellow +Write-Host ' git fetch upstream' +Write-Host ' git checkout main; git merge upstream/main' +Write-Host ' git push origin main' +Write-Host '' diff --git a/scripts/setup_fork.sh b/scripts/setup_fork.sh new file mode 100755 index 0000000..f7cfe47 --- /dev/null +++ b/scripts/setup_fork.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Contributor bootstrap. Run this once after cloning your fork. +# Configures git remotes (origin = your fork, upstream = dograh-hq/dograh), +# initializes the pipecat submodule, creates the Python venv, and copies +# the .env templates. + +set -e + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +RED='\033[0;31m' +NC='\033[0m' + +UPSTREAM_URL="https://github.com/dograh-hq/dograh.git" + +BASE_DIR="$(cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" && pwd)" +cd "$BASE_DIR" + +if ! git rev-parse --git-dir > /dev/null 2>&1; then + echo -e "${RED}Error: not a git repository. Run this from inside your cloned fork.${NC}" + exit 1 +fi + +echo -e "${BLUE}╔══════════════════════════════════════════════════════════════╗${NC}" +echo -e "${BLUE}║ Dograh Contributor Bootstrap ║${NC}" +echo -e "${BLUE}╚══════════════════════════════════════════════════════════════╝${NC}" +echo "" + +############################################################################### +### 1) Configure git remotes +############################################################################### + +echo -e "${BLUE}[1/4] Configuring git remotes${NC}" + +current_origin=$(git remote get-url origin 2>/dev/null || echo "") +canonical_https="https://github.com/dograh-hq/dograh.git" +canonical_ssh="git@github.com:dograh-hq/dograh.git" + +# If origin is missing or points at the canonical repo (i.e. user cloned the +# canonical repo directly without forking), prompt for the fork URL. +needs_fork_prompt=false +if [[ -z "$current_origin" ]]; then + needs_fork_prompt=true +elif [[ "$current_origin" == "$canonical_https" || "$current_origin" == "$canonical_ssh" ]]; then + echo -e "${YELLOW}origin currently points at the canonical repo ($current_origin).${NC}" + echo -e "${YELLOW}You should push to your own fork, not the canonical repo.${NC}" + needs_fork_prompt=true +fi + +if $needs_fork_prompt; then + echo -e "${YELLOW}Enter your fork URL (e.g. https://github.com//dograh.git):${NC}" + read -r -p "> " FORK_URL + if [[ -z "$FORK_URL" ]]; then + echo -e "${RED}Fork URL is required.${NC}" + exit 1 + fi + if [[ -n "$current_origin" ]]; then + git remote remove origin + fi + git remote add origin "$FORK_URL" + echo -e "${GREEN}✓ origin set to $FORK_URL${NC}" +else + echo -e "${GREEN}✓ origin already set: $current_origin${NC}" +fi + +existing_upstream=$(git remote get-url upstream 2>/dev/null || echo "") +if [[ -z "$existing_upstream" ]]; then + git remote add upstream "$UPSTREAM_URL" + echo -e "${GREEN}✓ upstream set to $UPSTREAM_URL${NC}" +elif [[ "$existing_upstream" != "$UPSTREAM_URL" && "$existing_upstream" != "$canonical_ssh" ]]; then + echo -e "${YELLOW}upstream currently points at $existing_upstream (expected $UPSTREAM_URL).${NC}" + echo -e "${YELLOW}Reset upstream to dograh-hq/dograh? [y/N]:${NC}" + read -r -p "> " RESET_UPSTREAM + if [[ "$RESET_UPSTREAM" =~ ^[Yy] ]]; then + git remote set-url upstream "$UPSTREAM_URL" + echo -e "${GREEN}✓ upstream reset to $UPSTREAM_URL${NC}" + else + echo -e "${YELLOW}Leaving upstream alone.${NC}" + fi +else + echo -e "${GREEN}✓ upstream already set${NC}" +fi + +echo "" +git remote -v +echo "" + +############################################################################### +### 2) Initialize submodules +############################################################################### + +echo -e "${BLUE}[2/4] Initializing pipecat submodule${NC}" +git submodule update --init --recursive +echo -e "${GREEN}✓ submodules initialized${NC}" +echo "" + +############################################################################### +### 3) Python venv +############################################################################### + +echo -e "${BLUE}[3/4] Python virtual environment${NC}" +VENV_PATH="$BASE_DIR/venv" + +if [[ -d "$VENV_PATH" && -f "$VENV_PATH/bin/activate" ]]; then + echo -e "${GREEN}✓ venv already exists at $VENV_PATH${NC}" +else + PY="" + for candidate in python3.13 python3 python; do + if command -v "$candidate" >/dev/null 2>&1; then + PY="$candidate" + break + fi + done + if [[ -z "$PY" ]]; then + echo -e "${RED}Error: no python interpreter found on PATH. Install Python 3.13.${NC}" + exit 1 + fi + "$PY" -m venv "$VENV_PATH" + echo -e "${GREEN}✓ venv created at $VENV_PATH using $PY ($("$PY" --version))${NC}" +fi +echo "" + +############################################################################### +### 4) .env files +############################################################################### + +echo -e "${BLUE}[4/4] Environment files${NC}" +for pair in "api/.env.example|api/.env" "ui/.env.example|ui/.env"; do + src="${pair%|*}" + dst="${pair#*|}" + if [[ -f "$dst" ]]; then + echo -e "${GREEN}✓ $dst already exists${NC}" + elif [[ -f "$src" ]]; then + cp "$src" "$dst" + echo -e "${GREEN}✓ created $dst from $src${NC}" + else + echo -e "${YELLOW}⚠ $src not found, skipping${NC}" + fi +done +echo "" + +############################################################################### +### Done +############################################################################### + +echo -e "${GREEN}╔══════════════════════════════════════════════════════════════╗${NC}" +echo -e "${GREEN}║ Bootstrap complete ║${NC}" +echo -e "${GREEN}╚══════════════════════════════════════════════════════════════╝${NC}" +echo "" +echo -e "${YELLOW}Next steps:${NC}" +echo " 1. source venv/bin/activate" +echo " 2. bash scripts/setup_requirements.sh" +echo " 3. (cd ui && npm install)" +echo " 4. docker compose -f docker-compose-local.yaml up -d" +echo " 5. bash scripts/start_services_dev.sh" +echo "" +echo -e "${YELLOW}To sync your fork with upstream later:${NC}" +echo " git fetch upstream" +echo " git checkout main && git merge upstream/main" +echo " git push origin main" +echo "" diff --git a/scripts/start_services_dev.ps1 b/scripts/start_services_dev.ps1 index 47e74b6..27967e5 100644 --- a/scripts/start_services_dev.ps1 +++ b/scripts/start_services_dev.ps1 @@ -46,7 +46,11 @@ if (Test-Path $EnvFile) { } } -if (-not $env:UVICORN_BASE_PORT) { $env:UVICORN_BASE_PORT = '8000' } +if (-not $env:UVICORN_BASE_PORT) { $env:UVICORN_BASE_PORT = '8000' } + +$HealthEndpoint = '/api/v1/health' +$HealthMaxAttempts = if ($env:HEALTH_MAX_ATTEMPTS) { [int]$env:HEALTH_MAX_ATTEMPTS } else { 30 } +$HealthInterval = if ($env:HEALTH_INTERVAL) { [int]$env:HEALTH_INTERVAL } else { 2 } ############################################################################### ### 2) Define services @@ -129,7 +133,35 @@ foreach ($spec in $serviceSpecs) { } ############################################################################### -### 8) Summary +### 8) Wait for uvicorn health check +############################################################################### + +$healthUrl = "http://127.0.0.1:$($env:UVICORN_BASE_PORT)$HealthEndpoint" +Write-Host "Waiting for uvicorn health check at $healthUrl ..." + +$healthy = $false +for ($attempt = 1; $attempt -le $HealthMaxAttempts; $attempt++) { + try { + $resp = Invoke-WebRequest -Uri $healthUrl -UseBasicParsing -TimeoutSec 2 -ErrorAction Stop + if ($resp.StatusCode -eq 200) { + Write-Host "OK uvicorn healthy (attempt $attempt)" + $healthy = $true + break + } + } catch { + # connection refused / timeout / non-200 — keep polling + } + Start-Sleep -Seconds $HealthInterval +} + +if (-not $healthy) { + Write-Host "FAIL uvicorn FAILED health check after $HealthMaxAttempts attempts." + Write-Host " Check logs: Get-Content logs/latest/uvicorn.log -Wait" + exit 1 +} + +############################################################################### +### 9) Summary ############################################################################### Write-Host "" diff --git a/scripts/start_services_dev.sh b/scripts/start_services_dev.sh index 23458aa..7fe8de2 100755 --- a/scripts/start_services_dev.sh +++ b/scripts/start_services_dev.sh @@ -19,6 +19,10 @@ VENV_PATH="$BASE_DIR/venv" LOG_TO_FILE=${LOG_TO_FILE:-true} +HEALTH_CHECK_ENDPOINT="/api/v1/health" +HEALTH_MAX_ATTEMPTS=${HEALTH_MAX_ATTEMPTS:-30} +HEALTH_INTERVAL=${HEALTH_INTERVAL:-2} + cd "$BASE_DIR" echo "Starting Dograh Services (DEV MODE) at $(date) in BASE_DIR: ${BASE_DIR}" echo "Auto-reload enabled for api/ directory changes" @@ -193,7 +197,32 @@ for i in "${!SERVICE_NAMES[@]}"; do done ############################################################################### -### 8) Summary +### 8) Wait for uvicorn health check +############################################################################### + +echo "Waiting for uvicorn health check at http://127.0.0.1:${UVICORN_BASE_PORT}${HEALTH_CHECK_ENDPOINT} ..." + +healthy=false +for ((attempt = 1; attempt <= HEALTH_MAX_ATTEMPTS; attempt++)); do + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${UVICORN_BASE_PORT}${HEALTH_CHECK_ENDPOINT}" 2>/dev/null || echo "000") + + if [[ "$http_code" == "200" ]]; then + echo "✓ uvicorn healthy (attempt $attempt)" + healthy=true + break + fi + sleep "$HEALTH_INTERVAL" +done + +if ! $healthy; then + echo "✗ uvicorn FAILED health check after $HEALTH_MAX_ATTEMPTS attempts." + echo " Check logs: tail -f $LOG_DIR/uvicorn.log" + exit 1 +fi + +############################################################################### +### 9) Summary ############################################################################### echo