diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a807603b8..20c79c42d 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -7,6 +7,7 @@ on: paths: - 'surfsense_web/**' - 'surfsense_backend/**' + - 'docker/docker-compose.e2e.yml' - '.github/workflows/e2e-tests.yml' workflow_dispatch: @@ -19,173 +20,36 @@ jobs: name: Journey runs-on: ubuntu-latest if: github.event.pull_request.draft == false - timeout-minutes: 45 - - # Postgres runs as a step (not a service) - services: - redis: - image: redis:8-alpine - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 + timeout-minutes: 30 env: - DATABASE_URL: postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense_e2e - CELERY_BROKER_URL: redis://localhost:6379/0 - CELERY_RESULT_BACKEND: redis://localhost:6379/0 - REDIS_APP_URL: redis://localhost:6379/0 - SECRET_KEY: ci-test-secret-key-not-for-production - AUTH_TYPE: LOCAL - REGISTRATION_ENABLED: "TRUE" - ETL_SERVICE: DOCLING - EMBEDDING_MODEL: sentence-transformers/all-MiniLM-L6-v2 - NEXT_FRONTEND_URL: http://localhost:3000 - - # Sentinel keys — fakes never read them; turns leaked real calls into 401s. - COMPOSIO_API_KEY: e2e-deny-real-call-sentinel - COMPOSIO_ENABLED: "TRUE" - OPENAI_API_KEY: e2e-deny-real-call-sentinel - ANTHROPIC_API_KEY: e2e-deny-real-call-sentinel - LITELLM_API_KEY: e2e-deny-real-call-sentinel - - MICROSOFT_CLIENT_ID: fake-microsoft-client-id - MICROSOFT_CLIENT_SECRET: fake-microsoft-client-secret - ONEDRIVE_REDIRECT_URI: http://localhost:8000/api/v1/auth/onedrive/connector/callback - DROPBOX_APP_KEY: fake-dropbox-app-key - DROPBOX_APP_SECRET: fake-dropbox-app-secret - DROPBOX_REDIRECT_URI: http://localhost:8000/api/v1/auth/dropbox/connector/callback - + # Test user that the backend creates via /auth/register before Playwright runs. + PLAYWRIGHT_TEST_EMAIL: e2e-test@surfsense.net + PLAYWRIGHT_TEST_PASSWORD: E2eTestPassword123! + # Frontend env: Playwright's webServer (surfsense_web/playwright.config.ts) + # spawns `pnpm build && pnpm start` in CI; these get baked into the build. NEXT_PUBLIC_FASTAPI_BACKEND_URL: http://localhost:8000 NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: LOCAL - PLAYWRIGHT_TEST_EMAIL: e2e-test@surfsense.net - PLAYWRIGHT_TEST_PASSWORD: E2eTestPassword123! - steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v6 - # Started early so it warms up while Python deps install. - - name: Start Postgres + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # ─── Backend stack ───────────────────────────────────────────────── + # Builds the e2e image (multi-stage, deps cached via GHA), brings up + # db + redis + backend + celery_worker, blocks until every healthcheck + # is green. No `uv` invocation on the runner; no PID files; no curl + # polling loops; readiness is gated by Docker healthchecks. + - name: Build & start backend stack run: | - docker run -d \ - --name surfsense_postgres \ - -p 5432:5432 \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=surfsense_e2e \ - pgvector/pgvector:pg17 \ - postgres \ - -c wal_level=logical \ - -c max_wal_senders=10 \ - -c max_replication_slots=10 + docker compose -f docker/docker-compose.e2e.yml \ + up -d --build --wait --wait-timeout 300 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.12' - - - name: Install uv - uses: astral-sh/setup-uv@v8.1.0 - - - name: Cache backend dependencies - uses: actions/cache@v5 - with: - path: | - ~/.cache/uv - surfsense_backend/.venv - key: python-deps-${{ hashFiles('surfsense_backend/uv.lock') }} - restore-keys: | - python-deps- - - - name: Cache HuggingFace models - uses: actions/cache@v5 - with: - path: ~/.cache/huggingface - key: hf-models-${{ env.EMBEDDING_MODEL }}-${{ env.ETL_SERVICE }} - - - name: Install backend dependencies - working-directory: surfsense_backend - run: uv sync - - - name: Wait for Postgres readiness - run: | - for i in $(seq 1 30); do - if docker exec surfsense_postgres pg_isready -U postgres -d surfsense_e2e > /dev/null 2>&1; then - echo "Postgres ready after ${i} attempts" - exit 0 - fi - sleep 2 - done - echo "::error::Postgres failed to become ready within 60s" - docker logs surfsense_postgres --tail 100 - exit 1 - - - name: Run database migrations - working-directory: surfsense_backend - run: uv run alembic upgrade head - - # Do NOT replace with `uvicorn main:app`. run_backend.py hijacks - # sys.modules["composio"] before app import; production binds it - # at import time so plain uvicorn would call the real SDK. - - name: Start backend (E2E entrypoint with sys.modules hijack) - working-directory: surfsense_backend - env: - HTTPS_PROXY: http://127.0.0.1:1 - HTTP_PROXY: http://127.0.0.1:1 - NO_PROXY: localhost,127.0.0.1,0.0.0.0,huggingface.co,*.huggingface.co,*.hf.co,cdn-lfs.huggingface.co - run: | - uv run python tests/e2e/run_backend.py \ - > backend.log 2>&1 & - echo $! > backend.pid - - # Worker is a separate interpreter, so the composio hijack must be reapplied. - - name: Start Celery worker (E2E entrypoint) - working-directory: surfsense_backend - env: - HTTPS_PROXY: http://127.0.0.1:1 - HTTP_PROXY: http://127.0.0.1:1 - NO_PROXY: localhost,127.0.0.1,0.0.0.0,huggingface.co,*.huggingface.co,*.hf.co,cdn-lfs.huggingface.co - run: | - uv run python tests/e2e/run_celery.py \ - > celery.log 2>&1 & - echo $! > celery.pid - - - name: Wait for backend readiness - run: | - for i in $(seq 1 60); do - if curl -sf http://localhost:8000/openapi.json > /dev/null; then - echo "Backend up after ${i} attempts" - exit 0 - fi - sleep 2 - done - echo "::error::Backend failed to start within 120s" - echo "===== backend.log (tail 200) =====" - tail -200 surfsense_backend/backend.log || true - echo "===== celery.log (tail 200) =====" - tail -200 surfsense_backend/celery.log || true - exit 1 - - - name: Wait for Celery worker readiness - working-directory: surfsense_backend - run: | - for i in $(seq 1 30); do - if uv run celery -A app.celery_app inspect ping --timeout 2 \ - > /dev/null 2>&1; then - echo "Celery worker up after ${i} attempts" - exit 0 - fi - sleep 2 - done - echo "::error::Celery worker failed to start within 60s" - echo "===== celery.log (tail 200) =====" - tail -200 celery.log || true - exit 1 + - name: Show backend stack status + if: always() + run: docker compose -f docker/docker-compose.e2e.yml ps - name: Register E2E test user run: | @@ -201,13 +65,14 @@ jobs: exit 1 fi - - name: Setup Node.js - uses: actions/setup-node@v6 + # ─── Frontend (host-side) ────────────────────────────────────────── + # Playwright's webServer block in playwright.config.ts spawns + # `pnpm build && pnpm start` in CI mode and waits for :3000. + - uses: actions/setup-node@v6 with: node-version: '20' - - name: Install pnpm - uses: pnpm/action-setup@v6 + - uses: pnpm/action-setup@v6 with: version: 10 @@ -221,8 +86,7 @@ jobs: with: path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} key: pnpm-${{ runner.os }}-${{ hashFiles('surfsense_web/pnpm-lock.yaml') }} - restore-keys: | - pnpm-${{ runner.os }}- + restore-keys: pnpm-${{ runner.os }}- - name: Install web dependencies working-directory: surfsense_web @@ -253,10 +117,26 @@ jobs: restore-keys: | nextjs-${{ runner.os }}-${{ hashFiles('surfsense_web/pnpm-lock.yaml') }}- + # ─── Tests ───────────────────────────────────────────────────────── - name: Run Playwright tests working-directory: surfsense_web run: pnpm test:e2e:prod + # ─── Failure diagnostics ─────────────────────────────────────────── + - name: Dump backend stack logs on failure + if: failure() + run: | + mkdir -p ./compose-logs + docker compose -f docker/docker-compose.e2e.yml logs --no-color --timestamps \ + > ./compose-logs/all-services.log 2>&1 || true + for svc in db redis backend celery_worker; do + docker compose -f docker/docker-compose.e2e.yml logs --no-color --timestamps "$svc" \ + > "./compose-logs/${svc}.log" 2>&1 || true + done + docker compose -f docker/docker-compose.e2e.yml ps \ + > ./compose-logs/ps.txt 2>&1 || true + + # ─── Artifacts ───────────────────────────────────────────────────── - name: Upload Playwright HTML report if: always() uses: actions/upload-artifact@v7 @@ -273,26 +153,15 @@ jobs: path: surfsense_web/test-results/ retention-days: 14 - - name: Upload backend + celery logs + - name: Upload backend stack logs if: failure() uses: actions/upload-artifact@v7 with: - name: backend-celery-logs - path: | - surfsense_backend/backend.log - surfsense_backend/celery.log + name: backend-stack-logs + path: ./compose-logs/ retention-days: 7 - - name: Stop backend + Celery worker + # ─── Teardown ────────────────────────────────────────────────────── + - name: Tear down backend stack if: always() - working-directory: surfsense_backend - run: | - for f in backend.pid celery.pid; do - if [ -f "$f" ]; then - kill "$(cat $f)" 2>/dev/null || true - fi - done - - - name: Stop Postgres - if: always() - run: docker rm -f surfsense_postgres 2>/dev/null || true + run: docker compose -f docker/docker-compose.e2e.yml down -v --remove-orphans diff --git a/docker/docker-compose.e2e.yml b/docker/docker-compose.e2e.yml new file mode 100644 index 000000000..87b4e7261 --- /dev/null +++ b/docker/docker-compose.e2e.yml @@ -0,0 +1,168 @@ +# ============================================================================= +# SurfSense — E2E Docker Compose stack +# ============================================================================= +# Hermetic backend stack for Playwright E2E tests: +# - db / redis on an internal-only network (no internet egress) +# - backend (FastAPI) joins the internal network AND a separate ingress +# bridge so the host runner can reach :8000 +# - celery_worker on the internal network only — zero egress surface +# +# The backend image is built from surfsense_backend/Dockerfile target=e2e, +# which adds tests/ via the `tests-source` additional context (tests/ is +# excluded from the main context by .dockerignore so production never ships +# test fakes). See surfsense_backend/Dockerfile for stage layout. +# +# Usage from repo root: +# docker compose -f docker/docker-compose.e2e.yml up -d --build --wait +# curl -X POST http://localhost:8000/auth/register ... +# ( run Playwright on host, pointing at localhost:8000 + localhost:3000 ) +# docker compose -f docker/docker-compose.e2e.yml down -v +# ============================================================================= + +name: surfsense-e2e + +x-backend-env: &backend-env + DATABASE_URL: postgresql+asyncpg://postgres:postgres@db:5432/surfsense_e2e + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + REDIS_APP_URL: redis://redis:6379/0 + CELERY_TASK_DEFAULT_QUEUE: surfsense + SECRET_KEY: ci-test-secret-key-not-for-production + AUTH_TYPE: LOCAL + REGISTRATION_ENABLED: "TRUE" + ETL_SERVICE: DOCLING + EMBEDDING_MODEL: sentence-transformers/all-MiniLM-L6-v2 + NEXT_FRONTEND_URL: http://host.docker.internal:3000 + # Sentinel keys — fakes never read them; turns leaked real calls into 401s. + COMPOSIO_API_KEY: e2e-deny-real-call-sentinel + COMPOSIO_ENABLED: "TRUE" + OPENAI_API_KEY: e2e-deny-real-call-sentinel + ANTHROPIC_API_KEY: e2e-deny-real-call-sentinel + LITELLM_API_KEY: e2e-deny-real-call-sentinel + MICROSOFT_CLIENT_ID: fake-microsoft-client-id + MICROSOFT_CLIENT_SECRET: fake-microsoft-client-secret + ONEDRIVE_REDIRECT_URI: http://localhost:8000/api/v1/auth/onedrive/connector/callback + DROPBOX_APP_KEY: fake-dropbox-app-key + DROPBOX_APP_SECRET: fake-dropbox-app-secret + DROPBOX_REDIRECT_URI: http://localhost:8000/api/v1/auth/dropbox/connector/callback + # Defense-in-depth: even though L3 egress is denied for the worker via + # `internal: true`, the backend still has a route via `ingress`. Setting + # HTTPS_PROXY to an unreachable port turns any leaked Python outbound HTTP + # call into a fast Connection refused. UNLIKE the old runner-shell setup, + # this proxy is set on the container env and `uv` is never invoked here, + # so there is no interaction with uv's implicit-sync behaviour. + HTTPS_PROXY: http://127.0.0.1:1 + HTTP_PROXY: http://127.0.0.1:1 + NO_PROXY: localhost,127.0.0.1,0.0.0.0,db,redis,host.docker.internal + +services: + db: + image: pgvector/pgvector:pg17 + command: > + postgres + -c wal_level=logical + -c max_wal_senders=10 + -c max_replication_slots=10 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: surfsense_e2e + # Ephemeral storage — every CI run gets a clean DB, no volume cleanup needed. + tmpfs: + - /var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d surfsense_e2e"] + interval: 2s + timeout: 3s + retries: 30 + networks: [internal] + + redis: + image: redis:8-alpine + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 3s + retries: 30 + networks: [internal] + + backend: + build: + context: ../surfsense_backend + dockerfile: Dockerfile + target: e2e + additional_contexts: + # tests/ is excluded from the main context by .dockerignore; + # the e2e stage's `COPY --from=tests-source` pulls it in here. + tests-source: ../surfsense_backend/tests + cache_from: + - type=gha,scope=surfsense-e2e-backend + cache_to: + - type=gha,mode=max,scope=surfsense-e2e-backend + image: surfsense-e2e-backend:local + environment: + <<: *backend-env + SERVICE_ROLE: api + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - "8000:8000" + depends_on: + db: { condition: service_healthy } + redis: { condition: service_healthy } + healthcheck: + # Use Python (already in the image) instead of curl/wget to avoid + # depending on either tool being installed in the runtime layers. + test: + - CMD + - python + - -c + - | + import sys, urllib.request + try: + r = urllib.request.urlopen("http://localhost:8000/openapi.json", timeout=2) + sys.exit(0 if r.status == 200 else 1) + except Exception: + sys.exit(1) + interval: 3s + timeout: 5s + retries: 60 + start_period: 30s + networks: + - internal # to reach db/redis + - ingress # so host can reach :8000 + + celery_worker: + image: surfsense-e2e-backend:local + pull_policy: never + # No build: section — reuses the image built by the `backend` service. + # Compose v2 builds shared images exactly once across services that + # reference the same `image:` tag. + environment: + <<: *backend-env + SERVICE_ROLE: worker + depends_on: + backend: { condition: service_healthy } + healthcheck: + test: + - CMD-SHELL + - "celery -A app.celery_app inspect ping --timeout 2 | grep -q pong" + interval: 5s + timeout: 5s + retries: 12 + start_period: 20s + networks: [internal] + +networks: + # Internal network: containers attached only to this network have NO route + # to the host or the internet. This is the L3 deny-egress mechanism that + # replaces the fragile HTTPS_PROXY-on-the-runner approach. + internal: + driver: bridge + internal: true + + # Regular bridge network. Only the `backend` service joins it, solely so + # the host can reach :8000 via the published port. celery_worker / db / + # redis stay off this network entirely. + ingress: + driver: bridge diff --git a/surfsense_backend/Dockerfile b/surfsense_backend/Dockerfile index 73d5819b9..a5b391a1e 100644 --- a/surfsense_backend/Dockerfile +++ b/surfsense_backend/Dockerfile @@ -1,8 +1,23 @@ -FROM python:3.12-slim +# ============================================================================= +# SurfSense Backend — Multi-stage Dockerfile +# ============================================================================= +# Stages: +# base — system deps + Pandoc 3.x +# deps — Python deps frozen from uv.lock (no dev deps) +# models — pre-baked offline assets (EasyOCR, Docling, Playwright) +# e2e — adds tests/ via additional_contexts, swaps entrypoint +# production — production runtime (LAST stage = default `docker build` target) +# +# IMPORTANT: `production` MUST remain the last stage. .github/workflows/docker-build.yml +# builds without `target:` and BuildKit defaults to the last stage. Reordering will +# silently break ghcr.io/modsetter/surfsense-backend. +# ============================================================================= + +# ─── Stage 1: base (system deps, Pandoc, certificates) ────────────────────── +FROM python:3.12-slim AS base WORKDIR /app -# Install system dependencies including SSL tools, CUDA dependencies, and Tesseract OCR RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ python3-dev \ @@ -22,21 +37,24 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ && rm -rf /var/lib/apt/lists/* -# Install Pandoc 3.x from GitHub as a fallback for Linux where pypandoc_binary -# may not bundle pandoc (apt ships 2.17 which has broken table rendering). -# pypandoc_binary bundles pandoc on Windows/macOS; on Linux it picks this up. +# Pandoc 3.x from GitHub Releases — apt ships 2.17 which has broken table rendering. +# pypandoc_binary bundles pandoc on Windows/macOS; on Linux it picks up this binary. RUN ARCH=$(dpkg --print-architecture) && \ wget -qO /tmp/pandoc.deb "https://github.com/jgm/pandoc/releases/download/3.9/pandoc-3.9-1-${ARCH}.deb" && \ dpkg -i /tmp/pandoc.deb && \ rm /tmp/pandoc.deb -# Update certificates and install SSL tools RUN update-ca-certificates RUN pip install --upgrade certifi pip-system-certs -# Copy requirements -COPY pyproject.toml . -COPY uv.lock . +ENV SSL_CERT_FILE=/usr/local/lib/python3.12/site-packages/certifi/cacert.pem +ENV REQUESTS_CA_BUNDLE=/usr/local/lib/python3.12/site-packages/certifi/cacert.pem + + +# ─── Stage 2: deps (Python deps frozen from uv.lock) ──────────────────────── +FROM base AS deps + +COPY pyproject.toml uv.lock ./ # Install all Python dependencies from uv.lock for deterministic builds. # @@ -49,9 +67,7 @@ COPY uv.lock . # Note on torch/CUDA: we do NOT install torch from a separate cu* index here. # PyPI's torch wheels for Linux x86_64 already ship CUDA-enabled and pull # nvidia-cudnn-cu13, nvidia-nccl-cu13, triton, etc. as install deps (all -# captured in uv.lock). Installing from cu121 first only wasted ~2GB of -# downloads that the lock-based install immediately replaced. If a specific -# CUDA version is needed (driver compatibility, etc.), wire it through +# captured in uv.lock). If a specific CUDA version is needed, wire it through # [tool.uv.sources] in pyproject.toml so the lock stays the source of truth. RUN pip install --no-cache-dir uv && \ uv export --frozen --no-dev --no-hashes --no-emit-project \ @@ -59,49 +75,32 @@ RUN pip install --no-cache-dir uv && \ uv pip install --system --no-cache-dir -r /tmp/requirements.txt && \ rm /tmp/requirements.txt -# Set SSL environment variables dynamically -RUN CERTIFI_PATH=$(python -c "import certifi; print(certifi.where())") && \ - echo "Setting SSL_CERT_FILE to $CERTIFI_PATH" && \ - echo "export SSL_CERT_FILE=$CERTIFI_PATH" >> /root/.bashrc && \ - echo "export REQUESTS_CA_BUNDLE=$CERTIFI_PATH" >> /root/.bashrc -ENV SSL_CERT_FILE=/usr/local/lib/python3.12/site-packages/certifi/cacert.pem -ENV REQUESTS_CA_BUNDLE=/usr/local/lib/python3.12/site-packages/certifi/cacert.pem + +# ─── Stage 3: models (pre-baked offline assets) ───────────────────────────── +FROM deps AS models # Pre-download EasyOCR models to avoid runtime SSL issues -RUN mkdir -p /root/.EasyOCR/model -RUN wget --no-check-certificate https://github.com/JaidedAI/EasyOCR/releases/download/v1.3/english_g2.zip -O /root/.EasyOCR/model/english_g2.zip || true -RUN wget --no-check-certificate https://github.com/JaidedAI/EasyOCR/releases/download/pre-v1.1.6/craft_mlt_25k.zip -O /root/.EasyOCR/model/craft_mlt_25k.zip || true -RUN cd /root/.EasyOCR/model && (unzip -o english_g2.zip || true) && (unzip -o craft_mlt_25k.zip || true) +RUN mkdir -p /root/.EasyOCR/model && \ + wget --no-check-certificate https://github.com/JaidedAI/EasyOCR/releases/download/v1.3/english_g2.zip -O /root/.EasyOCR/model/english_g2.zip || true && \ + wget --no-check-certificate https://github.com/JaidedAI/EasyOCR/releases/download/pre-v1.1.6/craft_mlt_25k.zip -O /root/.EasyOCR/model/craft_mlt_25k.zip || true && \ + cd /root/.EasyOCR/model && \ + (unzip -o english_g2.zip || true) && \ + (unzip -o craft_mlt_25k.zip || true) # Pre-download Docling models RUN python -c "try:\n from docling.document_converter import DocumentConverter\n conv = DocumentConverter()\nexcept:\n pass" || true -# Install Playwright browsers for web scraping (the playwright package itself -# is already installed via uv.lock above) +# Install Playwright browsers (the playwright python package itself is in deps) RUN playwright install chromium --with-deps -# Copy source code -COPY . . - -# Install the project itself in editable mode. Dependencies were already -# installed deterministically from uv.lock above, so --no-deps prevents any -# re-resolution that could pull newer versions. -RUN uv pip install --system --no-cache-dir --no-deps -e . - -# Copy and set permissions for entrypoint script -# Use dos2unix to ensure LF line endings (fixes CRLF issues from Windows checkouts) -COPY scripts/docker/entrypoint.sh /app/scripts/docker/entrypoint.sh -RUN dos2unix /app/scripts/docker/entrypoint.sh && chmod +x /app/scripts/docker/entrypoint.sh - # Shared temp directory for file uploads between API and Worker containers. # Python's tempfile module uses TMPDIR, so uploaded files land here. # Mount the SAME volume at /shared_tmp on both API and Worker in Coolify. RUN mkdir -p /shared_tmp -ENV TMPDIR=/shared_tmp -# Prevent uvloop compatibility issues ENV PYTHONPATH=/app ENV UVICORN_LOOP=asyncio +ENV TMPDIR=/shared_tmp # Tune glibc malloc to return freed memory to the OS more aggressively. # Without these, Python's gc.collect() frees objects but the underlying @@ -110,6 +109,58 @@ ENV MALLOC_MMAP_THRESHOLD_=65536 ENV MALLOC_TRIM_THRESHOLD_=131072 ENV MALLOC_MMAP_MAX_=65536 + +# ─── Stage 4: e2e (production source + tests/ + e2e entrypoint) ───────────── +# Built via `docker buildx build --target e2e`. The default build target is +# `production` (the last stage), so this stage is opt-in for CI only. +# +# `tests/` is excluded from the main build context by .dockerignore (so prod +# can never accidentally ship test fakes). The e2e stage receives tests/ +# through an "additional context" passed by docker-compose.e2e.yml — see +# https://docs.docker.com/reference/compose-file/build/#additional_contexts +FROM models AS e2e + +# Same source copy as production. .dockerignore filters out tests/. +COPY . . + +# Bring tests/ in via the named additional build context. CI passes +# --build-context tests-source=./tests +# (or the equivalent additional_contexts entry in docker-compose.e2e.yml). +COPY --from=tests-source . ./tests/ + +# Install the project itself in editable mode. Dependencies were already +# installed deterministically from uv.lock above, so --no-deps prevents any +# re-resolution that could pull newer versions. +RUN uv pip install --system --no-cache-dir --no-deps -e . + +COPY scripts/docker/entrypoint.e2e.sh /app/scripts/docker/entrypoint.e2e.sh +RUN dos2unix /app/scripts/docker/entrypoint.e2e.sh && chmod +x /app/scripts/docker/entrypoint.e2e.sh + +# SERVICE_ROLE is overridden per service in docker-compose.e2e.yml (api / worker). +ENV SERVICE_ROLE=api + +EXPOSE 8000-8001 +CMD ["/app/scripts/docker/entrypoint.e2e.sh"] + + +# ─── Stage 5: production (LAST stage — default `docker build` target) ─────── +# Behavior is byte-identical to the previous single-stage Dockerfile. +# .github/workflows/docker-build.yml builds without `target:` and BuildKit +# defaults to the last stage, so this MUST stay last. +FROM models AS production + +# Copy source code (tests/ excluded by .dockerignore — production never ships tests). +COPY . . + +# Install the project itself in editable mode. Dependencies were already +# installed deterministically from uv.lock above, so --no-deps prevents any +# re-resolution that could pull newer versions. +RUN uv pip install --system --no-cache-dir --no-deps -e . + +# Use dos2unix to ensure LF line endings (fixes CRLF issues from Windows checkouts) +COPY scripts/docker/entrypoint.sh /app/scripts/docker/entrypoint.sh +RUN dos2unix /app/scripts/docker/entrypoint.sh && chmod +x /app/scripts/docker/entrypoint.sh + # SERVICE_ROLE controls which process this container runs: # api – FastAPI backend only (runs migrations on startup) # worker – Celery worker only @@ -127,6 +178,5 @@ ENV CELERY_MAX_TASKS_PER_CHILD=50 # "" – both queues (default, for single-worker setups) ENV CELERY_QUEUES="" -# Run EXPOSE 8000-8001 -CMD ["/app/scripts/docker/entrypoint.sh"] \ No newline at end of file +CMD ["/app/scripts/docker/entrypoint.sh"] diff --git a/surfsense_backend/scripts/docker/entrypoint.e2e.sh b/surfsense_backend/scripts/docker/entrypoint.e2e.sh new file mode 100755 index 000000000..84cfe2568 --- /dev/null +++ b/surfsense_backend/scripts/docker/entrypoint.e2e.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# ============================================================================= +# E2E entrypoint for the multi-stage Dockerfile's `e2e` target. +# +# Dispatches on SERVICE_ROLE to the test-only entrypoints under tests/e2e/. +# Those scripts apply sys.modules hijacks and LLM/embedding patches BEFORE +# importing production app code (see tests/e2e/run_backend.py for rationale). +# +# Production never sees this file: tests/ is excluded from the production +# stage, and the production stage uses scripts/docker/entrypoint.sh. +# ============================================================================= +set -euo pipefail + +SERVICE_ROLE="${SERVICE_ROLE:-api}" +echo "[e2e-entrypoint] starting role=${SERVICE_ROLE}" + +wait_for_db() { + # Block until the database is reachable. We don't loop forever — Compose + # depends_on/healthchecks already gate on db readiness, this is just + # belt-and-suspenders so a slow first connection doesn't race migrations. + for i in {1..60}; do + if python -c "from app.db import engine; import asyncio; asyncio.run(engine.dispose())" 2>/dev/null; then + echo "[e2e-entrypoint] db reachable after ${i} attempts" + return 0 + fi + sleep 1 + done + echo "[e2e-entrypoint] ERROR: db not reachable after 60s" >&2 + return 1 +} + +case "${SERVICE_ROLE}" in + api) + wait_for_db + echo "[e2e-entrypoint] running alembic upgrade head" + alembic upgrade head + # `exec` so SIGTERM from `docker stop` reaches Python directly, + # without a shell wrapper interposing. + exec python tests/e2e/run_backend.py + ;; + worker) + # Worker doesn't run migrations — the api role does that exactly once. + # We still wait for db so Celery's broker connection check doesn't + # race against an unready Postgres on cold start. + wait_for_db + exec python tests/e2e/run_celery.py + ;; + *) + echo "[e2e-entrypoint] ERROR: unknown SERVICE_ROLE='${SERVICE_ROLE}' (expected: api | worker)" >&2 + exit 1 + ;; +esac diff --git a/surfsense_backend/tests/e2e/run_backend.py b/surfsense_backend/tests/e2e/run_backend.py index 4156a4ea4..a34327908 100644 --- a/surfsense_backend/tests/e2e/run_backend.py +++ b/surfsense_backend/tests/e2e/run_backend.py @@ -57,6 +57,29 @@ sys.modules["notion_client.errors"] = _fake_notion.errors from dotenv import load_dotenv # noqa: E402 load_dotenv() + +os.environ.setdefault( + "DATABASE_URL", + "postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense", +) +os.environ.setdefault("CELERY_BROKER_URL", "redis://localhost:6379/0") +os.environ.setdefault("CELERY_RESULT_BACKEND", "redis://localhost:6379/0") +os.environ.setdefault("REDIS_APP_URL", "redis://localhost:6379/0") +os.environ.setdefault("CELERY_TASK_DEFAULT_QUEUE", "surfsense") +os.environ.setdefault("SECRET_KEY", "local-e2e-secret-not-for-production") +os.environ.setdefault("AUTH_TYPE", "LOCAL") +os.environ.setdefault("REGISTRATION_ENABLED", "TRUE") +os.environ.setdefault("ETL_SERVICE", "DOCLING") +os.environ.setdefault("EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2") +os.environ.setdefault("NEXT_FRONTEND_URL", "http://localhost:3000") + +# Sentinel keys — fakes never read them; turns leaked real calls into 401s. +os.environ.setdefault("COMPOSIO_API_KEY", "local-deny-real-call-sentinel") +os.environ.setdefault("COMPOSIO_ENABLED", "TRUE") +os.environ.setdefault("OPENAI_API_KEY", "local-deny-real-call-sentinel") +os.environ.setdefault("ANTHROPIC_API_KEY", "local-deny-real-call-sentinel") +os.environ.setdefault("LITELLM_API_KEY", "local-deny-real-call-sentinel") + os.environ.setdefault("ATLASSIAN_CLIENT_ID", "fake-atlassian-client-id") os.environ.setdefault("ATLASSIAN_CLIENT_SECRET", "fake-atlassian-client-secret") os.environ.setdefault( diff --git a/surfsense_backend/tests/e2e/run_celery.py b/surfsense_backend/tests/e2e/run_celery.py index 407fecde3..787eb5486 100644 --- a/surfsense_backend/tests/e2e/run_celery.py +++ b/surfsense_backend/tests/e2e/run_celery.py @@ -44,6 +44,29 @@ sys.modules["notion_client.errors"] = _fake_notion.errors from dotenv import load_dotenv # noqa: E402 load_dotenv() + +os.environ.setdefault( + "DATABASE_URL", + "postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense", +) +os.environ.setdefault("CELERY_BROKER_URL", "redis://localhost:6379/0") +os.environ.setdefault("CELERY_RESULT_BACKEND", "redis://localhost:6379/0") +os.environ.setdefault("REDIS_APP_URL", "redis://localhost:6379/0") +os.environ.setdefault("CELERY_TASK_DEFAULT_QUEUE", "surfsense") +os.environ.setdefault("SECRET_KEY", "local-e2e-secret-not-for-production") +os.environ.setdefault("AUTH_TYPE", "LOCAL") +os.environ.setdefault("REGISTRATION_ENABLED", "TRUE") +os.environ.setdefault("ETL_SERVICE", "DOCLING") +os.environ.setdefault("EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2") +os.environ.setdefault("NEXT_FRONTEND_URL", "http://localhost:3000") + +# Sentinel keys — fakes never read them; turns leaked real calls into 401s. +os.environ.setdefault("COMPOSIO_API_KEY", "local-deny-real-call-sentinel") +os.environ.setdefault("COMPOSIO_ENABLED", "TRUE") +os.environ.setdefault("OPENAI_API_KEY", "local-deny-real-call-sentinel") +os.environ.setdefault("ANTHROPIC_API_KEY", "local-deny-real-call-sentinel") +os.environ.setdefault("LITELLM_API_KEY", "local-deny-real-call-sentinel") + os.environ.setdefault("ATLASSIAN_CLIENT_ID", "fake-atlassian-client-id") os.environ.setdefault("ATLASSIAN_CLIENT_SECRET", "fake-atlassian-client-secret") os.environ.setdefault( @@ -198,12 +221,19 @@ def _main() -> None: # so Drive indexing tasks are picked up). queue_name = os.getenv("CELERY_TASK_DEFAULT_QUEUE", "surfsense") queues = f"{queue_name},{queue_name}.connectors" + + # macOS forks-after-MPS-init crash prefork workers; threads avoid it. + default_pool = "threads" if sys.platform == "darwin" else "prefork" + pool = os.getenv("CELERY_POOL", default_pool) + concurrency = os.getenv("CELERY_CONCURRENCY", "2") + celery_app.worker_main( argv=[ "worker", "--loglevel=info", f"--queues={queues}", - "--concurrency=2", + f"--pool={pool}", + f"--concurrency={concurrency}", "--without-gossip", "--without-mingle", ] diff --git a/surfsense_web/playwright.config.ts b/surfsense_web/playwright.config.ts index 0dfdf80bf..0fecc73ef 100644 --- a/surfsense_web/playwright.config.ts +++ b/surfsense_web/playwright.config.ts @@ -4,6 +4,11 @@ const PORT = process.env.PORT || "3000"; const BACKEND_PORT = process.env.BACKEND_PORT || "8000"; const baseURL = process.env.PLAYWRIGHT_BASE_URL || `http://localhost:${PORT}`; +process.env.PLAYWRIGHT_TEST_EMAIL ??= "e2e-test@surfsense.net"; +process.env.PLAYWRIGHT_TEST_PASSWORD ??= "E2eTestPassword123!"; +process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL ??= `http://localhost:${BACKEND_PORT}`; +process.env.NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE ??= "LOCAL"; + /** * Playwright configuration for SurfSense web E2E tests. * @@ -60,9 +65,13 @@ export default defineConfig({ url: `http://localhost:${PORT}`, reuseExistingServer: !process.env.CI, timeout: process.env.CI ? 300_000 : 180_000, + stdout: "pipe", + stderr: "pipe", env: { - NEXT_PUBLIC_FASTAPI_BACKEND_URL: `http://localhost:${BACKEND_PORT}`, - NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: "LOCAL", + NEXT_PUBLIC_FASTAPI_BACKEND_URL: + process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL, + NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: + process.env.NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE, }, }, }); diff --git a/surfsense_web/tests/auth.setup.ts b/surfsense_web/tests/auth.setup.ts index e5d31a257..064552904 100644 --- a/surfsense_web/tests/auth.setup.ts +++ b/surfsense_web/tests/auth.setup.ts @@ -10,15 +10,14 @@ import { expect, test as setup } from "@playwright/test"; * POST /auth/jwt/login -> { access_token } * localStorage.setItem("surfsense_bearer_token", access_token) * - * Requires a seeded test user in the dev/test DB. Configure via env: - * PLAYWRIGHT_TEST_EMAIL, PLAYWRIGHT_TEST_PASSWORD - * NEXT_PUBLIC_FASTAPI_BACKEND_URL (defaults to http://localhost:8000) + * Requires a seeded test user in the dev/test DB. Defaults match the + * docker/docker-compose.e2e.yml local stack and can be overridden via env. */ const authFile = path.join(__dirname, "..", "playwright", ".auth", "user.json"); -const TEST_USER_EMAIL = process.env.PLAYWRIGHT_TEST_EMAIL || "test@surfsense.net"; -const TEST_USER_PASSWORD = process.env.PLAYWRIGHT_TEST_PASSWORD || "TestPassword123!"; +const TEST_USER_EMAIL = process.env.PLAYWRIGHT_TEST_EMAIL || "e2e-test@surfsense.net"; +const TEST_USER_PASSWORD = process.env.PLAYWRIGHT_TEST_PASSWORD || "E2eTestPassword123!"; const BACKEND_URL = process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000"; const STORAGE_KEY = "surfsense_bearer_token"; diff --git a/surfsense_web/tests/helpers/api/auth.ts b/surfsense_web/tests/helpers/api/auth.ts index c912afedc..02aeb6d69 100644 --- a/surfsense_web/tests/helpers/api/auth.ts +++ b/surfsense_web/tests/helpers/api/auth.ts @@ -11,8 +11,8 @@ import type { APIRequestContext } from "@playwright/test"; export const BACKEND_URL = process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000"; -const TEST_USER_EMAIL = process.env.PLAYWRIGHT_TEST_EMAIL || "test@surfsense.net"; -const TEST_USER_PASSWORD = process.env.PLAYWRIGHT_TEST_PASSWORD || "TestPassword123!"; +const TEST_USER_EMAIL = process.env.PLAYWRIGHT_TEST_EMAIL || "e2e-test@surfsense.net"; +const TEST_USER_PASSWORD = process.env.PLAYWRIGHT_TEST_PASSWORD || "E2eTestPassword123!"; export async function loginAsTestUser(request: APIRequestContext): Promise { const response = await request.post(`${BACKEND_URL}/auth/jwt/login`, {