apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-17 18:35:19 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: implement test-only token mint endpoint and update E2E test authentication flow

Browse source
This commit is contained in:
Anish Sarkar 2026-05-11 19:48:18 +05:30
parent 741d6e7eea
commit b247ff37df
8 changed files with 344 additions and 170 deletions
Download patch file Download diff file Expand all files Collapse all files

68
surfsense_backend/tests/e2e/auth_mint.py Normal file
View file

@ -0,0 +1,68 @@
"""Test-only token mint endpoint for the E2E backend entrypoint.
Mounted by ``tests/e2e/run_backend.py`` so Playwright can authenticate
the seeded e2e user without hitting ``/auth/jwt/login`` (rate-limited
to 5/min/IP in production). NEVER ships to production: this whole
``tests/`` tree is excluded from the production Docker image by
``surfsense_backend/.dockerignore``.
Authn: shared secret in ``X-E2E-Mint-Secret``. Same value is set on the
backend container env (``docker/docker-compose.e2e.yml``) and exported
to the Playwright runner (``.github/workflows/e2e-tests.yml``).
"""
from __future__ import annotations
import logging
import os
from fastapi import APIRouter, FastAPI, Header, HTTPException
from pydantic import BaseModel
from sqlalchemy import select
from app.db import User, async_session_maker
from app.users import get_jwt_strategy
_logger = logging.getLogger("surfsense.e2e.auth_mint")
class MintRequest(BaseModel):
email: str = "e2e-test@surfsense.net"
class MintResponse(BaseModel):
access_token: str
token_type: str = "bearer"
def _expected_secret() -> str:
return os.environ.get(
"E2E_MINT_SECRET", "local-e2e-mint-secret-not-for-production"
)
router = APIRouter(prefix="/__e2e__", tags=["__e2e__"])
@router.post("/auth/token", response_model=MintResponse)
async def mint_test_token(
body: MintRequest,
x_e2e_mint_secret: str = Header(..., alias="X-E2E-Mint-Secret"),
) -> MintResponse:
if x_e2e_mint_secret != _expected_secret():
raise HTTPException(status_code=403, detail="invalid e2e mint secret")
async with async_session_maker() as session:
result = await session.execute(select(User).where(User.email == body.email))
user = result.scalar_one_or_none()
if user is None:
raise HTTPException(
status_code=404, detail=f"e2e user {body.email!r} not seeded"
)
token = await get_jwt_strategy().write_token(user)
return MintResponse(access_token=token)
def install(app: FastAPI) -> None:
"""Mount the test-only mint router onto the given FastAPI app."""
app.include_router(router)
_logger.warning("[e2e] mounted POST /__e2e__/auth/token (test-only token mint)")

307
surfsense_backend/tests/e2e/run_backend.py
View file

@ -23,15 +23,12 @@ Usage:
from __future__ import annotations
import asyncio
import logging
import os
import sys
# ---------------------------------------------------------------------------
# 1) Hijack sys.modules BEFORE any production import.
# Production: composio_service.py:11 does `from composio import Composio`.
# With this hijack in place, that import resolves to our strict fake.
# ---------------------------------------------------------------------------
import uvicorn
# Make the surfsense_backend root importable as a top-level package so
# `import tests.e2e.fakes...` works regardless of how the entrypoint is
@ -42,120 +39,113 @@ _BACKEND_ROOT = os.path.abspath(os.path.join(_THIS_DIR, "..", ".."))
if _BACKEND_ROOT not in sys.path:
sys.path.insert(0, _BACKEND_ROOT)
import tests.e2e.fakes.composio_module as _fake_composio # noqa: E402
import tests.e2e.fakes.notion_module as _fake_notion # noqa: E402
sys.modules["composio"] = _fake_composio
sys.modules["notion_client"] = _fake_notion
sys.modules["notion_client.errors"] = _fake_notion.errors
# ---------------------------------------------------------------------------
# 2) Standard logging + dotenv so the rest of the app behaves like main.py.
# ---------------------------------------------------------------------------
from dotenv import load_dotenv # noqa: E402
load_dotenv()
os.environ.setdefault(
"DATABASE_URL",
"postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense",
)
os.environ.setdefault("CELERY_BROKER_URL", "redis://localhost:6379/0")
os.environ.setdefault("CELERY_RESULT_BACKEND", "redis://localhost:6379/0")
os.environ.setdefault("REDIS_APP_URL", "redis://localhost:6379/0")
os.environ.setdefault("CELERY_TASK_DEFAULT_QUEUE", "surfsense")
os.environ.setdefault("SECRET_KEY", "local-e2e-secret-not-for-production")
os.environ.setdefault("AUTH_TYPE", "LOCAL")
os.environ.setdefault("REGISTRATION_ENABLED", "TRUE")
os.environ.setdefault("ETL_SERVICE", "DOCLING")
os.environ.setdefault("EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
os.environ.setdefault("NEXT_FRONTEND_URL", "http://localhost:3000")
# Sentinel keys — fakes never read them; turns leaked real calls into 401s.
os.environ.setdefault("COMPOSIO_API_KEY", "local-deny-real-call-sentinel")
os.environ.setdefault("COMPOSIO_ENABLED", "TRUE")
os.environ.setdefault("OPENAI_API_KEY", "local-deny-real-call-sentinel")
os.environ.setdefault("ANTHROPIC_API_KEY", "local-deny-real-call-sentinel")
os.environ.setdefault("LITELLM_API_KEY", "local-deny-real-call-sentinel")
os.environ.setdefault("ATLASSIAN_CLIENT_ID", "fake-atlassian-client-id")
os.environ.setdefault("ATLASSIAN_CLIENT_SECRET", "fake-atlassian-client-secret")
os.environ.setdefault(
"CONFLUENCE_REDIRECT_URI",
"http://localhost:8000/api/v1/auth/confluence/connector/callback",
)
os.environ.setdefault("NOTION_CLIENT_ID", "fake-notion-client-id")
os.environ.setdefault("NOTION_CLIENT_SECRET", "fake-notion-client-secret")
os.environ.setdefault(
"NOTION_REDIRECT_URI",
"http://localhost:8000/api/v1/auth/notion/connector/callback",
)
os.environ.setdefault("MICROSOFT_CLIENT_ID", "fake-microsoft-client-id")
os.environ.setdefault("MICROSOFT_CLIENT_SECRET", "fake-microsoft-client-secret")
os.environ.setdefault(
"ONEDRIVE_REDIRECT_URI",
"http://localhost:8000/api/v1/auth/onedrive/connector/callback",
)
os.environ.setdefault("DROPBOX_APP_KEY", "fake-dropbox-app-key")
os.environ.setdefault("DROPBOX_APP_SECRET", "fake-dropbox-app-secret")
os.environ.setdefault(
"DROPBOX_REDIRECT_URI",
"http://localhost:8000/api/v1/auth/dropbox/connector/callback",
)
os.environ["SLACK_CLIENT_ID"] = "fake-slack-mcp-client-id"
os.environ["SLACK_CLIENT_SECRET"] = "fake-slack-mcp-client-secret"
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger("surfsense.e2e.backend")
logger.warning(
"*** SURFSENSE E2E BACKEND ENTRYPOINT — fake Composio + LLM + embeddings ***"
)
# ---------------------------------------------------------------------------
# 3) Now import the production app. Every module in app.* loads here,
# creating their bindings (some of which we will patch in step 4).
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# 4) Patch LLM + embedding bindings at every consumer site.
# Composio is already covered by the sys.modules hijack in step 1.
# ---------------------------------------------------------------------------
from unittest.mock import patch # noqa: E402
from app.app import app # noqa: E402
from tests.e2e.fakes import ( # noqa: E402
clickup_module as _fake_clickup_module,
confluence_indexer as _fake_confluence_indexer,
confluence_oauth as _fake_confluence_oauth,
dropbox_api as _fake_dropbox_api,
embeddings as _fake_embeddings,
jira_module as _fake_jira_module,
linear_module as _fake_linear_module,
mcp_oauth_runtime as _fake_mcp_oauth_runtime,
mcp_runtime as _fake_mcp_runtime,
native_google as _fake_native_google,
notion_module as _fake_notion_module,
onedrive_graph as _fake_onedrive_graph,
slack_module as _fake_slack_module,
)
from tests.e2e.fakes.chat_llm import ( # noqa: E402
fake_create_chat_litellm_from_agent_config,
fake_create_chat_litellm_from_config,
)
from tests.e2e.fakes.llm import fake_get_user_long_context_llm # noqa: E402
# Patches started during bootstrap are kept alive for the lifetime of the
# process. We never call .stop() on them.
_active_patches: list = []
def _hijack_external_sdks() -> None:
"""Replace composio + notion_client in sys.modules.
Production does ``from composio import Composio`` and
``import notion_client`` at import time. With this hijack in place,
those imports resolve to our strict fakes.
MUST run before _import_production_app().
"""
import tests.e2e.fakes.composio_module as _fake_composio
import tests.e2e.fakes.notion_module as _fake_notion
sys.modules["composio"] = _fake_composio
sys.modules["notion_client"] = _fake_notion
sys.modules["notion_client.errors"] = _fake_notion.errors
def _load_dotenv_and_set_env_defaults() -> None:
"""Load .env and set every env var the production config reads on import.
MUST run before _import_production_app(), since app.config consumes
these values at import time.
"""
from dotenv import load_dotenv
load_dotenv()
os.environ.setdefault(
"DATABASE_URL",
"postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense",
)
os.environ.setdefault("CELERY_BROKER_URL", "redis://localhost:6379/0")
os.environ.setdefault("CELERY_RESULT_BACKEND", "redis://localhost:6379/0")
os.environ.setdefault("REDIS_APP_URL", "redis://localhost:6379/0")
os.environ.setdefault("CELERY_TASK_DEFAULT_QUEUE", "surfsense")
os.environ.setdefault("SECRET_KEY", "local-e2e-secret-not-for-production")
os.environ.setdefault("AUTH_TYPE", "LOCAL")
os.environ.setdefault("REGISTRATION_ENABLED", "TRUE")
os.environ.setdefault("ETL_SERVICE", "DOCLING")
os.environ.setdefault("EMBEDDING_MODEL", "sentence-transformers/all-MiniLM-L6-v2")
os.environ.setdefault("NEXT_FRONTEND_URL", "http://localhost:3000")
# Sentinel keys — fakes never read them; turns leaked real calls into 401s.
os.environ.setdefault("COMPOSIO_API_KEY", "local-deny-real-call-sentinel")
os.environ.setdefault("COMPOSIO_ENABLED", "TRUE")
os.environ.setdefault("OPENAI_API_KEY", "local-deny-real-call-sentinel")
os.environ.setdefault("ANTHROPIC_API_KEY", "local-deny-real-call-sentinel")
os.environ.setdefault("LITELLM_API_KEY", "local-deny-real-call-sentinel")
os.environ.setdefault("ATLASSIAN_CLIENT_ID", "fake-atlassian-client-id")
os.environ.setdefault("ATLASSIAN_CLIENT_SECRET", "fake-atlassian-client-secret")
os.environ.setdefault(
"CONFLUENCE_REDIRECT_URI",
"http://localhost:8000/api/v1/auth/confluence/connector/callback",
)
os.environ.setdefault("NOTION_CLIENT_ID", "fake-notion-client-id")
os.environ.setdefault("NOTION_CLIENT_SECRET", "fake-notion-client-secret")
os.environ.setdefault(
"NOTION_REDIRECT_URI",
"http://localhost:8000/api/v1/auth/notion/connector/callback",
)
os.environ.setdefault("MICROSOFT_CLIENT_ID", "fake-microsoft-client-id")
os.environ.setdefault("MICROSOFT_CLIENT_SECRET", "fake-microsoft-client-secret")
os.environ.setdefault(
"ONEDRIVE_REDIRECT_URI",
"http://localhost:8000/api/v1/auth/onedrive/connector/callback",
)
os.environ.setdefault("DROPBOX_APP_KEY", "fake-dropbox-app-key")
os.environ.setdefault("DROPBOX_APP_SECRET", "fake-dropbox-app-secret")
os.environ.setdefault(
"DROPBOX_REDIRECT_URI",
"http://localhost:8000/api/v1/auth/dropbox/connector/callback",
)
os.environ["SLACK_CLIENT_ID"] = "fake-slack-mcp-client-id"
os.environ["SLACK_CLIENT_SECRET"] = "fake-slack-mcp-client-secret"
def _import_production_app():
"""Import and return the production FastAPI app.
Every module under ``app.*`` loads here, creating their bindings.
The LLM/embedding factories captured at this point will be replaced
by patches in _patch_llm_bindings() below.
"""
from app.app import app as production_app
return production_app
def _patch_llm_bindings() -> None:
"""Replace LLM factories at every known binding site."""
from unittest.mock import patch
from tests.e2e.fakes.chat_llm import (
fake_create_chat_litellm_from_agent_config,
fake_create_chat_litellm_from_config,
)
from tests.e2e.fakes.llm import fake_get_user_long_context_llm
targets = [
"app.services.llm_service.get_user_long_context_llm",
"app.tasks.connector_indexers.confluence_indexer.get_user_long_context_llm",
@ -213,38 +203,85 @@ def _patch_llm_bindings() -> None:
logger.warning("[fake-chat-llm] could not patch %s: %s.", target, exc)
_patch_llm_bindings()
_fake_embeddings.install(_active_patches)
_fake_confluence_oauth.install(_active_patches)
_fake_confluence_indexer.install(_active_patches)
_fake_native_google.install(_active_patches)
_fake_onedrive_graph.install(_active_patches)
_fake_dropbox_api.install(_active_patches)
_fake_notion_module.install(_active_patches)
_fake_linear_module.install(_active_patches)
_fake_jira_module.install(_active_patches)
_fake_clickup_module.install(_active_patches)
_fake_mcp_runtime.install(_active_patches)
_fake_mcp_oauth_runtime.install(_active_patches)
_fake_slack_module.install(_active_patches)
def _install_runtime_fakes() -> None:
"""Run each fake's install() against the active patch stack."""
from tests.e2e.fakes import (
clickup_module as _fake_clickup_module,
confluence_indexer as _fake_confluence_indexer,
confluence_oauth as _fake_confluence_oauth,
dropbox_api as _fake_dropbox_api,
embeddings as _fake_embeddings,
jira_module as _fake_jira_module,
linear_module as _fake_linear_module,
mcp_oauth_runtime as _fake_mcp_oauth_runtime,
mcp_runtime as _fake_mcp_runtime,
native_google as _fake_native_google,
notion_module as _fake_notion_module,
onedrive_graph as _fake_onedrive_graph,
slack_module as _fake_slack_module,
)
_fake_embeddings.install(_active_patches)
_fake_confluence_oauth.install(_active_patches)
_fake_confluence_indexer.install(_active_patches)
_fake_native_google.install(_active_patches)
_fake_onedrive_graph.install(_active_patches)
_fake_dropbox_api.install(_active_patches)
_fake_notion_module.install(_active_patches)
_fake_linear_module.install(_active_patches)
_fake_jira_module.install(_active_patches)
_fake_clickup_module.install(_active_patches)
_fake_mcp_runtime.install(_active_patches)
_fake_mcp_oauth_runtime.install(_active_patches)
_fake_slack_module.install(_active_patches)
# ---------------------------------------------------------------------------
# 5) Mount test-only middleware. Production never reaches this code.
# ---------------------------------------------------------------------------
def _install_test_only_app_extensions(app) -> None:
"""Mount test-only middleware + the /__e2e__ token mint router.
from tests.e2e.middleware.scenario import ScenarioMiddleware # noqa: E402
POST /__e2e__/auth/token bypasses /auth/jwt/login's 5/min/IP rate
limit so Playwright workers can authenticate without thrashing the
production auth surface. See tests/e2e/auth_mint.py.
"""
from tests.e2e.auth_mint import install as install_e2e_mint
from tests.e2e.middleware.scenario import ScenarioMiddleware
app.add_middleware(ScenarioMiddleware)
app.add_middleware(ScenarioMiddleware)
install_e2e_mint(app)
# ---------------------------------------------------------------------------
# 6) Start uvicorn, mirroring main.py's behaviour.
# ---------------------------------------------------------------------------
def _bootstrap():
"""Run the full E2E bootstrap and return the production FastAPI app.
import asyncio # noqa: E402
Ordering is load-bearing:
1) Hijack composio + notion_client in sys.modules.
2) Load .env + set env defaults (app.config reads env on import).
3) Configure logging.
4) Import production app (which transitively imports the now-faked
external SDKs and reads the env defaults).
5) Patch LLM / embedding bindings at every consumer site.
6) Mount test-only middleware + /__e2e__ routes onto the app.
"""
_hijack_external_sdks()
_load_dotenv_and_set_env_defaults()
import uvicorn # noqa: E402
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
logger.warning(
"*** SURFSENSE E2E BACKEND ENTRYPOINT — fake Composio + LLM + embeddings ***"
)
production_app = _import_production_app()
_patch_llm_bindings()
_install_runtime_fakes()
_install_test_only_app_extensions(production_app)
return production_app
app = _bootstrap()
def _main() -> None: