From 650b691a398d9f78b1875bdaf76b221aa10a8a94 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Tue, 12 May 2026 02:37:39 +0530 Subject: [PATCH] chore: enhance E2E tests by adding synthetic global LLM config and updating environment variables for Google OAuth --- .github/workflows/e2e-tests.yml | 3 +- surfsense_backend/.gitignore | 2 +- .../tests/e2e/fixtures/global_llm_config.yaml | 45 +++++++++++ surfsense_backend/tests/e2e/run_backend.py | 75 ++++++++++++++++++- surfsense_backend/tests/e2e/run_celery.py | 48 ++++++++++++ surfsense_web/playwright.config.ts | 2 +- .../documents/file-upload/journey.spec.ts | 4 +- 7 files changed, 170 insertions(+), 9 deletions(-) create mode 100644 surfsense_backend/tests/e2e/fixtures/global_llm_config.yaml diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index d2338f092..b87537dab 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -119,9 +119,10 @@ jobs: uses: actions/cache@v5 with: path: surfsense_web/.next/cache - key: nextjs-${{ runner.os }}-${{ hashFiles('surfsense_web/pnpm-lock.yaml') }}-${{ hashFiles('surfsense_web/**/*.{js,jsx,ts,tsx}') }} + key: nextjs-${{ runner.os }}-${{ hashFiles('surfsense_web/pnpm-lock.yaml') }}-${{ github.sha }} restore-keys: | nextjs-${{ runner.os }}-${{ hashFiles('surfsense_web/pnpm-lock.yaml') }}- + nextjs-${{ runner.os }}- # ─── Tests ───────────────────────────────────────────────────────── - name: Run Playwright tests diff --git a/surfsense_backend/.gitignore b/surfsense_backend/.gitignore index 1cd7fd32c..47fd53aef 100644 --- a/surfsense_backend/.gitignore +++ b/surfsense_backend/.gitignore @@ -13,5 +13,5 @@ celerybeat-schedule* celerybeat-schedule.* celerybeat-schedule.dir celerybeat-schedule.bak -global_llm_config.yaml +/app/config/global_llm_config.yaml app/templates/_generated/ \ No newline at end of file diff --git a/surfsense_backend/tests/e2e/fixtures/global_llm_config.yaml b/surfsense_backend/tests/e2e/fixtures/global_llm_config.yaml new file mode 100644 index 000000000..ef00ac0c4 --- /dev/null +++ b/surfsense_backend/tests/e2e/fixtures/global_llm_config.yaml @@ -0,0 +1,45 @@ +# Synthetic Global LLM configuration for E2E ONLY. +# +# Why this file exists: +# surfsense_backend/app/config/global_llm_config.yaml is gitignored +# (operators ship real API keys there). In CI that file does not exist, +# so app.config.load_global_llm_configs() returns [], every chat-stream +# test fails fast with "No usable global LLM configs are available for +# Auto mode" raised by auto_model_pin_service._global_candidates(). +# +# What this file does: +# tests/e2e/run_backend.py and tests/e2e/run_celery.py copy this file +# to app/config/global_llm_config.yaml at startup, BEFORE app.config +# is imported. The copy lives only inside the E2E Docker container. +# +# Why a fake api_key is safe: +# tests.e2e.fakes.chat_llm patches +# app.tasks.chat.stream_new_chat.create_chat_litellm_from_agent_config +# app.tasks.chat.stream_new_chat.create_chat_litellm_from_config +# so the resolved auto-pin id is never sent to a real LLM provider. +# The values below only need to pass +# auto_model_pin_service._is_usable_global_config() +# which requires id / model_name / provider / api_key all truthy. + +router_settings: + routing_strategy: "simple-shuffle" + num_retries: 0 + allowed_fails: 1 + cooldown_time: 1 + +global_llm_configs: + - id: 1001 + name: "E2E Fake Auto Model" + billing_tier: "free" + anonymous_enabled: false + seo_enabled: false + quality_score: 1.0 + provider: "OPENAI" + model_name: "fake-e2e-model" + api_key: "fake-e2e-api-key-not-for-production" + supports_image_input: false + quota_reserve_tokens: 1024 + rpm: 1000 + tpm: 100000 + litellm_params: + model: "openai/fake-e2e-model" diff --git a/surfsense_backend/tests/e2e/run_backend.py b/surfsense_backend/tests/e2e/run_backend.py index 7419173a7..d0c734751 100644 --- a/surfsense_backend/tests/e2e/run_backend.py +++ b/surfsense_backend/tests/e2e/run_backend.py @@ -120,10 +120,74 @@ def _load_dotenv_and_set_env_defaults() -> None: "DROPBOX_REDIRECT_URI", "http://localhost:8000/api/v1/auth/dropbox/connector/callback", ) + # Native Google OAuth — fake Flow in tests.e2e.fakes.native_google + # raises "Fake Google Flow requires redirect_uri." if these are empty, + # so connector/add routes return 500 in CI where no .env supplies them. + os.environ.setdefault( + "GOOGLE_DRIVE_REDIRECT_URI", + "http://localhost:8000/api/v1/auth/google/drive/connector/callback", + ) + os.environ.setdefault( + "GOOGLE_GMAIL_REDIRECT_URI", + "http://localhost:8000/api/v1/auth/google/gmail/connector/callback", + ) + os.environ.setdefault( + "GOOGLE_CALENDAR_REDIRECT_URI", + "http://localhost:8000/api/v1/auth/google/calendar/connector/callback", + ) os.environ["SLACK_CLIENT_ID"] = "fake-slack-mcp-client-id" os.environ["SLACK_CLIENT_SECRET"] = "fake-slack-mcp-client-secret" +def _install_synthetic_global_llm_config() -> None: + """Materialise a fake ``app/config/global_llm_config.yaml`` for E2E. + + The real file is gitignored (production operators ship their own with + real API keys), so a fresh CI checkout has no YAML at the path + ``app.config.load_global_llm_configs()`` reads. With an empty + ``GLOBAL_LLM_CONFIGS`` list, ``auto_model_pin_service`` raises + ``"No usable global LLM configs are available for Auto mode"`` on + every chat-stream request. + + We copy the synthetic fixture from ``tests/e2e/fixtures/`` into the + production-expected location BEFORE ``_import_production_app()`` so + ``app.config`` picks it up on import. Production code is untouched — + this is purely a test-time scaffold. + + Only installs when the destination is missing. A developer running + the E2E entrypoint locally keeps their real ``global_llm_config.yaml`` + intact (the patched ``create_chat_litellm_from_*`` factories make the + actual model values irrelevant either way). + + MUST run before _import_production_app(). + """ + import shutil + + src = os.path.join(_THIS_DIR, "fixtures", "global_llm_config.yaml") + dst = os.path.join( + _BACKEND_ROOT, "app", "config", "global_llm_config.yaml" + ) + + if not os.path.exists(src): + raise RuntimeError( + f"E2E synthetic global LLM config fixture missing at {src!r}. " + f"This file is checked into tests/e2e/fixtures/ — if it has gone " + f"missing, restore it from VCS before running the E2E entrypoint." + ) + + if os.path.exists(dst): + logger.info( + "[e2e-global-llm-config] %s already exists; leaving it alone " + "(local dev config preserved)", + dst, + ) + return + + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copyfile(src, dst) + logger.info("[e2e-global-llm-config] installed %s -> %s", src, dst) + + def _import_production_app(): """Import and return the production FastAPI app. @@ -259,10 +323,12 @@ def _bootstrap(): 1) Hijack composio + notion_client in sys.modules. 2) Load .env + set env defaults (app.config reads env on import). 3) Configure logging. - 4) Import production app (which transitively imports the now-faked - external SDKs and reads the env defaults). - 5) Patch LLM / embedding bindings at every consumer site. - 6) Mount test-only middleware + /__e2e__ routes onto the app. + 4) Materialise the synthetic global_llm_config.yaml so Auto-mode + pin resolution finds at least one usable candidate. + 5) Import production app (which transitively imports the now-faked + external SDKs and reads the env defaults + YAML). + 6) Patch LLM / embedding bindings at every consumer site. + 7) Mount test-only middleware + /__e2e__ routes onto the app. """ _hijack_external_sdks() _load_dotenv_and_set_env_defaults() @@ -276,6 +342,7 @@ def _bootstrap(): "*** SURFSENSE E2E BACKEND ENTRYPOINT — fake Composio + LLM + embeddings ***" ) + _install_synthetic_global_llm_config() production_app = _import_production_app() _patch_llm_bindings() _install_runtime_fakes() diff --git a/surfsense_backend/tests/e2e/run_celery.py b/surfsense_backend/tests/e2e/run_celery.py index 3b7c75bb1..56480a295 100644 --- a/surfsense_backend/tests/e2e/run_celery.py +++ b/surfsense_backend/tests/e2e/run_celery.py @@ -91,6 +91,20 @@ os.environ.setdefault( "DROPBOX_REDIRECT_URI", "http://localhost:8000/api/v1/auth/dropbox/connector/callback", ) +# Native Google OAuth — fake Flow in tests.e2e.fakes.native_google raises +# "Fake Google Flow requires redirect_uri." when these are empty. +os.environ.setdefault( + "GOOGLE_DRIVE_REDIRECT_URI", + "http://localhost:8000/api/v1/auth/google/drive/connector/callback", +) +os.environ.setdefault( + "GOOGLE_GMAIL_REDIRECT_URI", + "http://localhost:8000/api/v1/auth/google/gmail/connector/callback", +) +os.environ.setdefault( + "GOOGLE_CALENDAR_REDIRECT_URI", + "http://localhost:8000/api/v1/auth/google/calendar/connector/callback", +) os.environ["SLACK_CLIENT_ID"] = "fake-slack-mcp-client-id" os.environ["SLACK_CLIENT_SECRET"] = "fake-slack-mcp-client-secret" @@ -103,6 +117,40 @@ logger = logging.getLogger("surfsense.e2e.celery") logger.warning("*** SURFSENSE E2E CELERY WORKER — fake Composio + LLM + embeddings ***") +# --------------------------------------------------------------------------- +# 2.5) Materialise the synthetic global_llm_config.yaml so the worker's +# view of app.config.GLOBAL_LLM_CONFIGS matches the API container. +# Must run BEFORE the production celery_app import below, which +# transitively imports app.config. Install-only-if-missing so a +# developer's local config (with real API keys) is preserved. +# --------------------------------------------------------------------------- +import shutil as _shutil # noqa: E402 + +_e2e_llm_cfg_src = os.path.join(_THIS_DIR, "fixtures", "global_llm_config.yaml") +_e2e_llm_cfg_dst = os.path.join( + _BACKEND_ROOT, "app", "config", "global_llm_config.yaml" +) +if not os.path.exists(_e2e_llm_cfg_src): + raise RuntimeError( + f"E2E synthetic global LLM config fixture missing at {_e2e_llm_cfg_src!r}. " + f"Restore tests/e2e/fixtures/global_llm_config.yaml from VCS." + ) +if os.path.exists(_e2e_llm_cfg_dst): + logger.info( + "[e2e-global-llm-config] %s already exists; leaving it alone " + "(local dev config preserved)", + _e2e_llm_cfg_dst, + ) +else: + os.makedirs(os.path.dirname(_e2e_llm_cfg_dst), exist_ok=True) + _shutil.copyfile(_e2e_llm_cfg_src, _e2e_llm_cfg_dst) + logger.info( + "[e2e-global-llm-config] installed %s -> %s", + _e2e_llm_cfg_src, + _e2e_llm_cfg_dst, + ) + + # --------------------------------------------------------------------------- # 3) Import the production celery_app. All task modules load here. # --------------------------------------------------------------------------- diff --git a/surfsense_web/playwright.config.ts b/surfsense_web/playwright.config.ts index d645e978f..eb287635d 100644 --- a/surfsense_web/playwright.config.ts +++ b/surfsense_web/playwright.config.ts @@ -27,7 +27,7 @@ export default defineConfig({ expect: { timeout: 15_000 }, fullyParallel: true, forbidOnly: !!process.env.CI, - retries: process.env.CI ? 2 : 0, + retries: process.env.CI ? 1 : 0, workers: 1, reporter: process.env.CI ? [["html", { open: "never" }], ["github"], ["list"]] diff --git a/surfsense_web/tests/documents/file-upload/journey.spec.ts b/surfsense_web/tests/documents/file-upload/journey.spec.ts index 6ddfb522f..711963bf0 100644 --- a/surfsense_web/tests/documents/file-upload/journey.spec.ts +++ b/surfsense_web/tests/documents/file-upload/journey.spec.ts @@ -107,14 +107,14 @@ test.describe("Manual file upload journey", () => { }); }); - test("user uploads a PDF (DOCUMENT branch via real Docling)", async ({ + test("user uploads a PDF (DOCUMENT branch)", async ({ page, request, apiToken, searchSpace, chatThread, }) => { - test.setTimeout(240_000); // Docling cold-start can take 30-60s on first invocation. + test.setTimeout(180_000); await uploadAndAssert({ page,