From 31ec939da9a5d79c184e1f9ec7a64d0ff49d853f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 27 Jun 2026 03:13:50 +0530 Subject: [PATCH 001/217] fix(model_resolver): supply dummy API key for LM Studio when not provided --- surfsense_backend/app/services/model_resolver.py | 4 ++++ .../unit/services/test_model_connections.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/surfsense_backend/app/services/model_resolver.py b/surfsense_backend/app/services/model_resolver.py index f31b658a4..590290b0c 100644 --- a/surfsense_backend/app/services/model_resolver.py +++ b/surfsense_backend/app/services/model_resolver.py @@ -59,6 +59,10 @@ def to_litellm( kwargs: dict[str, Any] = {} if api_key: kwargs["api_key"] = api_key + elif provider == "lm_studio": + # LiteLLM's OpenAI-compatible adapter expects an api_key value even + # when LM Studio accepts unauthenticated local requests. + kwargs["api_key"] = "not-needed" prefix = spec.litellm_prefix or str(provider) model_string = f"{prefix}/{model_id}" if prefix else model_id diff --git a/surfsense_backend/tests/unit/services/test_model_connections.py b/surfsense_backend/tests/unit/services/test_model_connections.py index b4e7c18d7..efc056465 100644 --- a/surfsense_backend/tests/unit/services/test_model_connections.py +++ b/surfsense_backend/tests/unit/services/test_model_connections.py @@ -64,6 +64,22 @@ def test_openai_compatible_resolver_uses_explicit_api_base() -> None: assert ensure_v1("http://example.com/v1") == "http://example.com/v1" +def test_lm_studio_resolver_supplies_dummy_api_key_when_empty() -> None: + model, kwargs = to_litellm( + { + "provider": "lm_studio", + "base_url": "http://host.docker.internal:1234/v1", + "api_key": None, + "extra": {}, + }, + "tinyllama-1.1b-chat-v0.6", + ) + + assert model == "openai/tinyllama-1.1b-chat-v0.6" + assert kwargs["api_base"] == "http://host.docker.internal:1234/v1" + assert kwargs["api_key"] == "not-needed" + + def test_ollama_resolver_uses_native_api_base() -> None: model, kwargs = to_litellm( { From 0df51c60f372f7348382d5631512bf58c1918f3f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 27 Jun 2026 03:46:15 +0530 Subject: [PATCH 002/217] refactor(models): remove manual model registration backend --- .../app/routes/model_connections_routes.py | 44 ------------------- surfsense_backend/app/schemas/__init__.py | 2 - .../app/schemas/model_connections.py | 11 ----- .../app/services/model_connection_service.py | 8 ---- 4 files changed, 65 deletions(-) diff --git a/surfsense_backend/app/routes/model_connections_routes.py b/surfsense_backend/app/routes/model_connections_routes.py index 84e9b830d..b7edfcc77 100644 --- a/surfsense_backend/app/routes/model_connections_routes.py +++ b/surfsense_backend/app/routes/model_connections_routes.py @@ -21,7 +21,6 @@ from app.schemas import ( ConnectionCreate, ConnectionRead, ConnectionUpdate, - ModelCreate, ModelPreviewRead, ModelProviderRead, ModelRead, @@ -618,49 +617,6 @@ async def discover_connection_models( return [_model_read(model) for model in conn.models] -@router.post("/model-connections/{connection_id}/models", response_model=ModelRead) -async def add_manual_model( - connection_id: int, - data: ModelCreate, - session: AsyncSession = Depends(get_async_session), - auth: AuthContext = Depends(get_auth_context), -): - conn = await _load_connection(session, connection_id) - await _assert_connection_access( - session, auth, conn, Permission.LLM_CONFIGS_UPDATE.value - ) - - model_id = data.model_id.strip() - if not model_id: - raise HTTPException(status_code=400, detail="model_id is required") - if any(existing.model_id == model_id for existing in conn.models): - raise HTTPException( - status_code=400, detail="Model already exists on this connection" - ) - - capabilities = derive_capabilities(conn, model_id) - model = Model( - connection_id=conn.id, - model_id=model_id, - display_name=data.display_name or None, - source=ModelSource.MANUAL, - capabilities_override={}, - enabled=True, - catalog={}, - ) - _apply_model_facts(model, capabilities) - session.add(model) - await session.commit() - await session.refresh(model) - conn = await _load_connection(session, connection_id) - await _default_unset_roles(session, conn, list(conn.models)) - if conn.search_space_id is not None: - await _clear_invalid_roles(session, conn.search_space_id) - await session.commit() - await session.refresh(model) - return _model_read(model) - - @router.patch( "/model-connections/{connection_id}/models", response_model=list[ModelRead] ) diff --git a/surfsense_backend/app/schemas/__init__.py b/surfsense_backend/app/schemas/__init__.py index f111f0226..5f5252fbd 100644 --- a/surfsense_backend/app/schemas/__init__.py +++ b/surfsense_backend/app/schemas/__init__.py @@ -43,7 +43,6 @@ from .model_connections import ( ConnectionCreate, ConnectionRead, ConnectionUpdate, - ModelCreate, ModelPreviewRead, ModelProviderRead, ModelRead, @@ -203,7 +202,6 @@ __all__ = [ "MembershipRead", "MembershipReadWithUser", "MembershipUpdate", - "ModelCreate", "ModelPreviewRead", "ModelProviderRead", "ModelRead", diff --git a/surfsense_backend/app/schemas/model_connections.py b/surfsense_backend/app/schemas/model_connections.py index 0eec666c1..0f656dad3 100644 --- a/surfsense_backend/app/schemas/model_connections.py +++ b/surfsense_backend/app/schemas/model_connections.py @@ -93,17 +93,6 @@ class ConnectionUpdate(BaseModel): enabled: bool | None = None -class ModelCreate(BaseModel): - """Manually register a model id on a connection. - - For providers without a usable ``/models`` endpoint (Perplexity, MiniMax, - Azure deployments, etc.) or to pin a single model from a noisy provider. - """ - - model_id: str = Field(..., max_length=255) - display_name: str | None = Field(None, max_length=255) - - class ModelUpdate(BaseModel): display_name: str | None = Field(None, max_length=255) enabled: bool | None = None diff --git a/surfsense_backend/app/services/model_connection_service.py b/surfsense_backend/app/services/model_connection_service.py index cdfd1d725..be442bcb2 100644 --- a/surfsense_backend/app/services/model_connection_service.py +++ b/surfsense_backend/app/services/model_connection_service.py @@ -202,11 +202,6 @@ def _discovery_error_message(conn: Connection, exc: httpx.HTTPError) -> str: return _docker_hint(base_url, exc) -def _allowlist(conn: Connection) -> set[str]: - raw = (conn.extra or {}).get("model_ids") or [] - return {str(item).strip() for item in raw if str(item).strip()} - - def _litellm_info(model_string: str, model_id: str) -> dict[str, Any]: with contextlib.suppress(Exception): info = litellm.get_model_info(model=model_string) @@ -438,7 +433,6 @@ async def _discover_bedrock_models(conn: Connection) -> list[dict[str, Any]]: async def discover_models(conn: Connection) -> list[dict[str, Any]]: - allowlist = _allowlist(conn) spec = spec_for(conn.provider) try: @@ -459,8 +453,6 @@ async def discover_models(conn: Connection) -> list[dict[str, Any]]: except httpx.HTTPError as exc: raise ModelDiscoveryError(_discovery_error_message(conn, exc)) from exc - if allowlist: - results = [item for item in results if item["model_id"] in allowlist] return results From 3b507208bccf2ab1b149943059b9c51ace95c5d1 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 27 Jun 2026 03:46:24 +0530 Subject: [PATCH 003/217] refactor(models): remove manual model creation client API --- .../model-connections-mutation.atoms.ts | 23 +------------------ .../types/model-connections.types.ts | 6 ----- .../lib/apis/model-connections-api.service.ts | 15 ------------ 3 files changed, 1 insertion(+), 43 deletions(-) diff --git a/surfsense_web/atoms/model-connections/model-connections-mutation.atoms.ts b/surfsense_web/atoms/model-connections/model-connections-mutation.atoms.ts index f00bf76f9..e9ff559c0 100644 --- a/surfsense_web/atoms/model-connections/model-connections-mutation.atoms.ts +++ b/surfsense_web/atoms/model-connections/model-connections-mutation.atoms.ts @@ -4,7 +4,6 @@ import type { ConnectionCreateRequest, ConnectionRead, ConnectionUpdateRequest, - ModelCreateRequest, ModelPreviewRead, ModelRead, ModelRoles, @@ -95,13 +94,7 @@ export const verifyModelConnectionMutationAtom = atomWithMutation((get) => { if (result.ok) { toast.success("Connection verified"); } else { - // Non-fatal: many providers lack a /models endpoint yet still serve - // chat. Guide the user to add model IDs manually instead of alarming. - toast.warning( - result.message - ? `${result.message} Chat may still work — add model IDs manually.` - : "Couldn't list models. Chat may still work — add model IDs manually." - ); + toast.warning(result.message || "Couldn't verify this connection."); } invalidateModelConnections(searchSpaceId); }, @@ -148,20 +141,6 @@ export const testPreviewModelMutationAtom = atomWithMutation(() => { }; }); -export const addManualModelMutationAtom = atomWithMutation((get) => { - const searchSpaceId = Number(get(activeSearchSpaceIdAtom)); - return { - mutationKey: ["models", "add-manual"], - mutationFn: ({ connectionId, data }: { connectionId: number; data: ModelCreateRequest }) => - modelConnectionsApiService.addManualModel(connectionId, data), - onSuccess: () => { - toast.success("Model added"); - invalidateModelConnections(searchSpaceId); - }, - onError: (error: Error) => toast.error(error.message || "Failed to add model"), - }; -}); - export const updateModelMutationAtom = atomWithMutation((get) => { const searchSpaceId = Number(get(activeSearchSpaceIdAtom)); return { diff --git a/surfsense_web/contracts/types/model-connections.types.ts b/surfsense_web/contracts/types/model-connections.types.ts index 0f0c7591e..73dff23eb 100644 --- a/surfsense_web/contracts/types/model-connections.types.ts +++ b/surfsense_web/contracts/types/model-connections.types.ts @@ -74,11 +74,6 @@ export const connectionUpdateRequest = z.object({ enabled: z.boolean().optional(), }); -export const modelCreateRequest = z.object({ - model_id: z.string().min(1), - display_name: z.string().nullable().optional(), -}); - export const modelUpdateRequest = z.object({ display_name: z.string().nullable().optional(), enabled: z.boolean().optional(), @@ -135,7 +130,6 @@ export type ConnectionRead = z.infer; export type ConnectionCreateRequest = z.infer; export type ModelTestPreviewRequest = z.infer; export type ConnectionUpdateRequest = z.infer; -export type ModelCreateRequest = z.infer; export type ModelUpdateRequest = z.infer; export type ModelsBulkUpdateRequest = z.infer; export type ModelRoles = z.infer; diff --git a/surfsense_web/lib/apis/model-connections-api.service.ts b/surfsense_web/lib/apis/model-connections-api.service.ts index c69bcbef2..6083642f5 100644 --- a/surfsense_web/lib/apis/model-connections-api.service.ts +++ b/surfsense_web/lib/apis/model-connections-api.service.ts @@ -8,7 +8,6 @@ import { connectionUpdateRequest, type GlobalLlmConfigStatus, globalLlmConfigStatus, - type ModelCreateRequest, type ModelPreviewRead, type ModelProviderRead, type ModelRead, @@ -16,7 +15,6 @@ import { type ModelsBulkUpdateRequest, type ModelTestPreviewRequest, type ModelUpdateRequest, - modelCreateRequest, modelListResponse, modelPreviewListResponse, modelProviderListResponse, @@ -112,19 +110,6 @@ class ModelConnectionsApiService { }); }; - addManualModel = async ( - connectionId: number, - request: ModelCreateRequest - ): Promise => { - const parsed = modelCreateRequest.safeParse(request); - if (!parsed.success) { - throw new ValidationError(parsed.error.issues.map((issue) => issue.message).join(", ")); - } - return baseApiService.post(`/api/v1/model-connections/${connectionId}/models`, modelRead, { - body: parsed.data, - }); - }; - updateModel = async (id: number, request: ModelUpdateRequest): Promise => { const parsed = modelUpdateRequest.safeParse(request); if (!parsed.success) { From 13a2637df7038213f3a4df13aec136758466cd8d Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 27 Jun 2026 03:46:36 +0530 Subject: [PATCH 004/217] refactor(models): simplify provider model selection UI --- .../bedrock-connect-form.tsx | 2 +- .../connection-settings-dialog.tsx | 48 ------------------- .../model-provider-connections-panel.tsx | 17 ------- .../models-selection-panel.tsx | 48 ------------------- .../provider-connect-dialog.tsx | 27 ++++++----- .../model-connections/vertex-connect-form.tsx | 2 +- 6 files changed, 16 insertions(+), 128 deletions(-) diff --git a/surfsense_web/components/settings/model-connections/bedrock-connect-form.tsx b/surfsense_web/components/settings/model-connections/bedrock-connect-form.tsx index f76308421..0da545bd8 100644 --- a/surfsense_web/components/settings/model-connections/bedrock-connect-form.tsx +++ b/surfsense_web/components/settings/model-connections/bedrock-connect-form.tsx @@ -113,7 +113,7 @@ export function BedrockConnectForm({ onDraftChange }: ProviderConnectFormProps)

) : null}

- Add Bedrock model IDs from the provider's settings after connecting. + After entering credentials, refresh models to discover the Bedrock catalog for this region.

); diff --git a/surfsense_web/components/settings/model-connections/connection-settings-dialog.tsx b/surfsense_web/components/settings/model-connections/connection-settings-dialog.tsx index 1f16c3bd0..0120e28ed 100644 --- a/surfsense_web/components/settings/model-connections/connection-settings-dialog.tsx +++ b/surfsense_web/components/settings/model-connections/connection-settings-dialog.tsx @@ -2,7 +2,6 @@ import { useAtomValue } from "jotai"; import { Eye, EyeOff, Settings } from "lucide-react"; import { useMemo, useState } from "react"; import { - addManualModelMutationAtom, bulkUpdateModelsMutationAtom, discoverConnectionModelsMutationAtom, testPreviewModelMutationAtom, @@ -50,26 +49,17 @@ export function ConnectionSettingsDialog({ const discoverModels = useAtomValue(discoverConnectionModelsMutationAtom); const testPreviewModel = useAtomValue(testPreviewModelMutationAtom); const updateConnection = useAtomValue(updateModelConnectionMutationAtom); - const addManualModel = useAtomValue(addManualModelMutationAtom); const bulkUpdateModels = useAtomValue(bulkUpdateModelsMutationAtom); - const allowlist = Array.isArray(connection.extra?.model_ids) - ? (connection.extra.model_ids as string[]) - : []; const [isOpen, setIsOpen] = useState(false); const [baseUrlDraft, setBaseUrlDraft] = useState(connection.base_url ?? ""); const [apiKeyDraft, setApiKeyDraft] = useState(""); const [showApiKey, setShowApiKey] = useState(false); - const [allowlistText, setAllowlistText] = useState(allowlist.join(", ")); const [isSavingConnectionSettings, setIsSavingConnectionSettings] = useState(false); const [draftEnabledModelIds, setDraftEnabledModelIds] = useState(() => enabledModelIds(connection.models) ); - const isLocal = - connection.provider === "ollama_chat" || - connection.provider === "lm_studio" || - !connection.base_url?.startsWith("https"); const hasConnectionChanges = baseUrlDraft.trim() !== (connection.base_url ?? "") || apiKeyDraft.trim() !== (connection.api_key ?? ""); @@ -93,7 +83,6 @@ export function ConnectionSettingsDialog({ setBaseUrlDraft(connection.base_url ?? ""); setApiKeyDraft(connection.api_key ?? ""); setShowApiKey(false); - setAllowlistText(allowlist.join(", ")); setIsSavingConnectionSettings(false); setDraftEnabledModelIds(enabledModelIds(connection.models)); } @@ -168,17 +157,6 @@ export function ConnectionSettingsDialog({ } } - function saveAllowlist() { - const ids = allowlistText - .split(",") - .map((value) => value.trim()) - .filter(Boolean); - updateConnection.mutate({ - id: connection.id, - data: { extra: { ...(connection.extra ?? {}), model_ids: ids } }, - }); - } - function handleToggleModel(model: SelectableModel, enabled: boolean) { if (typeof model.id !== "number") return; const modelId = model.id; @@ -276,41 +254,15 @@ export function ConnectionSettingsDialog({ - {!isLocal ? ( -
- -
- setAllowlistText(event.target.value)} - placeholder="Comma-separated, e.g. anthropic/claude-sonnet-4-5, google/gemini-2.5-pro" - /> - -
-

- Leave empty to discover all models. Recommended for providers with large catalogs. -

-
- ) : null} - discoverModels.mutate(connection.id)} - onAddManual={(modelId) => - addManualModel.mutate({ - connectionId: connection.id, - data: { model_id: modelId }, - }) - } onToggleModel={handleToggleModel} onBulkToggle={handleBulkToggle} /> diff --git a/surfsense_web/components/settings/model-connections/model-provider-connections-panel.tsx b/surfsense_web/components/settings/model-connections/model-provider-connections-panel.tsx index a703ab1c8..13fd39894 100644 --- a/surfsense_web/components/settings/model-connections/model-provider-connections-panel.tsx +++ b/surfsense_web/components/settings/model-connections/model-provider-connections-panel.tsx @@ -201,22 +201,6 @@ export function ModelProviderConnectionsPanel({ ); } - function addConnectModel(modelId: string) { - setConnectModels((current) => { - if (current.some((model) => model.model_id === modelId)) return current; - return [ - ...current, - { - model_id: modelId, - display_name: modelId, - source: "MANUAL", - enabled: true, - metadata: {}, - }, - ]; - }); - } - function toggleConnectModel(model: SelectableModel, enabled: boolean) { setConnectModels((current) => current.map((item) => (item.model_id === model.model_id ? { ...item, enabled } : item)) @@ -277,7 +261,6 @@ export function ModelProviderConnectionsPanel({ previewModels={connectModels} isPreviewingModels={previewModels.isPending} onPreviewModels={refreshConnectModels} - onAddPreviewModel={addConnectModel} onTogglePreviewModel={toggleConnectModel} onBulkTogglePreviewModels={bulkToggleConnectModels} /> diff --git a/surfsense_web/components/settings/model-connections/models-selection-panel.tsx b/surfsense_web/components/settings/model-connections/models-selection-panel.tsx index 3c6990afb..6aa0c2661 100644 --- a/surfsense_web/components/settings/model-connections/models-selection-panel.tsx +++ b/surfsense_web/components/settings/model-connections/models-selection-panel.tsx @@ -1,10 +1,7 @@ import { RefreshCw } from "lucide-react"; import { useState } from "react"; -import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; -import { Input } from "@/components/ui/input"; -import { Spinner } from "@/components/ui/spinner"; import { capability, capabilityLabels, @@ -18,14 +15,11 @@ interface ModelsSelectionPanelProps { models: SelectableModel[]; description?: string; emptyMessage?: string; - manualInputPlaceholder?: string; refreshLabel?: string; isRefreshing?: boolean; - isAddingManual?: boolean; isUpdatingModel?: boolean; isBulkUpdating?: boolean; onRefresh?: () => void; - onAddManual?: (modelId: string) => void; onToggleModel?: (model: SelectableModel, enabled: boolean) => void; onBulkToggle?: (models: SelectableModel[], enabled: boolean) => void; } @@ -34,18 +28,14 @@ export function ModelsSelectionPanel({ models, description = "Select models to make available for this provider.", emptyMessage = "No models available.", - manualInputPlaceholder = "Add a model ID manually", refreshLabel = "Refresh models", isRefreshing = false, - isAddingManual = false, isUpdatingModel = false, isBulkUpdating = false, onRefresh, - onAddManual, onToggleModel, onBulkToggle, }: ModelsSelectionPanelProps) { - const [manualModelId, setManualModelId] = useState(""); const [modelFilter, setModelFilter] = useState(null); const filteredModels = modelFilter @@ -54,13 +44,6 @@ export function ModelsSelectionPanel({ const allFilteredModelsEnabled = filteredModels.length > 0 && filteredModels.every((model) => model.enabled); - function addModel() { - const modelId = manualModelId.trim(); - if (!modelId || !onAddManual) return; - onAddManual(modelId); - setManualModelId(""); - } - function toggleFilteredModels() { const nextEnabled = !allFilteredModelsEnabled; const changedModels = filteredModels.filter((model) => model.enabled !== nextEnabled); @@ -100,32 +83,6 @@ export function ModelsSelectionPanel({ - {onAddManual ? ( -
- setManualModelId(event.target.value)} - onKeyDown={(event) => { - if (event.key === "Enter") { - event.preventDefault(); - addModel(); - } - }} - placeholder={manualInputPlaceholder} - /> - -
- ) : null} - {models.length > 0 ? (
Filter models @@ -179,11 +136,6 @@ export function ModelsSelectionPanel({
{modelLabel(model)} - {model.source === "MANUAL" ? ( - - manual - - ) : null}
{capabilityLabels(model) || "No discovered capabilities"} diff --git a/surfsense_web/components/settings/model-connections/provider-connect-dialog.tsx b/surfsense_web/components/settings/model-connections/provider-connect-dialog.tsx index 2eee2cf8c..0b795064d 100644 --- a/surfsense_web/components/settings/model-connections/provider-connect-dialog.tsx +++ b/surfsense_web/components/settings/model-connections/provider-connect-dialog.tsx @@ -33,7 +33,6 @@ interface ProviderConnectDialogProps { previewModels?: SelectableModel[]; isPreviewingModels?: boolean; onPreviewModels?: (draft: ConnectionDraft) => void; - onAddPreviewModel?: (modelId: string) => void; onTogglePreviewModel?: (model: SelectableModel, enabled: boolean) => void; onBulkTogglePreviewModels?: (models: SelectableModel[], enabled: boolean) => void; } @@ -53,7 +52,6 @@ export function ProviderConnectDialog({ previewModels = [], isPreviewingModels = false, onPreviewModels, - onAddPreviewModel, onTogglePreviewModel, onBulkTogglePreviewModels, }: ProviderConnectDialogProps) { @@ -130,18 +128,21 @@ export function ProviderConnectDialog({ )} - + {!isAzure ? ( + <> + - onPreviewModels?.(currentDraft) : undefined} - onAddManual={onAddPreviewModel} - onToggleModel={onTogglePreviewModel} - onBulkToggle={onBulkTogglePreviewModels} - /> + onPreviewModels?.(currentDraft) : undefined} + onToggleModel={onTogglePreviewModel} + onBulkToggle={onBulkTogglePreviewModels} + /> + + ) : null}
onOpenChange(false)} diff --git a/surfsense_web/components/settings/model-connections/vertex-connect-form.tsx b/surfsense_web/components/settings/model-connections/vertex-connect-form.tsx index 1027742bc..3701ea14e 100644 --- a/surfsense_web/components/settings/model-connections/vertex-connect-form.tsx +++ b/surfsense_web/components/settings/model-connections/vertex-connect-form.tsx @@ -111,7 +111,7 @@ export function VertexConnectForm({ onDraftChange }: ProviderConnectFormProps) {
)}

- Add Vertex AI model IDs from the provider's settings after connecting. + SurfSense will show supported Vertex AI models from the provider catalog.

); From 4ceb5827f14f6f3358924145779fb1383e5935a4 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:27:48 +0530 Subject: [PATCH 005/217] feat(models): add refresh control enhancements to model selection and provider connect dialogs - Introduced `isRefreshDisabled` prop to `ModelsSelectionPanel` to control refresh button state. - Updated `ProviderConnectDialog` to conditionally enable refresh functionality based on model support and submission status. --- .../settings/model-connections/models-selection-panel.tsx | 4 +++- .../settings/model-connections/provider-connect-dialog.tsx | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/surfsense_web/components/settings/model-connections/models-selection-panel.tsx b/surfsense_web/components/settings/model-connections/models-selection-panel.tsx index 6aa0c2661..b6258b7b2 100644 --- a/surfsense_web/components/settings/model-connections/models-selection-panel.tsx +++ b/surfsense_web/components/settings/model-connections/models-selection-panel.tsx @@ -17,6 +17,7 @@ interface ModelsSelectionPanelProps { emptyMessage?: string; refreshLabel?: string; isRefreshing?: boolean; + isRefreshDisabled?: boolean; isUpdatingModel?: boolean; isBulkUpdating?: boolean; onRefresh?: () => void; @@ -30,6 +31,7 @@ export function ModelsSelectionPanel({ emptyMessage = "No models available.", refreshLabel = "Refresh models", isRefreshing = false, + isRefreshDisabled = false, isUpdatingModel = false, isBulkUpdating = false, onRefresh, @@ -74,7 +76,7 @@ export function ModelsSelectionPanel({ size="icon" type="button" onClick={onRefresh} - disabled={isRefreshing} + disabled={isRefreshing || isRefreshDisabled} aria-label={refreshLabel} > diff --git a/surfsense_web/components/settings/model-connections/provider-connect-dialog.tsx b/surfsense_web/components/settings/model-connections/provider-connect-dialog.tsx index 0b795064d..2fba3bcbf 100644 --- a/surfsense_web/components/settings/model-connections/provider-connect-dialog.tsx +++ b/surfsense_web/components/settings/model-connections/provider-connect-dialog.tsx @@ -92,7 +92,8 @@ export function ProviderConnectDialog({ return "Select the models to enable for this provider"; })(); - const canRefreshModels = !isAzure && !isVertex && (!isBedrock || canSubmit); + const supportsModelRefresh = !isAzure && !isVertex; + const canRefreshModels = supportsModelRefresh && canSubmit; const hasEnabledModel = previewModels.some((model) => model.enabled) || Boolean(currentDraft.seedModelId); const canConnect = canSubmit && hasEnabledModel; @@ -136,8 +137,9 @@ export function ProviderConnectDialog({ models={previewModels} description={modelDescription} isRefreshing={isPreviewingModels} + isRefreshDisabled={!canRefreshModels} refreshLabel={`Refresh ${meta.name} models`} - onRefresh={canRefreshModels ? () => onPreviewModels?.(currentDraft) : undefined} + onRefresh={supportsModelRefresh ? () => onPreviewModels?.(currentDraft) : undefined} onToggleModel={onTogglePreviewModel} onBulkToggle={onBulkTogglePreviewModels} /> From 249bba08868b81d3e7c0b2796a47f2c10c89e86e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 006/217] feat: indeed scraper input/output schemas --- .../platforms/indeed_jobs/schemas.py | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/schemas.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/schemas.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/schemas.py new file mode 100644 index 000000000..27be45d9a --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/schemas.py @@ -0,0 +1,122 @@ +# ruff: noqa: N815 - field names intentionally use the public camelCase API +"""Input/output models for the Indeed scraper. + +Anonymous scraper: there is no auth field. Fields absent from a listing (full +description, benefits) stay ``None``/``[]`` until a detail fetch fills them. +""" + +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +IndeedSort = Literal["relevance", "date"] +IndeedJobType = Literal[ + "fulltime", + "parttime", + "contract", + "internship", + "temporary", + "permanent", + "seasonal", + "freelance", +] +IndeedLevel = Literal["entry_level", "mid_level", "senior_level"] +IndeedRemote = Literal["remote", "hybrid"] +SalaryPeriod = Literal["hour", "day", "week", "month", "year"] + + +class StartUrl(BaseModel): + """A direct URL entry; extra keys ignored.""" + + model_config = ConfigDict(extra="allow") + + url: str + + +class IndeedScrapeInput(BaseModel): + """Indeed scraper input. Caps are collector policy, enforced by ``scrape_indeed``.""" + + model_config = ConfigDict(extra="allow") + + # Discovery: direct URLs and/or search queries. + startUrls: list[StartUrl] = Field(default_factory=list) + queries: list[str] = Field(default_factory=list) + + # Search parameters applied to ``queries``. + country: str = "us" + location: str | None = None + radius: int | None = None + jobType: IndeedJobType | None = None + level: IndeedLevel | None = None + remote: IndeedRemote | None = None + fromDays: int | None = None + sort: IndeedSort = "relevance" + + # Fetch each job's detail page for the full description. + scrapeJobDetails: bool = False + + maxItems: int = Field(default=25, ge=0) + maxItemsPerQuery: int = Field(default=25, ge=0) + + +class Salary(BaseModel): + """Salary block; fields are ``None`` when Indeed omits pay.""" + + model_config = ConfigDict(extra="allow") + + salaryText: str | None = None + salaryMin: float | None = None + salaryMax: float | None = None + currency: str | None = None + period: SalaryPeriod | None = None + isEstimated: bool | None = None + + +class IndeedItem(BaseModel): + """One job posting. ``extra="allow"`` keeps the contract additive.""" + + model_config = ConfigDict(extra="allow") + + jobKey: str | None = None + title: str | None = None + jobUrl: str | None = None + applyUrl: str | None = None + + company: str | None = None + companyUrl: str | None = None + companyRating: float | None = None + companyReviewCount: int | None = None + + formattedLocation: str | None = None + city: str | None = None + state: str | None = None + postalCode: str | None = None + country: str | None = None + isRemote: bool | None = None + remoteType: str | None = None + + jobTypes: list[str] = Field(default_factory=list) + salary: Salary = Field(default_factory=Salary) + benefits: list[str] = Field(default_factory=list) + + descriptionText: str | None = None + descriptionHtml: str | None = None + + sponsored: bool | None = None + isNew: bool | None = None + urgentlyHiring: bool | None = None + expired: bool | None = None + indeedApplyEnabled: bool | None = None + + age: str | None = None + datePublished: str | None = None + createdAt: str | None = None + scrapedAt: str | None = None + + source: str = "indeed" + + def to_output(self) -> dict[str, Any]: + """Serialize to the flat output dict, keeping extras.""" + return self.model_dump(exclude_none=False) From 01c3f097e6bfc53fa0eb961f7555566052f2ca26 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 007/217] feat: indeed jobcards blob extraction and job parser --- .../platforms/indeed_jobs/parsers.py | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py new file mode 100644 index 000000000..ab2b97cf9 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py @@ -0,0 +1,228 @@ +"""Pure HTML/JSON -> item mapping for the Indeed scraper. + +I/O-free and deterministic so it can be unit-tested against captured fixtures; +the orchestrator stamps ``scrapedAt``. + +Indeed embeds job data as a JS assignment:: + + window.mosaic.providerData["mosaic-provider-jobcards"]={"metaData":{...}}; + +The literal recurs hundreds of times in the page, so :func:`extract_jobcards_blob` +anchors on the assignment (``...]=``) and brace-matches the balanced object. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from html import unescape +from re import sub as _re_sub +from typing import Any + +_DEFAULT_BASE = "https://www.indeed.com" + +_JOBCARDS_ANCHOR = 'window.mosaic.providerData["mosaic-provider-jobcards"]=' + +# Indeed's extractedSalary.type -> our SalaryPeriod. +_SALARY_PERIODS = { + "HOURLY": "hour", + "DAILY": "day", + "WEEKLY": "week", + "MONTHLY": "month", + "YEARLY": "year", +} + + +def _brace_match(text: str, start: int) -> str | None: + """Return the balanced ``{...}``/``[...]`` blob at ``text[start]``, quote-aware.""" + open_ch = text[start] if start < len(text) else "" + close_ch = {"[": "]", "{": "}"}.get(open_ch) + if close_ch is None: + return None + depth = 0 + i = start + n = len(text) + while i < n: + ch = text[i] + if ch == open_ch: + depth += 1 + elif ch == close_ch: + depth -= 1 + if depth == 0: + return text[start : i + 1] + elif ch == '"': + i += 1 + while i < n and text[i] != '"': + if text[i] == "\\": + i += 1 + i += 1 + i += 1 + return None + + +def extract_jobcards_blob(html: str) -> dict | None: + """Decode the ``mosaic-provider-jobcards`` assignment, or ``None`` if absent.""" + import json + + idx = html.find(_JOBCARDS_ANCHOR) + if idx == -1: + return None + brace = html.find("{", idx + len(_JOBCARDS_ANCHOR)) + if brace == -1: + return None + blob = _brace_match(html, brace) + if not blob: + return None + try: + data = json.loads(blob) + except ValueError: + return None + return data if isinstance(data, dict) else None + + +def job_results(blob: dict | None) -> list[dict[str, Any]]: + """Return the raw job records from a decoded blob.""" + if not isinstance(blob, dict): + return [] + results = ( + blob.get("metaData", {}).get("mosaicProviderJobCardsModel", {}).get("results") + ) + if not isinstance(results, list): + return [] + return [r for r in results if isinstance(r, dict)] + + +def _utc_from_ms(value: Any) -> str | None: + """Epoch milliseconds -> millisecond ISO string.""" + if isinstance(value, bool) or not isinstance(value, int | float): + return None + dt = datetime.fromtimestamp(float(value) / 1000.0, tz=UTC) + return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def _int(value: Any) -> int | None: + """Coerce to int, dropping bools.""" + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + return None + + +def _abs_url(path: Any, base_url: str) -> str | None: + """Resolve an Indeed-relative path against ``base_url``; keep absolute URLs.""" + if not isinstance(path, str) or not path: + return None + if path.startswith("http"): + return path + return f"{base_url}{path if path.startswith('/') else '/' + path}" + + +def _clean_snippet(snippet: Any) -> str | None: + """Strip tags and decode entities into plain text.""" + if not isinstance(snippet, str) or not snippet: + return None + text = _re_sub(r"<[^>]+>", " ", snippet) + text = unescape(text) + return _re_sub(r"\s+", " ", text).strip() or None + + +def _taxonomy(raw: dict[str, Any]) -> dict[str, list[str]]: + """Flatten ``taxonomyAttributes`` into ``{group label: [attribute labels]}``.""" + out: dict[str, list[str]] = {} + for group in raw.get("taxonomyAttributes") or []: + if not isinstance(group, dict): + continue + label = group.get("label") + attrs = group.get("attributes") + if isinstance(label, str) and isinstance(attrs, list): + out[label] = [ + a["label"] + for a in attrs + if isinstance(a, dict) and isinstance(a.get("label"), str) + ] + return out + + +def _job_types(raw: dict[str, Any], taxo: dict[str, list[str]]) -> list[str]: + """Job types from ``jobTypes`` then the taxonomy, deduped and order-stable.""" + seen: dict[str, None] = {} + for jt in raw.get("jobTypes") or []: + if isinstance(jt, str): + seen.setdefault(jt, None) + for label in ("job-types", "job-types-cc"): + for jt in taxo.get(label, []): + seen.setdefault(jt, None) + return list(seen) + + +def _salary(raw: dict[str, Any]) -> dict[str, Any]: + """Flatten salary from ``salarySnippet`` (text) + ``extractedSalary`` (bounds).""" + snippet = raw.get("salarySnippet") or {} + extracted = raw.get("extractedSalary") or {} + estimated = raw.get("estimatedSalary") or {} + source = extracted or estimated + text = snippet.get("text") if isinstance(snippet, dict) else None + return { + "salaryText": text if isinstance(text, str) else None, + "salaryMin": source.get("min") if isinstance(source, dict) else None, + "salaryMax": source.get("max") if isinstance(source, dict) else None, + "currency": snippet.get("currency") if isinstance(snippet, dict) else None, + "period": _SALARY_PERIODS.get( + source.get("type") if isinstance(source, dict) else None + ), + "isEstimated": bool(estimated) and not extracted, + } + + +def _is_remote(raw: dict[str, Any], taxo: dict[str, list[str]]) -> bool: + """Resolve remote/hybrid across Indeed's several signals.""" + if raw.get("remoteLocation") is True: + return True + if isinstance(raw.get("remoteWorkModel"), dict): + return True + return bool(taxo.get("remote")) + + +def parse_job(raw: dict[str, Any], *, base_url: str = _DEFAULT_BASE) -> dict[str, Any]: + """Map one raw ``results[]`` record to a flat item dict. + + ``base_url`` is the country domain the record came from, so job and company + URLs resolve to the right host. + """ + taxo = _taxonomy(raw) + job_key = raw.get("jobkey") + remote_model = raw.get("remoteWorkModel") + return { + "jobKey": job_key if isinstance(job_key, str) else None, + "title": raw.get("displayTitle") or raw.get("title"), + "jobUrl": f"{base_url}/viewjob?jk={job_key}" if job_key else None, + "applyUrl": raw.get("thirdPartyApplyUrl") or None, + "company": raw.get("company") or raw.get("truncatedCompany"), + "companyUrl": _abs_url(raw.get("companyOverviewLink"), base_url), + "companyRating": raw.get("companyRating"), + "companyReviewCount": _int(raw.get("companyReviewCount")), + "formattedLocation": raw.get("formattedLocation"), + "city": raw.get("jobLocationCity"), + "state": raw.get("jobLocationState"), + "postalCode": raw.get("jobLocationPostal"), + "country": raw.get("country"), + "isRemote": _is_remote(raw, taxo), + "remoteType": remote_model.get("type") + if isinstance(remote_model, dict) + else None, + "jobTypes": _job_types(raw, taxo), + "salary": _salary(raw), + "benefits": taxo.get("benefits", []), + "descriptionText": _clean_snippet(raw.get("snippet")), + "descriptionHtml": None, + "sponsored": raw.get("sponsored"), + "isNew": raw.get("newJob"), + "urgentlyHiring": raw.get("urgentlyHiring"), + "expired": raw.get("expired"), + "indeedApplyEnabled": raw.get("indeedApplyEnabled"), + "age": raw.get("formattedRelativeTime"), + "datePublished": _utc_from_ms(raw.get("pubDate")), + "createdAt": _utc_from_ms(raw.get("createDate")), + } From d87c86dcc41bd4d8a4013d2ff954f31bc86b9663 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 008/217] feat: indeed url classifier and search url builder --- .../platforms/indeed_jobs/url_resolver.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/url_resolver.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/url_resolver.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/url_resolver.py new file mode 100644 index 000000000..c69d3fe19 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/url_resolver.py @@ -0,0 +1,124 @@ +"""Classify Indeed URLs and build search URLs. + +Recognizes search pages (``/jobs?q=&l=``), company pages (``/cmp//jobs``), +and single jobs (``/viewjob?jk=``); other hosts resolve to ``None``. Also owns +the country->domain map so classification and URL building share one source. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal +from urllib.parse import parse_qs, urlencode, urlparse + +ResolvedKind = Literal["search", "company", "job"] + +# Locale subdomains that deviate from the ISO code; others map to .indeed.com. +_DOMAIN_OVERRIDES = {"us": "www", "gb": "uk"} + +_JT_VALUES = frozenset( + { + "fulltime", + "parttime", + "contract", + "internship", + "temporary", + "permanent", + "seasonal", + "freelance", + } +) + + +@dataclass(frozen=True) +class ResolvedUrl: + kind: ResolvedKind + value: str # search query, company slug, or job key + url: str + domain: str + location: str | None = None + params: dict[str, str] = field(default_factory=dict) + + +def _is_indeed_host(hostname: str | None) -> bool: + if not hostname: + return False + h = hostname.lower() + return h == "indeed.com" or h.endswith(".indeed.com") + + +def country_domain(country: str) -> str: + """Country code -> Indeed host, e.g. ``us`` -> ``www.indeed.com``.""" + cc = (country or "us").strip().lower() + return f"{_DOMAIN_OVERRIDES.get(cc, cc)}.indeed.com" + + +def resolve_url(url: str) -> ResolvedUrl | None: + """Classify an Indeed URL into a scrape job, or ``None`` if unrecognized.""" + parsed = urlparse(url) + if not _is_indeed_host(parsed.hostname): + return None + domain = parsed.hostname or "www.indeed.com" + path = (parsed.path or "").rstrip("/") + query = parse_qs(parsed.query) + segments = [s for s in path.split("/") if s] + + # /viewjob?jk= + if path.endswith("/viewjob") or segments[:1] == ["viewjob"]: + jk = query.get("jk", [None])[0] + return ResolvedUrl("job", jk, url, domain) if jk else None + + # /cmp//jobs + if segments[:1] == ["cmp"] and "jobs" in segments and len(segments) >= 2: + return ResolvedUrl("company", segments[1], url, domain) + + # /jobs?q=&l= + if path.endswith("/jobs") or segments[-1:] == ["jobs"]: + q = query.get("q", [""])[0] + loc = query.get("l", [None])[0] + extra = { + k: v[0] + for k, v in query.items() + if k in ("radius", "sort", "fromage", "jt", "explvl") and v + } + return ResolvedUrl("search", q, url, domain, location=loc, params=extra) + + return None + + +def build_search_url( + query: str, + *, + country: str = "us", + location: str | None = None, + radius: int | None = None, + job_type: str | None = None, + level: str | None = None, + remote: str | None = None, + from_days: int | None = None, + sort: str = "relevance", + start: int = 0, +) -> str: + """Build an Indeed ``/jobs`` search URL. + + Remote/hybrid is passed as a query keyword; Indeed's structured ``sc`` + attribute codes rotate and aren't stable to hardcode. + """ + domain = country_domain(country) + q = f"{query} {remote}".strip() if remote else query + params: dict[str, str] = {"q": q} + if location: + params["l"] = location + if radius is not None: + params["radius"] = str(radius) + if job_type in _JT_VALUES: + params["jt"] = job_type # type: ignore[assignment] + if level: + params["explvl"] = level + if from_days is not None: + params["fromage"] = str(from_days) + if sort == "date": + params["sort"] = "date" + if start: + params["start"] = str(start) + return f"https://{domain}/jobs?{urlencode(params)}" From 44f9f0301c81f7c6e778e8985b178f667112ab67 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 009/217] feat: indeed warmed browser session with rotate-on-block --- .../platforms/indeed_jobs/fetch.py | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py new file mode 100644 index 000000000..9f913a579 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py @@ -0,0 +1,184 @@ +"""Browser-session fetch seam for the Indeed scraper. + +Indeed fronts its origin with Cloudflare plus an anonymous-bot check that bounces +cold sessions to ``secure.indeed.com/auth``. The working recipe: a persistent +camoufox session that solves Cloudflare, warms on the domain home page, then +navigates to ``/jobs`` in the same context so the clearance carries. + +:class:`IndeedSession` warms per domain once, retries a blocked page on a fresh +residential IP, and caps each navigation with a hard timeout so a stuck solve +can't stall a run. All egress is through the residential proxy. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from contextlib import asynccontextmanager, suppress +from datetime import UTC, datetime +from typing import Any, Protocol +from urllib.parse import urlparse + +from app.utils.proxy import get_proxy_url + +logger = logging.getLogger(__name__) + + +class IndeedAccessBlockedError(RuntimeError): + """Every rotated IP was bounced to Indeed's security wall.""" + + +# Per navigation; a stuck Cloudflare solve otherwise hangs the whole run. +_PAGE_TIMEOUT_S = 75.0 +# Browser-internal timeout; kept above the page timeout so ours fires first. +_SESSION_TIMEOUT_MS = 90_000 +_MAX_ROTATIONS = 3 + +# Markers of a Cloudflare / security-check interstitial served instead of jobs. +_BLOCK_MARKERS = ( + "secure.indeed.com", + "bot-detection", + "security check", + "challenge-platform", + "just a moment", + "verify you are human", + "hcaptcha", +) + + +def now_iso() -> str: + """UTC timestamp in the millisecond ISO shape used by scraper output.""" + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +class _Session(Protocol): + """Minimal browser-session surface used here (real or fake).""" + + async def start(self) -> Any: ... + async def fetch(self, url: str, **kwargs: Any) -> Any: ... + async def close(self) -> Any: ... + + +def _default_session_factory() -> _Session: + """Build a proxied, Cloudflare-solving camoufox session. + + ``disable_resources`` skips images/fonts/media; job data is inline in the + document, so this only trims bandwidth. + """ + from scrapling.fetchers import AsyncStealthySession + + return AsyncStealthySession( + headless=True, + solve_cloudflare=True, + network_idle=True, + block_webrtc=True, + disable_resources=True, + timeout=_SESSION_TIMEOUT_MS, + proxy=get_proxy_url(), + ) + + +def _html(page: Any) -> str: + """Best-effort HTML body across scrapling response shapes.""" + for attr in ("html_content", "body", "text"): + val = getattr(page, attr, None) + if isinstance(val, bytes): + val = val.decode("utf-8", "replace") + if isinstance(val, str) and val: + return val + return "" + + +def _looks_blocked(html: str, final_url: str) -> bool: + """Whether a response is an interstitial rather than a real page.""" + if not html: + return True + haystack = (final_url + " " + html[:6000]).lower() + return any(marker in haystack for marker in _BLOCK_MARKERS) + + +class IndeedSession: + """One warmed browser session that rotates its exit IP when blocked.""" + + def __init__( + self, session_factory: Callable[[], _Session] = _default_session_factory + ) -> None: + self._factory = session_factory + self._session: _Session | None = None + self._warmed: set[str] = set() + self.rotations = 0 + + async def start(self) -> None: + self._session = self._factory() + await self._session.start() + + async def close(self) -> None: + if self._session is not None: + with suppress(Exception): + await self._session.close() + self._session = None + self._warmed.clear() + + async def _rotate(self) -> None: + """Drop the session for a fresh exit IP; clears warmed domains.""" + await self.close() + self.rotations += 1 + await self.start() + logger.info("[indeed] rotated session (rotation #%d)", self.rotations) + + async def _timed_fetch(self, url: str, **kwargs: Any) -> Any: + assert self._session is not None + coro: Awaitable[Any] = self._session.fetch(url, **kwargs) + return await asyncio.wait_for(coro, timeout=_PAGE_TIMEOUT_S) + + async def _ensure_warm(self, domain: str) -> None: + """Land on the domain home with a Google referer before scraping it.""" + if domain in self._warmed: + return + with suppress(Exception): + await self._timed_fetch(f"https://{domain}/", google_search=True) + self._warmed.add(domain) + + async def fetch_html(self, url: str) -> str: + """Return a search/company/job page's HTML through the warmed session. + + Rotates the IP and re-warms on a security-wall bounce or timeout; raises + :class:`IndeedAccessBlockedError` once every rotation is exhausted. + """ + if self._session is None: + await self.start() + domain = urlparse(url).hostname or "www.indeed.com" + attempt = 0 + while True: + try: + await self._ensure_warm(domain) + page = await self._timed_fetch(url) + html = _html(page) + if not _looks_blocked(html, str(getattr(page, "url", "") or "")): + return html + logger.info("[indeed] blocked on %s", url) + except TimeoutError: + logger.warning("[indeed] fetch timed out on %s", url) + except Exception as e: + logger.warning("[indeed] fetch failed on %s: %s", url, e) + + if attempt >= _MAX_ROTATIONS: + raise IndeedAccessBlockedError( + f"Indeed refused {url} on {attempt} rotated IPs" + ) + attempt += 1 + await self._rotate() + + +@asynccontextmanager +async def open_session( + session_factory: Callable[[], _Session] = _default_session_factory, +): + """Open an :class:`IndeedSession` and guarantee teardown.""" + session = IndeedSession(session_factory) + await session.start() + try: + yield session + finally: + await session.close() From 824041ef7fef6c3ec09c2509098ee21e059ef405 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 010/217] feat: indeed scraper orchestrator with pagination and dedupe --- .../platforms/indeed_jobs/scraper.py | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py new file mode 100644 index 000000000..4f66f2ebf --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py @@ -0,0 +1,156 @@ +"""Orchestrator for the Indeed scraper. + +:func:`iter_indeed` streams flat job items from one warmed browser session: +``startUrls`` (search/company pages) or ``queries`` are paged by the ``start`` +offset, deduped by ``jobKey``, and stopped on the first page that adds nothing. +:func:`scrape_indeed` collects the stream under a caller ``limit``. + +Targets run sequentially on a single session — a browser per target would cost +far more than the sequential paging it would save. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse + +from .fetch import IndeedSession, now_iso, open_session +from .parsers import extract_jobcards_blob, job_results, parse_job +from .schemas import IndeedItem, IndeedScrapeInput +from .url_resolver import build_search_url, resolve_url + +logger = logging.getLogger(__name__) + +__all__ = ["iter_indeed", "scrape_indeed"] + +# Indeed's search pagination increments the ``start`` offset by 10. +_PAGE_STEP = 10 +# Indeed stops serving useful results past ~1000; cap pages well before that. +_MAX_PAGES = 10 + + +def _emit(partial: dict[str, Any]) -> dict[str, Any]: + """Stamp ``scrapedAt`` and normalize through the output model.""" + return IndeedItem(**{**partial, "scrapedAt": now_iso()}).to_output() + + +def _with_start(url: str, start: int) -> str: + """Return ``url`` with its ``start`` query param set (removed when 0).""" + parsed = urlparse(url) + params = [(k, v) for k, v in parse_qsl(parsed.query) if k != "start"] + if start: + params.append(("start", str(start))) + return urlunparse(parsed._replace(query=urlencode(params))) + + +async def _paginate( + session: IndeedSession, first_url: str, *, domain: str, max_items: int +) -> AsyncIterator[dict[str, Any]]: + """Yield items across pages of one search/company URL, deduped by ``jobKey``.""" + if max_items <= 0: + return + base_url = f"https://{domain}" + seen: set[str] = set() + emitted = 0 + for page in range(_MAX_PAGES): + html = await session.fetch_html(_with_start(first_url, page * _PAGE_STEP)) + raws = job_results(extract_jobcards_blob(html)) + if not raws: + return + added = 0 + for raw in raws: + item = parse_job(raw, base_url=base_url) + job_key = item.get("jobKey") + if isinstance(job_key, str): + if job_key in seen: + continue + seen.add(job_key) + yield _emit(item) + emitted += 1 + added += 1 + if emitted >= max_items: + return + if added == 0: # a whole page of duplicates: end of useful results + return + + +def _targets(input_model: IndeedScrapeInput) -> list[tuple[str, str]]: + """Resolve inputs to ``(url, domain)`` search/company targets. + + ``startUrls`` take precedence over ``queries``. Job (``/viewjob``) URLs are + skipped here; detail enrichment is a separate flow. + """ + if input_model.startUrls: + out: list[tuple[str, str]] = [] + for entry in input_model.startUrls: + resolved = resolve_url(entry.url) + if resolved is None or resolved.kind == "job": + logger.warning("[indeed] skipping unsupported URL: %s", entry.url) + continue + out.append((resolved.url, resolved.domain)) + return out + + domain = None + urls: list[tuple[str, str]] = [] + for query in input_model.queries: + url = build_search_url( + query, + country=input_model.country, + location=input_model.location, + radius=input_model.radius, + job_type=input_model.jobType, + level=input_model.level, + remote=input_model.remote, + from_days=input_model.fromDays, + sort=input_model.sort, + ) + domain = domain or urlparse(url).hostname or "www.indeed.com" + urls.append((url, domain)) + return urls + + +async def iter_indeed( + input_model: IndeedScrapeInput, session: IndeedSession +) -> AsyncIterator[dict[str, Any]]: + """Stream flat job items for every target, deduped by ``jobKey`` across all.""" + global_seen: set[str] = set() + for url, domain in _targets(input_model): + async for item in _paginate( + session, url, domain=domain, max_items=input_model.maxItemsPerQuery + ): + job_key = item.get("jobKey") + if isinstance(job_key, str): + if job_key in global_seen: + continue + global_seen.add(job_key) + yield item + + +async def scrape_indeed( + input_model: IndeedScrapeInput, + *, + limit: int | None = None, + session: IndeedSession | None = None, +) -> list[dict[str, Any]]: + """Collect :func:`iter_indeed` into a list under an optional ``limit``. + + Opens a warmed session when one is not supplied. ``limit`` is a request-time + guard, not a ceiling baked into the stream. + """ + from app.capabilities.core.progress import emit_progress + + async def _collect(sess: IndeedSession) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + async for item in iter_indeed(input_model, sess): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="item") + if limit is not None and len(results) >= limit: + break + return results + + if session is not None: + return await _collect(session) + async with open_session() as sess: + return await _collect(sess) From 1d367148d009bf3eab4a3fc320eb7c1d7038e90a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:41:54 +0200 Subject: [PATCH 011/217] feat: indeed_jobs package exports --- .../proprietary/platforms/indeed_jobs/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/indeed_jobs/__init__.py diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/__init__.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/__init__.py new file mode 100644 index 000000000..eb59671d7 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/__init__.py @@ -0,0 +1,13 @@ +"""Platform-native Indeed jobs scraper (anonymous, warmed browser session).""" + +from .fetch import IndeedAccessBlockedError +from .schemas import IndeedItem, IndeedScrapeInput +from .scraper import iter_indeed, scrape_indeed + +__all__ = [ + "IndeedAccessBlockedError", + "IndeedItem", + "IndeedScrapeInput", + "iter_indeed", + "scrape_indeed", +] From 794b127136b411060a8aba037ad40ae8280ac0ae Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 012/217] test: indeed_jobs test package --- surfsense_backend/tests/unit/platforms/indeed_jobs/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/__init__.py diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/__init__.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/__init__.py new file mode 100644 index 000000000..e69de29bb From 3d6da91bf36112bf0995628395ff7625ce061bb7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 013/217] test: indeed trimmed jobcards fixture --- .../indeed_jobs/fixtures/sample_jobcards.json | 1633 +++++++++++++++++ 1 file changed, 1633 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_jobcards.json diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_jobcards.json b/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_jobcards.json new file mode 100644 index 000000000..9faa61ef7 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_jobcards.json @@ -0,0 +1,1633 @@ +{ + "metaData": { + "mosaicProviderJobCardsModel": { + "results": [ + { + "adBlob": "-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N1lyYEY1nzdWnmTq7hlQAqMT4EZoUMRYAQrNEOj5nb4ZE_F_ZCNcUyWwXqUnsb3IFHbnllXq0f5pYhflUexN3LD6mf9j8U6fPKOFmTQuul154CxQeo8aExEApf08b2iFfAx7Eci6fsR4i5wlRjvKA5EumK1U1Eauxzua2oUUaLKdAW9137H0CYgdcoAMF84PbbBCj5eW5R0p7Tfgi9NmbhJFRaEZv-u8JEg9U1zcdmaIh6RuPDU7bjQIuak5MO3DwaEJlHXEg4v7-XpR0uTAehVEkbNEFQdl0EwMnI32HxxwGJ_r7Xy7viI3kh6lEdrpZeF1ca18i9FlF_nce_H7BdzO2qmsM11dLrzIwGnipd9-NCmqh-jZJbJLooc2Yl3D8379SRHWpbfMz_a9aa1x_miN7s9YFssX_W8wrIqe3eHIaV-pRMzXMJYIxrcyPztopcC5udht92l9RFkJ8GjCK-zMHwiJRuCIqjML2cMsQblSxIpv5pb9M88dlUMlDd6dMpQBDYRrFe1PYyRu_QTWc_ggES6VZu7PJ7BcMPSFZ7cSuzuPT49lMRw7lYs3P1aakqoiFThrgWJAWATgalPtzZy754uEQG6ez5Gh-KW-nvDhDDfUS-fhvIMcDFftBf6gyU6_wEuH9GI7sOTR8b7jnHS285ryLd212LhBy0GJjliQdMxRmWK0qDqFr8z9q7Ufi9AHPafSIWTFw==", + "adId": "464623129", + "advn": "9533474731693465", + "appliedOrGreater": false, + "applyCount": -1, + "assistedApply": false, + "autoSourcerJob": false, + "bidPosition": 1, + "blobKey": "SoBj6_M3hwflOvyN8p0LbzkdCdPP", + "company": "TherapyNotes.com", + "companyBrandingAttributes": { + "brandingReasons": [ + "PAID_BRANDING" + ], + "brandingReasonsAsString": [ + "PAID_BRANDING" + ], + "headerImageUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_headerimage/1960x400/9d05368a0956c3266ea4291e95d34331", + "logoUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_squarelogo/256x256/eac29e66f882990ae917c959915eb291", + "paidBrandingDealIds": [ + "58664", + "95349" + ], + "showJobBranding": true, + "shownForBrandedJobPackage": false + }, + "companyIdEncrypted": "d10e9cc0e75a9595", + "companyOverviewLink": "/cmp/Therapynotes", + "companyOverviewLinkCampaignId": "serp-linkcompanyname-content", + "companyRating": 3.8, + "companyReviewCount": 30, + "companyReviewLink": "/cmp/Therapynotes/reviews", + "companyReviewLinkCampaignId": "cmplinktst2", + "country": "US", + "createDate": 1774276267415, + "d2iEnabled": false, + "displayTitle": "Software Developer", + "dradisJob": false, + "employerAssistEnabled": false, + "employerResponseTime": 1, + "employerResponsive": true, + "encryptedFccompanyId": "c5ef9b9375a8bbf1", + "encryptedResultData": "VwIPTVJ1cTn5AN7Q-tSqGRXGNe2wB2UYx73qSczFnGU", + "enhancedAttributesModel": {}, + "enticers": [], + "expired": false, + "extractTrackingUrls": "", + "extractedSalary": { + "max": 115000, + "min": 65000, + "type": "YEARLY" + }, + "fccompanyId": -1, + "featuredCompanyAttributes": {}, + "featuredEmployer": false, + "featuredEmployerCandidate": false, + "feedId": 822778, + "formattedLocation": "Philadelphia, PA", + "formattedRelativeTime": "30+ days ago", + "gatedVjp": false, + "hideMetaData": false, + "highVolumeHiringModel": { + "highVolumeHiring": false + }, + "hiringEventJob": false, + "homepageJobFeedSectionId": "0", + "indeedApplyEnabled": true, + "indeedApplyFinishAppUrlEnabled": false, + "indeedApplyResumeType": "required", + "indeedApplyable": true, + "isJobVisited": false, + "isMobileThirdPartyApplyable": false, + "isNoResumeJob": false, + "isSubsidiaryJob": false, + "isTopRatedEmployer": false, + "jobCardRequirementsModel": { + "additionalRequirementsCount": 0, + "jobOnlyRequirements": [], + "jobTagRequirements": [], + "requirementsHeaderShown": false, + "screenerQuestionRequirements": [] + }, + "jobFlairPackageEnabled": false, + "jobLocationCity": "Philadelphia", + "jobLocationState": "PA", + "jobSeekerMatchSummaryModel": { + "sortedEntityDisplayText": [], + "sortedMatchingEntityDisplayText": [], + "sortedMisMatchingEntityDisplayText": [ + ".NET Core", + "Agile software development", + "Bachelor's degree", + "Communication skills", + "Computer Science", + "Computer science", + "Design (software development lifecycle)", + "Developing and maintaining backend systems", + "Entity Framework", + "Event-driven architecture", + "Front-end component implementation", + "HTML", + "JavaScript frameworks", + "Planning (software development lifecycle)", + "PostgreSQL", + "Project development phase management", + "Providing code feedback", + "Responsive design", + "SASS/LESS", + "SQL databases", + "Scalability", + "Scalable systems", + "Software Engineering", + "Software engineering", + "TypeScript", + "Web applications", + "Web services design" + ], + "taxoEntityMatchesNegative": [ + { + "displayText": "SASS/LESS", + "id": "", + "rawName": "SASS/LESS", + "source": "JOB", + "strictness": "", + "suid": "FE6SC" + }, + { + "displayText": "Computer science", + "id": "", + "rawName": "Computer science", + "source": "JOB", + "strictness": "", + "suid": "4E4WW" + }, + { + "displayText": "JavaScript frameworks", + "id": "", + "rawName": "JavaScript frameworks", + "source": "JOB", + "strictness": "", + "suid": "EUPT5" + }, + { + "displayText": "HTML", + "id": "", + "rawName": "HTML", + "source": "JOB", + "strictness": "", + "suid": "Y7U37" + }, + { + "displayText": "Developing and maintaining backend systems", + "id": "", + "rawName": "Developing and maintaining backend systems", + "source": "JOB", + "strictness": "", + "suid": "Q3QNF" + }, + { + "displayText": "Design (software development lifecycle)", + "id": "", + "rawName": "Design (software development lifecycle)", + "source": "JOB", + "strictness": "", + "suid": "XA6CW" + }, + { + "displayText": "Responsive design", + "id": "", + "rawName": "Responsive design", + "source": "JOB", + "strictness": "", + "suid": "36ZHE" + }, + { + "displayText": "Scalable systems", + "id": "", + "rawName": "Scalable systems", + "source": "JOB", + "strictness": "", + "suid": "DP9CS" + }, + { + "displayText": "SQL databases", + "id": "", + "rawName": "SQL databases", + "source": "JOB", + "strictness": "", + "suid": "7G8UN" + }, + { + "displayText": "Software Engineering", + "id": "", + "rawName": "Software Engineering", + "source": "JOB", + "strictness": "", + "suid": "7F6H9" + }, + { + "displayText": "Web applications", + "id": "", + "rawName": "Web applications", + "source": "JOB", + "strictness": "", + "suid": "QM3S8" + }, + { + "displayText": "Entity Framework", + "id": "", + "rawName": "Entity Framework", + "source": "JOB", + "strictness": "", + "suid": "X5HXD" + }, + { + "displayText": "Computer Science", + "id": "", + "rawName": "Computer Science", + "source": "JOB", + "strictness": "", + "suid": "6XNCP" + }, + { + "displayText": "Providing code feedback", + "id": "", + "rawName": "Providing code feedback", + "source": "JOB", + "strictness": "", + "suid": "Z4F5N" + }, + { + "displayText": "PostgreSQL", + "id": "", + "rawName": "PostgreSQL", + "source": "JOB", + "strictness": "", + "suid": "JCKB4" + }, + { + "displayText": ".NET Core", + "id": "", + "rawName": ".NET Core", + "source": "JOB", + "strictness": "", + "suid": "AGW49" + }, + { + "displayText": "TypeScript", + "id": "", + "rawName": "TypeScript", + "source": "JOB", + "strictness": "", + "suid": "WD7PP" + }, + { + "displayText": "Scalability", + "id": "", + "rawName": "Scalability", + "source": "JOB", + "strictness": "", + "suid": "PYTWK" + }, + { + "displayText": "Agile software development", + "id": "", + "rawName": "Agile software development", + "source": "JOB", + "strictness": "", + "suid": "QWGDE" + }, + { + "displayText": "Web services design", + "id": "", + "rawName": "Web services design", + "source": "JOB", + "strictness": "", + "suid": "ZN5SR" + }, + { + "displayText": "Project development phase management", + "id": "", + "rawName": "Project development phase management", + "source": "JOB", + "strictness": "", + "suid": "4UEV6" + }, + { + "displayText": "Bachelor's degree", + "id": "", + "rawName": "Bachelor's degree", + "source": "JOB", + "strictness": "", + "suid": "HFDVW" + }, + { + "displayText": "Software engineering", + "id": "", + "rawName": "Software engineering", + "source": "JOB", + "strictness": "", + "suid": "4R4AM" + }, + { + "displayText": "Event-driven architecture", + "id": "", + "rawName": "Event-driven architecture", + "source": "JOB", + "strictness": "", + "suid": "G82AR" + }, + { + "displayText": "Front-end component implementation", + "id": "", + "rawName": "Front-end component implementation", + "source": "JOB", + "strictness": "", + "suid": "RP4CB" + }, + { + "displayText": "Communication skills", + "id": "", + "rawName": "Communication skills", + "source": "JOB", + "strictness": "", + "suid": "WSBNK" + }, + { + "displayText": "Planning (software development lifecycle)", + "id": "", + "rawName": "Planning (software development lifecycle)", + "source": "JOB", + "strictness": "", + "suid": "X9WB7" + } + ], + "taxoEntityMatchesPositive": [], + "trafficLight": "yellow" + }, + "jobTypes": [], + "jobkey": "7b993d015f4b257a", + "jsiEnabled": false, + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N1lyYEY1nzdWnmTq7hlQAqMT4EZoUMRYAQrNEOj5nb4ZE_F_ZCNcUyWwXqUnsb3IFHbnllXq0f5pYhflUexN3LD6mf9j8U6fPKOFmTQuul154CxQeo8aExEApf08b2iFfAx7Eci6fsR4i5wlRjvKA5EumK1U1Eauxzua2oUUaLKdAW9137H0CYgdcoAMF84PbbBCj5eW5R0p7Tfgi9NmbhJFRaEZv-u8JEg9U1zcdmaIh6RuPDU7bjQIuak5MO3DwaEJlHXEg4v7-XpR0uTAehVEkbNEFQdl0EwMnI32HxxwGJ_r7Xy7viI3kh6lEdrpZeF1ca18i9FlF_nce_H7BdzO2qmsM11dLrzIwGnipd9-NCmqh-jZJbJLooc2Yl3D8379SRHWpbfMz_a9aa1x_miN7s9YFssX_W8wrIqe3eHIaV-pRMzXMJYIxrcyPztopcC5udht92l9RFkJ8GjCK-zMHwiJRuCIqjML2cMsQblSxIpv5pb9M88dlUMlDd6dMpQBDYRrFe1PYyRu_QTWc_ggES6VZu7PJ7BcMPSFZ7cSuzuPT49lMRw7lYs3P1aakqoiFThrgWJAWATgalPtzZy754uEQG6ez5Gh-KW-nvDhDDfUS-fhvIMcDFftBf6gyU6_wEuH9GI7sOTR8b7jnHS285ryLd212LhBy0GJjliQdMxRmWK0qDqFr8z9q7Ufi9AHPafSIWTFw==&xkcb=SoBj6_M3hwflOvyN8p0LbzkdCdPP&p=0&camk=C3EPSzFlQw-JWkl1H7Zuog==&vjs=3", + "locationCount": 1, + "minimumCount": 1, + "mobtk": "1jtgbqmn921k6001", + "moreLocUrl": "/jobs?q=software+engineer&nl=&l=Remote&radius=50&jtid=6804610d02b67694&jcid=d10e9cc0e75a9595&grp=tcl", + "mouseDownHandlerOption": { + "adId": 464623129, + "advn": "9533474731693465", + "extractTrackingUrls": [], + "from": "vjs", + "jobKey": "7b993d015f4b257a", + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N1lyYEY1nzdWnmTq7hlQAqMT4EZoUMRYAQrNEOj5nb4ZE_F_ZCNcUyWwXqUnsb3IFHbnllXq0f5pYhflUexN3LD6mf9j8U6fPKOFmTQuul154CxQeo8aExEApf08b2iFfAx7Eci6fsR4i5wlRjvKA5EumK1U1Eauxzua2oUUaLKdAW9137H0CYgdcoAMF84PbbBCj5eW5R0p7Tfgi9NmbhJFRaEZv-u8JEg9U1zcdmaIh6RuPDU7bjQIuak5MO3DwaEJlHXEg4v7-XpR0uTAehVEkbNEFQdl0EwMnI32HxxwGJ_r7Xy7viI3kh6lEdrpZeF1ca18i9FlF_nce_H7BdzO2qmsM11dLrzIwGnipd9-NCmqh-jZJbJLooc2Yl3D8379SRHWpbfMz_a9aa1x_miN7s9YFssX_W8wrIqe3eHIaV-pRMzXMJYIxrcyPztopcC5udht92l9RFkJ8GjCK-zMHwiJRuCIqjML2cMsQblSxIpv5pb9M88dlUMlDd6dMpQBDYRrFe1PYyRu_QTWc_ggES6VZu7PJ7BcMPSFZ7cSuzuPT49lMRw7lYs3P1aakqoiFThrgWJAWATgalPtzZy754uEQG6ez5Gh-KW-nvDhDDfUS-fhvIMcDFftBf6gyU6_wEuH9GI7sOTR8b7jnHS285ryLd212LhBy0GJjliQdMxRmWK0qDqFr8z9q7Ufi9AHPafSIWTFw==&xkcb=SoBj6_M3hwflOvyN8p0LbzkdCdPP&p=0&camk=C3EPSzFlQw-JWkl1H7Zuog==&vjs=3", + "tk": "1jtgbqmn921k6001" + }, + "newJob": false, + "normTitle": "Software Engineer", + "numHires": 0, + "openInterviewsInterviewsOnTheSpot": false, + "openInterviewsJob": false, + "openInterviewsOffersOnTheSpot": false, + "openInterviewsPhoneJob": false, + "organicApplyStartCount": 6979, + "overrideIndeedApplyText": true, + "packageTier": "NONE", + "preciseLocationModel": { + "obfuscateLocation": false, + "overrideJCMPreciseLocationModel": true + }, + "pubDate": 1774242000000, + "rankedBenefits": { + "GENERIC": [ + "YQ98H", + "RZAT2", + "FQJ2X", + "CFRGS", + "4C2ZW" + ], + "OCCUPATION_TFIDF": [ + "4C2ZW", + "CFRGS", + "FQJ2X", + "RZAT2", + "YQ98H" + ] + }, + "rankingScoresModel": { + "bid": 123996, + "bidPosition": 1, + "eApply": 0.020990705, + "eAttainability": 0.08190364, + "eQualified": 0 + }, + "recommendationReasonModel": { + "reason": null + }, + "redirectToThirdPartySite": false, + "remoteLocation": false, + "remoteWorkModel": { + "inlineText": true, + "text": "Remote", + "type": "REMOTE_ALWAYS" + }, + "resultBeforeExpansion": false, + "resumeMatch": false, + "salarySnippet": { + "currency": "USD", + "salaryTextFormatted": false, + "source": "EXTRACTION", + "text": "$65,000 - $115,000 a year" + }, + "saved": false, + "savedApplication": false, + "screenerQuestionsURL": "iqp://JOB/69c158406b0a663247e222ee?sourceUrl=iqp%3A%2F%2FINDEXEDJOB%2F10074441540%3Fcountry%3DUS&advertiserId=32113445&aggJobId=10074441540&country=US&language=en", + "searchUID": "1jtgbqmn921k6001", + "showCommutePromo": false, + "showEarlyApply": false, + "showJobType": false, + "showRelativeDate": true, + "showSponsoredLabel": false, + "showStrongerAppliedLabel": false, + "smartFillEnabled": false, + "smbD2iEnabled": false, + "snippet": "
    \n
  • We are looking for a passionate engineer skilled in building scalable and responsive web applications and services using Angular and ASP.
    • \n
", + "sourceId": 3368918, + "sponsored": true, + "taxoAttributes": [ + "Profit sharing" + ], + "taxoAttributesDisplayLimit": 3, + "taxoLogAttributes": [ + "4C2ZW" + ], + "taxonomyAttributes": [ + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types" + }, + { + "attributes": [], + "label": "shifts" + }, + { + "attributes": [ + { + "label": "Remote", + "suid": "DSQF7" + } + ], + "label": "remote" + }, + { + "attributes": [ + { + "label": "Dental insurance", + "suid": "FQJ2X" + }, + { + "label": "Disability insurance", + "suid": "CFRGS" + }, + { + "label": "Retirement plan", + "suid": "YQ98H" + }, + { + "label": "Vision insurance", + "suid": "RZAT2" + }, + { + "label": "Profit sharing", + "suid": "4C2ZW" + } + ], + "label": "benefits" + }, + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types-cc" + }, + { + "attributes": [], + "label": "schedules" + } + ], + "thirdPartyApplyUrl": "http://www.indeed.com//applystart?jk=7b993d015f4b257a&from=jobsearch&mvj=0&jobsearchTk=1jtgbqmn921k6001&spon=1&adid=464623129&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N1lyYEY1nzdWnmTq7hlQAqMT4EZoUMRYAQrNEOj5nb4ZE_F_ZCNcUyWwXqUnsb3IFHbnllXq0f5pYhflUexN3LD6mf9j8U6fPKOFmTQuul154CxQeo8aExEApf08b2iFfAx7Eci6fsR4i5wlRjvKA5EumK1U1Eauxzua2oUUaLKdAW9137H0CYgdcoAMF84PbbBCj5eW5R0p7Tfgi9NmbhJFRaEZv-u8JEg9U1zcdmaIh6RuPDU7bjQIuak5MO3DwaEJlHXEg4v7-XpR0uTAehVEkbNEFQdl0EwMnI32HxxwGJ_r7Xy7viI3kh6lEdrpZeF1ca18i9FlF_nce_H7BdzO2qmsM11dLrzIwGnipd9-NCmqh-jZJbJLooc2Yl3D8379SRHWpbfMz_a9aa1x_miN7s9YFssX_W8wrIqe3eHIaV-pRMzXMJYIxrcyPztopcC5udht92l9RFkJ8GjCK-zMHwiJRuCIqjML2cMsQblSxIpv5pb9M88dlUMlDd6dMpQBDYRrFe1PYyRu_QTWc_ggES6VZu7PJ7BcMPSFZ7cSuzuPT49lMRw7lYs3P1aakqoiFThrgWJAWATgalPtzZy754uEQG6ez5Gh-KW-nvDhDDfUS-fhvIMcDFftBf6gyU6_wEuH9GI7sOTR8b7jnHS285ryLd212LhBy0GJjliQdMxRmWK0qDqFr8z9q7Ufi9AHPafSIWTFw%3D%3D&xkcb=SoBj6_M3hwflOvyN8p0LbzkdCdPP&sjdu=Xbbwb0dbVlAPs0TIG7JumEX25uTOF1Oj2WnXk16AywHWA2PUcrZQMHMXHdXk2eTvbAixkyUF3j-OOmJlYeYtIjeDAvq7rEASgcC0zRMWutBvuyCgKBpkTMsm3zcuFYK1uIzH-8PzvPSrzMEzGtRTg5jv-iXDq5yVFQaIv8kgOWkCg9YikGZNJgeIivR2iCfdxWXtDkF3x7BjrIiI14W0QsCgmbR1Dg8qS-_CO9vND2NLAJoHK8df0UY8vlo7n1P7mzaDYyEyfb0INk3TiuEWMemQii8wUThkfLKJY9xDEST1O0JfmjdhQ3CuSF3QpabBfwaJIpgPfWh5SU8ES2tfW40LHQ6gsm9PLZCCJd2RWBsAg-ynqMVmNvHBgTO9S5FFBEcILkSKcC_MyVjNUK-8qfhKnQQPXgamiQj-imh4f5H-01jn0qSbiIaNBBbWs8WaiEDX1JaFE1ANmzavVGRxShPILHL_US7TCQ871Vr89ga7Y0v9ivrNvS2rdS5oF4sLUvcuVq9FhTVrjZpfmMNrEcehBwAJB_oy0hwGoit3I9QwJBE7yTcTRuabRpUOY8NGnuBwzl_YpEii8y3zKUv_S56bGdV6hXrIYnlrC_lQUZISI14iKUD-Dc7DUmh-vDl7MpfMvKQUktHpIeGdJMdxqweC1QdHFvhFbZkfRgjDhnNc0jNtP6Mr34zH5R5LQ8XbdLc6hlCebmo1Cv6kdjNRaLtdI6Mq1Y9Gq0gPMRVyTIPtovc7LsBlfXwcpxl21GixCJ_-bI6UaPK3TmPi0p_29vNanWx2E5_SGwELeYDW3egDxwPLtz4Smx_kFWcH6inYTsQbi68YonwAEahac1GjyjGvrjYKsoS680WKuzZS0Mutx0IkRyRloqlBvNHBK06L-Wx8ojMF4axEl43YmtkRZeyfVWrwEc9a9vcONhKAJbgOACubfzVac9cjNyYppXgW-XK5SWGo9lcjkVg5r63SM4pv1skycGMDMD2AbX8kTiFoukUeCElgDzeLhXKgJGiMwyYHrbKUQRUDVrgVJ81WasDodKiQH-yZ_-PiXCHPergRl5I31036jtOI1ylLDUgKg6ut2knm_9eRe_tC2EEhxXtT7ZaLatnvaE4b2G0o_Q-izpmOx92rL3gkpo33T97hPc7J1sZW7kwt9HXHyZBXX5xZ6pdvi2b3fmxnjLYK0Ro&vjfrom=vjs&astse=d6aae2e110970c59&assa=588", + "tier": { + "matchedPreferences": { + "longMatchedPreferences": [], + "stringMatchedPreferences": [] + }, + "type": "DEFAULT" + }, + "title": "Software Developer", + "translatedAttributes": [], + "translatedCmiJobTags": [], + "truncatedCompany": "TherapyNotes.com", + "urgentlyHiring": false, + "viewJobLink": "/viewjob?jk=7b993d015f4b257a&from=vjs&tk=1jtgbqmn921k6001&viewtype=embedded&advn=9533474731693465&adid=464623129&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N1lyYEY1nzdWnmTq7hlQAqMT4EZoUMRYAQrNEOj5nb4ZE_F_ZCNcUyWwXqUnsb3IFHbnllXq0f5pYhflUexN3LD6mf9j8U6fPKOFmTQuul154CxQeo8aExEApf08b2iFfAx7Eci6fsR4i5wlRjvKA5EumK1U1Eauxzua2oUUaLKdAW9137H0CYgdcoAMF84PbbBCj5eW5R0p7Tfgi9NmbhJFRaEZv-u8JEg9U1zcdmaIh6RuPDU7bjQIuak5MO3DwaEJlHXEg4v7-XpR0uTAehVEkbNEFQdl0EwMnI32HxxwGJ_r7Xy7viI3kh6lEdrpZeF1ca18i9FlF_nce_H7BdzO2qmsM11dLrzIwGnipd9-NCmqh-jZJbJLooc2Yl3D8379SRHWpbfMz_a9aa1x_miN7s9YFssX_W8wrIqe3eHIaV-pRMzXMJYIxrcyPztopcC5udht92l9RFkJ8GjCK-zMHwiJRuCIqjML2cMsQblSxIpv5pb9M88dlUMlDd6dMpQBDYRrFe1PYyRu_QTWc_ggES6VZu7PJ7BcMPSFZ7cSuzuPT49lMRw7lYs3P1aakqoiFThrgWJAWATgalPtzZy754uEQG6ez5Gh-KW-nvDhDDfUS-fhvIMcDFftBf6gyU6_wEuH9GI7sOTR8b7jnHS285ryLd212LhBy0GJjliQdMxRmWK0qDqFr8z9q7Ufi9AHPafSIWTFw%3D%3D&xkcb=SoBj6_M3hwflOvyN8p0LbzkdCdPP&continueUrl=%2Fjobs%3Fq%3Dsoftware%2Bengineer%26l%3DRemote", + "vjFeaturedEmployerCandidate": false + }, + { + "adBlob": "-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N2MhaGUtj2kz1cGhiRJCPZR4-FTtJq5aUy42k66aUDfG-jeElTJaJF_uhS-sSFhT8XjUynnZ9RdOuIClMgX3_QeGQ1eZ8ZugnnoExqs4lHbEtd3kidPweUJRF0MyFycMxMCn9PPBMAr2RWtUUo1uwl1XqX3eSw72oEa1W-RXDX2rtTZXHOprn3W7VUJ7KBfEuz9dn-7yLd2EqAKhhT-rHN9Lm1ES3UePr384UySNQNOC05nSAXRD7sWGEFSLN7BMYsDwZMQA39fRPGioSyhsoNKnGbkEiEgeUpRce_grNxxR7lU_oVHb-QemsQv_jZWwwZ9Q2j74LtVfpNdO_U1is58Hju2-BHih_ov1gUxcwRJGEIQcmQGtqsaarwGZalEQeCafsMzlN1QpmPGpH55FsGE3d_71A39-xRsCn0gFkouk_KEV0G8g_DoU5G9ZMEGdSbPDpLHYwszFA6QFKKjcTqzcy7hntV6BtA5gVnr_pQWNY3XNhrT59XGBm34NnzOfGGT8kCDoyd8juFITsfyMFHEMDkIrek0JfU-1Y-FPzmaLCpVNxPkRC3e8xXPqqzzcXshv5_ZRxpTNgBuUrd_EdhZtv-yKLdC0FtfGSSl8rm9Aqexw0CS7Wivy6DdNTGpHNsOybkoisYqEDeYaDN_zuSYmznXafHCwT-FgULG5_GSI2d9kU__07ht6bD0RzHr-Jk=", + "adId": "464623129", + "advn": "9533474731693465", + "appliedOrGreater": false, + "applyCount": -1, + "assistedApply": false, + "autoSourcerJob": false, + "bidPosition": 4, + "blobKey": "SoDX6_M3hwflOvyN8p0KbzkdCdPP", + "company": "TherapyNotes.com", + "companyBrandingAttributes": { + "brandingReasons": [ + "PAID_BRANDING" + ], + "brandingReasonsAsString": [ + "PAID_BRANDING" + ], + "headerImageUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_headerimage/1960x400/9d05368a0956c3266ea4291e95d34331", + "logoUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_squarelogo/256x256/eac29e66f882990ae917c959915eb291", + "paidBrandingDealIds": [ + "58664", + "95349" + ], + "showJobBranding": true, + "shownForBrandedJobPackage": false + }, + "companyIdEncrypted": "d10e9cc0e75a9595", + "companyOverviewLink": "/cmp/Therapynotes", + "companyOverviewLinkCampaignId": "serp-linkcompanyname-content", + "companyRating": 3.8, + "companyReviewCount": 30, + "companyReviewLink": "/cmp/Therapynotes/reviews", + "companyReviewLinkCampaignId": "cmplinktst2", + "country": "US", + "createDate": 1782827792000, + "d2iEnabled": false, + "displayTitle": "Senior Software Developer (Agentic Development)", + "dradisJob": false, + "employerAssistEnabled": false, + "employerResponseTime": 1, + "employerResponsive": true, + "encryptedFccompanyId": "c5ef9b9375a8bbf1", + "encryptedResultData": "VwIPTVJ1cTn5AN7Q-tSqGRXGNe2wB2UYx73qSczFnGU", + "enhancedAttributesModel": {}, + "enticers": [], + "expired": false, + "extractTrackingUrls": "", + "extractedSalary": { + "max": 135000, + "min": 110000, + "type": "YEARLY" + }, + "fccompanyId": -1, + "featuredCompanyAttributes": {}, + "featuredEmployer": false, + "featuredEmployerCandidate": false, + "feedId": 822778, + "formattedLocation": "Philadelphia, PA", + "formattedRelativeTime": "13 days ago", + "gatedVjp": false, + "hideMetaData": false, + "highVolumeHiringModel": { + "highVolumeHiring": false + }, + "hiringEventJob": false, + "homepageJobFeedSectionId": "0", + "indeedApplyEnabled": true, + "indeedApplyFinishAppUrlEnabled": false, + "indeedApplyResumeType": "required", + "indeedApplyable": true, + "isJobVisited": false, + "isMobileThirdPartyApplyable": false, + "isNoResumeJob": false, + "isSubsidiaryJob": false, + "isTopRatedEmployer": false, + "jobCardRequirementsModel": { + "additionalRequirementsCount": 0, + "jobOnlyRequirements": [], + "jobTagRequirements": [], + "requirementsHeaderShown": false, + "screenerQuestionRequirements": [] + }, + "jobFlairPackageEnabled": false, + "jobLocationCity": "Philadelphia", + "jobLocationState": "PA", + "jobSeekerMatchSummaryModel": { + "sortedEntityDisplayText": [], + "sortedMatchingEntityDisplayText": [], + "sortedMisMatchingEntityDisplayText": [ + ".NET Core", + "AI models", + "AI platforms (beyond public GPTs)", + "AI use case identification", + "AI-driven automation", + "API integrations", + "Agile software development", + "Automation software", + "Bachelor's degree", + "Collaboration with product development teams", + "Communication skills", + "Computer Science", + "Computer science", + "Continuous improvement for developers", + "Cross-functional communication", + "Design (software development lifecycle)", + "Developer tools", + "Employee onboarding", + "Engineering process optimization", + "Entity Framework", + "Full-stack development", + "Generative AI", + "HTML", + "IT quality control", + "JavaScript frameworks", + "Mentoring", + "PostgreSQL", + "Process design", + "Product management", + "Prompt engineering", + "Providing code feedback", + "Quality assurance experience within IT", + "Quality control", + "Release management", + "SASS/LESS", + "SQL databases", + "SaaS", + "Scalability", + "Scalable systems", + "Software Engineering", + "Software engineering", + "Software testing", + "System design", + "System design for system development", + "Team training", + "Technology security practices", + "Tooling", + "TypeScript" + ], + "taxoEntityMatchesNegative": [ + { + "displayText": "SASS/LESS", + "id": "", + "rawName": "SASS/LESS", + "source": "JOB", + "strictness": "", + "suid": "FE6SC" + }, + { + "displayText": "Software testing", + "id": "", + "rawName": "Software testing", + "source": "JOB", + "strictness": "", + "suid": "PZP9P" + }, + { + "displayText": "Computer science", + "id": "", + "rawName": "Computer science", + "source": "JOB", + "strictness": "", + "suid": "4E4WW" + }, + { + "displayText": "JavaScript frameworks", + "id": "", + "rawName": "JavaScript frameworks", + "source": "JOB", + "strictness": "", + "suid": "EUPT5" + }, + { + "displayText": "HTML", + "id": "", + "rawName": "HTML", + "source": "JOB", + "strictness": "", + "suid": "Y7U37" + }, + { + "displayText": "System design", + "id": "", + "rawName": "System design", + "source": "JOB", + "strictness": "", + "suid": "C5C2F" + }, + { + "displayText": "Mentoring", + "id": "", + "rawName": "Mentoring", + "source": "JOB", + "strictness": "", + "suid": "MGSEB" + }, + { + "displayText": "Cross-functional communication", + "id": "", + "rawName": "Cross-functional communication", + "source": "JOB", + "strictness": "", + "suid": "Y8657" + }, + { + "displayText": "Process design", + "id": "", + "rawName": "Process design", + "source": "JOB", + "strictness": "", + "suid": "C4MUU" + }, + { + "displayText": "AI use case identification", + "id": "", + "rawName": "AI use case identification", + "source": "JOB", + "strictness": "", + "suid": "4KV6M" + }, + { + "displayText": "Design (software development lifecycle)", + "id": "", + "rawName": "Design (software development lifecycle)", + "source": "JOB", + "strictness": "", + "suid": "XA6CW" + }, + { + "displayText": "Scalable systems", + "id": "", + "rawName": "Scalable systems", + "source": "JOB", + "strictness": "", + "suid": "DP9CS" + }, + { + "displayText": "Quality control", + "id": "", + "rawName": "Quality control", + "source": "JOB", + "strictness": "", + "suid": "K9VX6" + }, + { + "displayText": "AI models", + "id": "", + "rawName": "AI models", + "source": "JOB", + "strictness": "", + "suid": "22JPW" + }, + { + "displayText": "SQL databases", + "id": "", + "rawName": "SQL databases", + "source": "JOB", + "strictness": "", + "suid": "7G8UN" + }, + { + "displayText": "Continuous improvement for developers", + "id": "", + "rawName": "Continuous improvement for developers", + "source": "JOB", + "strictness": "", + "suid": "TZQ7R" + }, + { + "displayText": "Software Engineering", + "id": "", + "rawName": "Software Engineering", + "source": "JOB", + "strictness": "", + "suid": "7F6H9" + }, + { + "displayText": "Quality assurance experience within IT", + "id": "", + "rawName": "Quality assurance experience within IT", + "source": "JOB", + "strictness": "", + "suid": "EC9V5" + }, + { + "displayText": "Entity Framework", + "id": "", + "rawName": "Entity Framework", + "source": "JOB", + "strictness": "", + "suid": "X5HXD" + }, + { + "displayText": "Computer Science", + "id": "", + "rawName": "Computer Science", + "source": "JOB", + "strictness": "", + "suid": "6XNCP" + }, + { + "displayText": "Product management", + "id": "", + "rawName": "Product management", + "source": "JOB", + "strictness": "", + "suid": "JT9DE" + }, + { + "displayText": "Providing code feedback", + "id": "", + "rawName": "Providing code feedback", + "source": "JOB", + "strictness": "", + "suid": "Z4F5N" + }, + { + "displayText": "Engineering process optimization", + "id": "", + "rawName": "Engineering process optimization", + "source": "JOB", + "strictness": "", + "suid": "CCQ9J" + }, + { + "displayText": "Collaboration with product development teams", + "id": "", + "rawName": "Collaboration with product development teams", + "source": "JOB", + "strictness": "", + "suid": "GEYN2" + }, + { + "displayText": "PostgreSQL", + "id": "", + "rawName": "PostgreSQL", + "source": "JOB", + "strictness": "", + "suid": "JCKB4" + }, + { + "displayText": "AI-driven automation", + "id": "", + "rawName": "AI-driven automation", + "source": "JOB", + "strictness": "", + "suid": "TX6QF" + }, + { + "displayText": "IT quality control", + "id": "", + "rawName": "IT quality control", + "source": "JOB", + "strictness": "", + "suid": "AHSXR" + }, + { + "displayText": "Release management", + "id": "", + "rawName": "Release management", + "source": "JOB", + "strictness": "", + "suid": "J3CT2" + }, + { + "displayText": "Prompt engineering", + "id": "", + "rawName": "Prompt engineering", + "source": "JOB", + "strictness": "", + "suid": "EV2FP" + }, + { + "displayText": "Tooling", + "id": "", + "rawName": "Tooling", + "source": "JOB", + "strictness": "", + "suid": "93YM8" + }, + { + "displayText": "AI platforms (beyond public GPTs)", + "id": "", + "rawName": "AI platforms (beyond public GPTs)", + "source": "JOB", + "strictness": "", + "suid": "DP8RP" + }, + { + "displayText": ".NET Core", + "id": "", + "rawName": ".NET Core", + "source": "JOB", + "strictness": "", + "suid": "AGW49" + }, + { + "displayText": "TypeScript", + "id": "", + "rawName": "TypeScript", + "source": "JOB", + "strictness": "", + "suid": "WD7PP" + }, + { + "displayText": "Developer tools", + "id": "", + "rawName": "Developer tools", + "source": "JOB", + "strictness": "", + "suid": "WE46Y" + }, + { + "displayText": "SaaS", + "id": "", + "rawName": "SaaS", + "source": "JOB", + "strictness": "", + "suid": "UC5ZW" + }, + { + "displayText": "Team training", + "id": "", + "rawName": "Team training", + "source": "JOB", + "strictness": "", + "suid": "JSNTU" + }, + { + "displayText": "Full-stack development", + "id": "", + "rawName": "Full-stack development", + "source": "JOB", + "strictness": "", + "suid": "8PA9N" + }, + { + "displayText": "Scalability", + "id": "", + "rawName": "Scalability", + "source": "JOB", + "strictness": "", + "suid": "PYTWK" + }, + { + "displayText": "Automation software", + "id": "", + "rawName": "Automation software", + "source": "JOB", + "strictness": "", + "suid": "ANHQ8" + }, + { + "displayText": "Agile software development", + "id": "", + "rawName": "Agile software development", + "source": "JOB", + "strictness": "", + "suid": "QWGDE" + }, + { + "displayText": "System design for system development", + "id": "", + "rawName": "System design for system development", + "source": "JOB", + "strictness": "", + "suid": "FNWKU" + }, + { + "displayText": "Generative AI", + "id": "", + "rawName": "Generative AI", + "source": "JOB", + "strictness": "", + "suid": "Y6B2U" + }, + { + "displayText": "Bachelor's degree", + "id": "", + "rawName": "Bachelor's degree", + "source": "JOB", + "strictness": "", + "suid": "HFDVW" + }, + { + "displayText": "API integrations", + "id": "", + "rawName": "API integrations", + "source": "JOB", + "strictness": "", + "suid": "TXVT6" + }, + { + "displayText": "Software engineering", + "id": "", + "rawName": "Software engineering", + "source": "JOB", + "strictness": "", + "suid": "4R4AM" + }, + { + "displayText": "Communication skills", + "id": "", + "rawName": "Communication skills", + "source": "JOB", + "strictness": "", + "suid": "WSBNK" + }, + { + "displayText": "Technology security practices", + "id": "", + "rawName": "Technology security practices", + "source": "JOB", + "strictness": "", + "suid": "EKUZN" + }, + { + "displayText": "Employee onboarding", + "id": "", + "rawName": "Employee onboarding", + "source": "JOB", + "strictness": "", + "suid": "3ERWX" + } + ], + "taxoEntityMatchesPositive": [], + "trafficLight": "yellow" + }, + "jobTypes": [], + "jobkey": "1964995584dc3547", + "jsiEnabled": false, + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N2MhaGUtj2kz1cGhiRJCPZR4-FTtJq5aUy42k66aUDfG-jeElTJaJF_uhS-sSFhT8XjUynnZ9RdOuIClMgX3_QeGQ1eZ8ZugnnoExqs4lHbEtd3kidPweUJRF0MyFycMxMCn9PPBMAr2RWtUUo1uwl1XqX3eSw72oEa1W-RXDX2rtTZXHOprn3W7VUJ7KBfEuz9dn-7yLd2EqAKhhT-rHN9Lm1ES3UePr384UySNQNOC05nSAXRD7sWGEFSLN7BMYsDwZMQA39fRPGioSyhsoNKnGbkEiEgeUpRce_grNxxR7lU_oVHb-QemsQv_jZWwwZ9Q2j74LtVfpNdO_U1is58Hju2-BHih_ov1gUxcwRJGEIQcmQGtqsaarwGZalEQeCafsMzlN1QpmPGpH55FsGE3d_71A39-xRsCn0gFkouk_KEV0G8g_DoU5G9ZMEGdSbPDpLHYwszFA6QFKKjcTqzcy7hntV6BtA5gVnr_pQWNY3XNhrT59XGBm34NnzOfGGT8kCDoyd8juFITsfyMFHEMDkIrek0JfU-1Y-FPzmaLCpVNxPkRC3e8xXPqqzzcXshv5_ZRxpTNgBuUrd_EdhZtv-yKLdC0FtfGSSl8rm9Aqexw0CS7Wivy6DdNTGpHNsOybkoisYqEDeYaDN_zuSYmznXafHCwT-FgULG5_GSI2d9kU__07ht6bD0RzHr-Jk=&xkcb=SoDX6_M3hwflOvyN8p0KbzkdCdPP&p=1&camk=C3EPSzFlQw-JWkl1H7Zuog==&vjs=3", + "locationCount": 1, + "minimumCount": 1, + "mobtk": "1jtgbqmn921k6001", + "moreLocUrl": "/jobs?q=software+engineer&nl=&l=Remote&radius=50&jtid=7ca63af8b5ec590b&jcid=d10e9cc0e75a9595&grp=tcl", + "mouseDownHandlerOption": { + "adId": 464623129, + "advn": "9533474731693465", + "extractTrackingUrls": [], + "from": "vjs", + "jobKey": "1964995584dc3547", + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N2MhaGUtj2kz1cGhiRJCPZR4-FTtJq5aUy42k66aUDfG-jeElTJaJF_uhS-sSFhT8XjUynnZ9RdOuIClMgX3_QeGQ1eZ8ZugnnoExqs4lHbEtd3kidPweUJRF0MyFycMxMCn9PPBMAr2RWtUUo1uwl1XqX3eSw72oEa1W-RXDX2rtTZXHOprn3W7VUJ7KBfEuz9dn-7yLd2EqAKhhT-rHN9Lm1ES3UePr384UySNQNOC05nSAXRD7sWGEFSLN7BMYsDwZMQA39fRPGioSyhsoNKnGbkEiEgeUpRce_grNxxR7lU_oVHb-QemsQv_jZWwwZ9Q2j74LtVfpNdO_U1is58Hju2-BHih_ov1gUxcwRJGEIQcmQGtqsaarwGZalEQeCafsMzlN1QpmPGpH55FsGE3d_71A39-xRsCn0gFkouk_KEV0G8g_DoU5G9ZMEGdSbPDpLHYwszFA6QFKKjcTqzcy7hntV6BtA5gVnr_pQWNY3XNhrT59XGBm34NnzOfGGT8kCDoyd8juFITsfyMFHEMDkIrek0JfU-1Y-FPzmaLCpVNxPkRC3e8xXPqqzzcXshv5_ZRxpTNgBuUrd_EdhZtv-yKLdC0FtfGSSl8rm9Aqexw0CS7Wivy6DdNTGpHNsOybkoisYqEDeYaDN_zuSYmznXafHCwT-FgULG5_GSI2d9kU__07ht6bD0RzHr-Jk=&xkcb=SoDX6_M3hwflOvyN8p0KbzkdCdPP&p=1&camk=C3EPSzFlQw-JWkl1H7Zuog==&vjs=3", + "tk": "1jtgbqmn921k6001" + }, + "newJob": false, + "normTitle": "Senior Software Engineer", + "numHires": 0, + "openInterviewsInterviewsOnTheSpot": false, + "openInterviewsJob": false, + "openInterviewsOffersOnTheSpot": false, + "openInterviewsPhoneJob": false, + "organicApplyStartCount": 55, + "overrideIndeedApplyText": true, + "packageTier": "NONE", + "preciseLocationModel": { + "obfuscateLocation": false, + "overrideJCMPreciseLocationModel": true + }, + "pubDate": 1782795600000, + "rankedBenefits": { + "GENERIC": [ + "YQ98H", + "RZAT2", + "FQJ2X", + "CFRGS", + "4C2ZW" + ], + "OCCUPATION_TFIDF": [ + "4C2ZW", + "CFRGS", + "FQJ2X", + "RZAT2", + "YQ98H" + ] + }, + "rankingScoresModel": { + "bid": 90638, + "bidPosition": 4, + "eApply": 0.011701469, + "eAttainability": 0, + "eQualified": 0 + }, + "recommendationReasonModel": { + "reason": null + }, + "redirectToThirdPartySite": false, + "remoteLocation": false, + "remoteWorkModel": { + "inlineText": true, + "text": "Remote", + "type": "REMOTE_ALWAYS" + }, + "resultBeforeExpansion": false, + "resumeMatch": false, + "salarySnippet": { + "currency": "USD", + "salaryTextFormatted": false, + "source": "EXTRACTION", + "text": "$110,000 - $135,000 a year" + }, + "saved": false, + "savedApplication": false, + "screenerQuestionsURL": "iqp://INDEXEDJOB/10183863070?country=US", + "searchUID": "1jtgbqmn921k6001", + "showCommutePromo": false, + "showEarlyApply": false, + "showJobType": false, + "showRelativeDate": true, + "showSponsoredLabel": false, + "showStrongerAppliedLabel": false, + "smartFillEnabled": false, + "smbD2iEnabled": false, + "snippet": "
    \n
  • Ability to design reliable, maintainable, and scalable software architectures.
    • \n
  • You engineer tools, processes and workflows that meaningfully reduce friction in\u2026
    • \n
", + "sourceId": 3368918, + "sponsored": true, + "taxoAttributes": [ + "Profit sharing" + ], + "taxoAttributesDisplayLimit": 3, + "taxoLogAttributes": [ + "4C2ZW" + ], + "taxonomyAttributes": [ + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types" + }, + { + "attributes": [], + "label": "shifts" + }, + { + "attributes": [ + { + "label": "Remote", + "suid": "DSQF7" + } + ], + "label": "remote" + }, + { + "attributes": [ + { + "label": "Dental insurance", + "suid": "FQJ2X" + }, + { + "label": "Disability insurance", + "suid": "CFRGS" + }, + { + "label": "Vision insurance", + "suid": "RZAT2" + }, + { + "label": "Retirement plan", + "suid": "YQ98H" + }, + { + "label": "Profit sharing", + "suid": "4C2ZW" + } + ], + "label": "benefits" + }, + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types-cc" + }, + { + "attributes": [], + "label": "schedules" + } + ], + "thirdPartyApplyUrl": "http://www.indeed.com//applystart?jk=1964995584dc3547&from=jobsearch&mvj=0&jobsearchTk=1jtgbqmn921k6001&spon=1&adid=464623129&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N2MhaGUtj2kz1cGhiRJCPZR4-FTtJq5aUy42k66aUDfG-jeElTJaJF_uhS-sSFhT8XjUynnZ9RdOuIClMgX3_QeGQ1eZ8ZugnnoExqs4lHbEtd3kidPweUJRF0MyFycMxMCn9PPBMAr2RWtUUo1uwl1XqX3eSw72oEa1W-RXDX2rtTZXHOprn3W7VUJ7KBfEuz9dn-7yLd2EqAKhhT-rHN9Lm1ES3UePr384UySNQNOC05nSAXRD7sWGEFSLN7BMYsDwZMQA39fRPGioSyhsoNKnGbkEiEgeUpRce_grNxxR7lU_oVHb-QemsQv_jZWwwZ9Q2j74LtVfpNdO_U1is58Hju2-BHih_ov1gUxcwRJGEIQcmQGtqsaarwGZalEQeCafsMzlN1QpmPGpH55FsGE3d_71A39-xRsCn0gFkouk_KEV0G8g_DoU5G9ZMEGdSbPDpLHYwszFA6QFKKjcTqzcy7hntV6BtA5gVnr_pQWNY3XNhrT59XGBm34NnzOfGGT8kCDoyd8juFITsfyMFHEMDkIrek0JfU-1Y-FPzmaLCpVNxPkRC3e8xXPqqzzcXshv5_ZRxpTNgBuUrd_EdhZtv-yKLdC0FtfGSSl8rm9Aqexw0CS7Wivy6DdNTGpHNsOybkoisYqEDeYaDN_zuSYmznXafHCwT-FgULG5_GSI2d9kU__07ht6bD0RzHr-Jk%3D&xkcb=SoDX6_M3hwflOvyN8p0KbzkdCdPP&sjdu=Xbbwb0dbVlAPs0TIG7JumEX25uTOF1Oj2WnXk16AywHWA2PUcrZQMHMXHdXk2eTvbAixkyUF3j-OOmJlYeYtIjeDAvq7rEASgcC0zRMWutD3E9nQwSeMuCzdft7Qzt3FrQrTab10PIp0rWyA5dpjRE_z_G8OF7DNUc4gvnFFI-gpZwJRNYrOmZ5GZc_T8c2uHtcA7yEHKanT-ZgKiAexbe5qjDvO2kXqUnLLpW7YdwVJyc9eBIFg_vfY1-syU0GBwcsCuyrZU8nv0Xq8G93mw-7llCMyNJkF-WtwsM9Ue6aeLchHe0XGbSYl1Qcgf0sY-JQpXW7ju_Bk-MDoBCtis20YCc1bVmoQthnzEBWFp_9TYbQKJLr11DQp75s5Vdmsf-d5IQTi2PMp0iW7xymdYEQ3LDGdq2iavfaMS8A8OQ6rOQXa9jJFvJk9Ta5XeU_mpy5fSFg1oxkr0xp2QA2Utj3433zpiWFKDsr3Df6bWNvXjJoRzwIWKUpx2esmkKj7JybAVwMqmDoSkpfVABPfjuxY1hcCETtdEgmRbv6rBt0LOZX4H3srbX-kBOf6bWogp0PCe91mjDN2aw61THWKieSLyEzPUCYuoYr2onoda8SvUcksZX8zNPm_6b_s-yzzK9gO8ayTD6J-4jLltcKnpVPDYlxxCOFzCE00ROrg3Fdsv5xMIDVLwW7-zd6Rj8UrSVI77Ngt6Xuv4QptvZuF1trAUcMXIh9eq5jBrwb3_p3iAChwWpQm2LoaxnbeXmY4lWLNx-GuVCMQC9PPCdO2IZVUI2AHBR-WjvuESBVvRXlAXYLKObNGVDkrcUxV8nvoCcVndtP9TF5ZORGRAdWScvy63d0aKgnOkxJ3fSSHwTlRDOEY_tIYg31V8b1f21PG3DeLB17KWd6yFJNAPxgz99fRLl22swix5fq0lRJsIkN8K5eNtzDQ8b2M5QNT_-vr7ydAs3NiWqZbkns80Rhva-g4ppYeirl4F-oE27jkbsyR0mT19gIbooC-0xxw5rhbJVd3xcU-Qns5G3Nr8utN_PCj8MNuo8AomjLddrI-OA_DFCxJ9dMx4OQltXVjCk4TJI5pSLZxWVsDOsIirhwPN4O6LTFptkPn8PQP0M-MqfFUyGuzU73dybEBIAltr8aORGKf6dZx5anCwoZfBAU3zQ&vjfrom=vjs&astse=da7fff3285780cc4&assa=589", + "tier": { + "matchedPreferences": { + "longMatchedPreferences": [], + "stringMatchedPreferences": [] + }, + "type": "DEFAULT" + }, + "title": "Senior Software Developer (Agentic Development)", + "translatedAttributes": [], + "translatedCmiJobTags": [], + "truncatedCompany": "TherapyNotes.com", + "urgentlyHiring": false, + "viewJobLink": "/viewjob?jk=1964995584dc3547&from=vjs&tk=1jtgbqmn921k6001&viewtype=embedded&advn=9533474731693465&adid=464623129&ad=-6NYlbfkN0AfEq5j7saNRG5aU_mvVTuWOB9eeCeHIli2OtKfZJ_lU68CQK3WO6fr1U8P9BG_8N2MhaGUtj2kz1cGhiRJCPZR4-FTtJq5aUy42k66aUDfG-jeElTJaJF_uhS-sSFhT8XjUynnZ9RdOuIClMgX3_QeGQ1eZ8ZugnnoExqs4lHbEtd3kidPweUJRF0MyFycMxMCn9PPBMAr2RWtUUo1uwl1XqX3eSw72oEa1W-RXDX2rtTZXHOprn3W7VUJ7KBfEuz9dn-7yLd2EqAKhhT-rHN9Lm1ES3UePr384UySNQNOC05nSAXRD7sWGEFSLN7BMYsDwZMQA39fRPGioSyhsoNKnGbkEiEgeUpRce_grNxxR7lU_oVHb-QemsQv_jZWwwZ9Q2j74LtVfpNdO_U1is58Hju2-BHih_ov1gUxcwRJGEIQcmQGtqsaarwGZalEQeCafsMzlN1QpmPGpH55FsGE3d_71A39-xRsCn0gFkouk_KEV0G8g_DoU5G9ZMEGdSbPDpLHYwszFA6QFKKjcTqzcy7hntV6BtA5gVnr_pQWNY3XNhrT59XGBm34NnzOfGGT8kCDoyd8juFITsfyMFHEMDkIrek0JfU-1Y-FPzmaLCpVNxPkRC3e8xXPqqzzcXshv5_ZRxpTNgBuUrd_EdhZtv-yKLdC0FtfGSSl8rm9Aqexw0CS7Wivy6DdNTGpHNsOybkoisYqEDeYaDN_zuSYmznXafHCwT-FgULG5_GSI2d9kU__07ht6bD0RzHr-Jk%3D&xkcb=SoDX6_M3hwflOvyN8p0KbzkdCdPP&continueUrl=%2Fjobs%3Fq%3Dsoftware%2Bengineer%26l%3DRemote", + "vjFeaturedEmployerCandidate": false + }, + { + "adBlob": "-6NYlbfkN0CnM4TERr6XuVBW3VTRACvDXDx9S3Pbn6a0SwhkmoLTiLCFUpITpg4QVB7SR39MoP0BLR841cgbHYPJopI9fiQa67MfGt-TS3gF6m2xSij4aAarTk90jQV5vKgeHuaG-dJ4P0bDI4Zj_HA51wzDk11uApXr5YRiDtyBwiQNLf5d9Lyctn13orSbPQeRCVH0zqauP5hiYL4ZkfdYeGe3I1Wbq855uSspMHp95AmMxgY4r8gBmfjoH2ZckgGERlYYLnGOotBKSI7i1CVIHtUDpAaF8MlAFpdDsF7JJ3NyS5QmqxHMQDQvH-MTyCDpC3bADAnfw6-khs-18fT3MQkZxaiHCnFCeCieu5HX8FLJW0Vbld4azqHzvV6iSucVFTJi1AKqwERerK9DWIfC0zose7qxVHyZkCqyCO7MtebmgcFgz8G-TMEKd3KEBdjQHW4g989DJ5vUEw0r_96qW6vxxph2br8GDqh0gRdNttZbE8iRAL2dJQGEiGkx-SEOJbw4SoVYUwpjBejBIJ-rdG4cyC5ppwrlxPXTlnAcqq9Ie16k5dyg-_YEGJgAXpTkGiS_jUefF4WFy9TLA5mJmWV1UGK991Seolocw55nIqu8uewnGZrKlzn_URF3fqCCkPEu048MFp3V5B32llPFdw6ecDZM4ixH_Y6Ex2ja_KQrD-VhNLIsQT0l0xiRsuD4NxCSVzIO3oh4AR-qxkF088JN5s0b7vYNrPpc20UUPEMmP9UTsO9TnPKrxhEPZX0qoIvEdXQwWA__4p652c_mTlKFK9F-VOq0Af-rMFWJFT2H9Cstjh4J7RyLp70DgUD5IJYLysP3eEEvknyDAOnB3IbnpMF4vePgU0vTSD-QHvvd-GxNquZ1d-hGjAfcdG70mwq736OitfXAGrZBSH7tm4yOCxSllF5t1z0TIE-LuM1UKKvYFt1H8B8TI-mhgtXwx6FhLAytzRKYOQlO5A==", + "adId": "461480873", + "advn": "7831672186689475", + "appliedOrGreater": false, + "applyCount": -1, + "assistedApply": false, + "autoSourcerJob": false, + "bidPosition": 3, + "blobKey": "SoBK6_M3hwflOvyN8p0JbzkdCdPP", + "company": "DataAnnotation", + "companyBrandingAttributes": { + "brandingReasons": [ + "PAID_BRANDING" + ], + "brandingReasonsAsString": [ + "PAID_BRANDING" + ], + "headerImageUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_headerimage/1960x400/e42fbb2b9f70343758932438638f973c", + "logoUrl": "https://d2q79iu7y748jz.cloudfront.net/s/_squarelogo/256x256/cc8dc489da5211695773d9f9020812b0", + "paidBrandingDealIds": [ + "88129" + ], + "showJobBranding": true, + "shownForBrandedJobPackage": false + }, + "companyIdEncrypted": "342de9268ba1a972", + "companyOverviewLink": "/cmp/Dataannotation", + "companyOverviewLinkCampaignId": "serp-linkcompanyname-content", + "companyRating": 4.1, + "companyReviewCount": 1661, + "companyReviewLink": "/cmp/Dataannotation/reviews", + "companyReviewLinkCampaignId": "cmplinktst2", + "country": "US", + "createDate": 1778281962938, + "d2iEnabled": false, + "displayTitle": "Java Developer - AI Trainer", + "dradisJob": false, + "employerAssistEnabled": false, + "employerResponsive": false, + "encryptedFccompanyId": "e3735f59dc6a6354", + "encryptedResultData": "VwIPTVJ1cTn5AN7Q-tSqGRXGNe2wB2UYx73qSczFnGU", + "enhancedAttributesModel": {}, + "enticers": [], + "expired": false, + "extractTrackingUrls": "", + "extractedSalary": { + "max": 100, + "min": 0, + "type": "HOURLY" + }, + "fccompanyId": -1, + "featuredCompanyAttributes": {}, + "featuredEmployer": false, + "featuredEmployerCandidate": false, + "feedId": 915432, + "formattedLocation": "Remote", + "formattedRelativeTime": "30+ days ago", + "gatedVjp": false, + "hideMetaData": false, + "highVolumeHiringModel": { + "highVolumeHiring": false + }, + "hiringEventJob": false, + "homepageJobFeedSectionId": "0", + "indeedApplyEnabled": true, + "indeedApplyFinishAppUrlEnabled": true, + "indeedApplyResumeType": "required", + "indeedApplyable": true, + "isJobVisited": false, + "isMobileThirdPartyApplyable": false, + "isNoResumeJob": false, + "isSubsidiaryJob": false, + "isTopRatedEmployer": false, + "jobCardRequirementsModel": { + "additionalRequirementsCount": 0, + "jobOnlyRequirements": [], + "jobTagRequirements": [], + "requirementsHeaderShown": false, + "screenerQuestionRequirements": [] + }, + "jobFlairPackageEnabled": false, + "jobLocationCity": "Remote", + "jobSeekerMatchSummaryModel": { + "sortedEntityDisplayText": [], + "sortedMatchingEntityDisplayText": [], + "sortedMisMatchingEntityDisplayText": [ + "AI models", + "Bachelor's degree", + "C#", + "Data analytics", + "Data visualization", + "English", + "Go", + "Grammar Experience", + "JavaScript frameworks", + "Kotlin", + "Model training", + "Providing code feedback", + "Software coding", + "Software design", + "Software engineering", + "Swift", + "TypeScript", + "Writing skills" + ], + "taxoEntityMatchesNegative": [ + { + "displayText": "Software design", + "id": "", + "rawName": "Software design", + "source": "JOB", + "strictness": "", + "suid": "7DZKQ" + }, + { + "displayText": "Go", + "id": "", + "rawName": "Go", + "source": "JOB", + "strictness": "", + "suid": "6ETQT" + }, + { + "displayText": "JavaScript frameworks", + "id": "", + "rawName": "JavaScript frameworks", + "source": "JOB", + "strictness": "", + "suid": "EUPT5" + }, + { + "displayText": "English", + "id": "", + "rawName": "English", + "source": "JOB", + "strictness": "", + "suid": "D866K" + }, + { + "displayText": "TypeScript", + "id": "", + "rawName": "TypeScript", + "source": "JOB", + "strictness": "", + "suid": "WD7PP" + }, + { + "displayText": "Model training", + "id": "", + "rawName": "Model training", + "source": "JOB", + "strictness": "", + "suid": "QZ89D" + }, + { + "displayText": "AI models", + "id": "", + "rawName": "AI models", + "source": "JOB", + "strictness": "", + "suid": "22JPW" + }, + { + "displayText": "Kotlin", + "id": "", + "rawName": "Kotlin", + "source": "JOB", + "strictness": "", + "suid": "4KE5P" + }, + { + "displayText": "Data visualization", + "id": "", + "rawName": "Data visualization", + "source": "JOB", + "strictness": "", + "suid": "TKG4S" + }, + { + "displayText": "Bachelor's degree", + "id": "", + "rawName": "Bachelor's degree", + "source": "JOB", + "strictness": "", + "suid": "HFDVW" + }, + { + "displayText": "Software engineering", + "id": "", + "rawName": "Software engineering", + "source": "JOB", + "strictness": "", + "suid": "4R4AM" + }, + { + "displayText": "Data analytics", + "id": "", + "rawName": "Data analytics", + "source": "JOB", + "strictness": "", + "suid": "NGW4T" + }, + { + "displayText": "Software coding", + "id": "", + "rawName": "Software coding", + "source": "JOB", + "strictness": "", + "suid": "9MWQS" + }, + { + "displayText": "Writing skills", + "id": "", + "rawName": "Writing skills", + "source": "JOB", + "strictness": "", + "suid": "A7SFW" + }, + { + "displayText": "Grammar Experience", + "id": "", + "rawName": "Grammar Experience", + "source": "JOB", + "strictness": "", + "suid": "VTSX9" + }, + { + "displayText": "Providing code feedback", + "id": "", + "rawName": "Providing code feedback", + "source": "JOB", + "strictness": "", + "suid": "Z4F5N" + }, + { + "displayText": "C#", + "id": "", + "rawName": "C#", + "source": "JOB", + "strictness": "", + "suid": "AP3T4" + }, + { + "displayText": "Swift", + "id": "", + "rawName": "Swift", + "source": "JOB", + "strictness": "", + "suid": "U5AXZ" + } + ], + "taxoEntityMatchesPositive": [], + "trafficLight": "yellow" + }, + "jobTypes": [], + "jobkey": "40da1d788bab69b0", + "jsiEnabled": false, + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0CnM4TERr6XuVBW3VTRACvDXDx9S3Pbn6a0SwhkmoLTiLCFUpITpg4QVB7SR39MoP0BLR841cgbHYPJopI9fiQa67MfGt-TS3gF6m2xSij4aAarTk90jQV5vKgeHuaG-dJ4P0bDI4Zj_HA51wzDk11uApXr5YRiDtyBwiQNLf5d9Lyctn13orSbPQeRCVH0zqauP5hiYL4ZkfdYeGe3I1Wbq855uSspMHp95AmMxgY4r8gBmfjoH2ZckgGERlYYLnGOotBKSI7i1CVIHtUDpAaF8MlAFpdDsF7JJ3NyS5QmqxHMQDQvH-MTyCDpC3bADAnfw6-khs-18fT3MQkZxaiHCnFCeCieu5HX8FLJW0Vbld4azqHzvV6iSucVFTJi1AKqwERerK9DWIfC0zose7qxVHyZkCqyCO7MtebmgcFgz8G-TMEKd3KEBdjQHW4g989DJ5vUEw0r_96qW6vxxph2br8GDqh0gRdNttZbE8iRAL2dJQGEiGkx-SEOJbw4SoVYUwpjBejBIJ-rdG4cyC5ppwrlxPXTlnAcqq9Ie16k5dyg-_YEGJgAXpTkGiS_jUefF4WFy9TLA5mJmWV1UGK991Seolocw55nIqu8uewnGZrKlzn_URF3fqCCkPEu048MFp3V5B32llPFdw6ecDZM4ixH_Y6Ex2ja_KQrD-VhNLIsQT0l0xiRsuD4NxCSVzIO3oh4AR-qxkF088JN5s0b7vYNrPpc20UUPEMmP9UTsO9TnPKrxhEPZX0qoIvEdXQwWA__4p652c_mTlKFK9F-VOq0Af-rMFWJFT2H9Cstjh4J7RyLp70DgUD5IJYLysP3eEEvknyDAOnB3IbnpMF4vePgU0vTSD-QHvvd-GxNquZ1d-hGjAfcdG70mwq736OitfXAGrZBSH7tm4yOCxSllF5t1z0TIE-LuM1UKKvYFt1H8B8TI-mhgtXwx6FhLAytzRKYOQlO5A==&xkcb=SoBK6_M3hwflOvyN8p0JbzkdCdPP&p=2&camk=C3EPSzFlQw95L2GYPjH5nw==&vjs=3", + "locationCount": 1, + "minimumCount": 1, + "mobtk": "1jtgbqmn921k6001", + "moreLocUrl": "/jobs?q=software+engineer&nl=&l=Remote&radius=50&jtid=c1b76b0304ad72c6&jcid=342de9268ba1a972&grp=tcl", + "mouseDownHandlerOption": { + "adId": 461480873, + "advn": "7831672186689475", + "extractTrackingUrls": [], + "from": "vjs", + "jobKey": "40da1d788bab69b0", + "link": "/pagead/clk?mo=r&ad=-6NYlbfkN0CnM4TERr6XuVBW3VTRACvDXDx9S3Pbn6a0SwhkmoLTiLCFUpITpg4QVB7SR39MoP0BLR841cgbHYPJopI9fiQa67MfGt-TS3gF6m2xSij4aAarTk90jQV5vKgeHuaG-dJ4P0bDI4Zj_HA51wzDk11uApXr5YRiDtyBwiQNLf5d9Lyctn13orSbPQeRCVH0zqauP5hiYL4ZkfdYeGe3I1Wbq855uSspMHp95AmMxgY4r8gBmfjoH2ZckgGERlYYLnGOotBKSI7i1CVIHtUDpAaF8MlAFpdDsF7JJ3NyS5QmqxHMQDQvH-MTyCDpC3bADAnfw6-khs-18fT3MQkZxaiHCnFCeCieu5HX8FLJW0Vbld4azqHzvV6iSucVFTJi1AKqwERerK9DWIfC0zose7qxVHyZkCqyCO7MtebmgcFgz8G-TMEKd3KEBdjQHW4g989DJ5vUEw0r_96qW6vxxph2br8GDqh0gRdNttZbE8iRAL2dJQGEiGkx-SEOJbw4SoVYUwpjBejBIJ-rdG4cyC5ppwrlxPXTlnAcqq9Ie16k5dyg-_YEGJgAXpTkGiS_jUefF4WFy9TLA5mJmWV1UGK991Seolocw55nIqu8uewnGZrKlzn_URF3fqCCkPEu048MFp3V5B32llPFdw6ecDZM4ixH_Y6Ex2ja_KQrD-VhNLIsQT0l0xiRsuD4NxCSVzIO3oh4AR-qxkF088JN5s0b7vYNrPpc20UUPEMmP9UTsO9TnPKrxhEPZX0qoIvEdXQwWA__4p652c_mTlKFK9F-VOq0Af-rMFWJFT2H9Cstjh4J7RyLp70DgUD5IJYLysP3eEEvknyDAOnB3IbnpMF4vePgU0vTSD-QHvvd-GxNquZ1d-hGjAfcdG70mwq736OitfXAGrZBSH7tm4yOCxSllF5t1z0TIE-LuM1UKKvYFt1H8B8TI-mhgtXwx6FhLAytzRKYOQlO5A==&xkcb=SoBK6_M3hwflOvyN8p0JbzkdCdPP&p=2&camk=C3EPSzFlQw95L2GYPjH5nw==&vjs=3", + "tk": "1jtgbqmn921k6001" + }, + "newJob": false, + "normTitle": "Java Developer", + "numHires": 1096, + "openInterviewsInterviewsOnTheSpot": false, + "openInterviewsJob": false, + "openInterviewsOffersOnTheSpot": false, + "openInterviewsPhoneJob": false, + "organicApplyStartCount": 47202, + "overrideIndeedApplyText": true, + "packageTier": "NONE", + "preciseLocationModel": { + "obfuscateLocation": false, + "overrideJCMPreciseLocationModel": true + }, + "pubDate": 1778216400000, + "rankedBenefits": { + "GENERIC": [ + "WZ9TD" + ], + "OCCUPATION_TFIDF": [ + "WZ9TD" + ] + }, + "rankingScoresModel": { + "bid": 106902, + "bidPosition": 3, + "eApply": 0.014480283, + "eAttainability": 0, + "eQualified": 0 + }, + "recommendationReasonModel": { + "reason": null + }, + "redirectToThirdPartySite": false, + "remoteLocation": true, + "remoteWorkModel": { + "inlineText": true, + "text": "Remote", + "type": "REMOTE_ALWAYS" + }, + "resultBeforeExpansion": false, + "resumeMatch": false, + "salarySnippet": { + "currency": "USD", + "salaryTextFormatted": false, + "source": "EXTRACTION", + "text": "Up to $100 an hour" + }, + "saved": false, + "savedApplication": false, + "screenerQuestionsURL": "iqp://INDEXEDJOB/10128416810?sourceUrl=https%3A%2F%2Fapp.dataannotation.tech%2Fpartners%2Findeed%2Fscreener-questions%2FPROG_SA&country=US&language=en", + "searchUID": "1jtgbqmn921k6001", + "showCommutePromo": false, + "showEarlyApply": false, + "showJobType": false, + "showRelativeDate": true, + "showSponsoredLabel": false, + "showStrongerAppliedLabel": false, + "smartFillEnabled": false, + "smbD2iEnabled": false, + "snippet": "
    \n
  • Help train AI chatbots while gaining the flexibility of remote work and choosing your own schedule.
    • \n
  • We are looking for an existing Java Developer - AI Trainer\u2026
    • \n
", + "sourceId": 44932918, + "sponsored": true, + "taxoAttributes": [ + "Hourly pay" + ], + "taxoAttributesDisplayLimit": 3, + "taxoLogAttributes": [ + "8AD9K" + ], + "taxonomyAttributes": [ + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types" + }, + { + "attributes": [], + "label": "shifts" + }, + { + "attributes": [ + { + "label": "Remote", + "suid": "DSQF7" + } + ], + "label": "remote" + }, + { + "attributes": [ + { + "label": "Flexible schedule", + "suid": "WZ9TD" + } + ], + "label": "benefits" + }, + { + "attributes": [ + { + "label": "Full-time", + "suid": "CF3CP" + } + ], + "label": "job-types-cc" + }, + { + "attributes": [ + { + "label": "On demand", + "suid": "KPAXE" + } + ], + "label": "schedules" + } + ], + "thirdPartyApplyUrl": "http://www.indeed.com//applystart?jk=40da1d788bab69b0&from=jobsearch&mvj=0&jobsearchTk=1jtgbqmn921k6001&spon=1&adid=461480873&ad=-6NYlbfkN0CnM4TERr6XuVBW3VTRACvDXDx9S3Pbn6a0SwhkmoLTiLCFUpITpg4QVB7SR39MoP0BLR841cgbHYPJopI9fiQa67MfGt-TS3gF6m2xSij4aAarTk90jQV5vKgeHuaG-dJ4P0bDI4Zj_HA51wzDk11uApXr5YRiDtyBwiQNLf5d9Lyctn13orSbPQeRCVH0zqauP5hiYL4ZkfdYeGe3I1Wbq855uSspMHp95AmMxgY4r8gBmfjoH2ZckgGERlYYLnGOotBKSI7i1CVIHtUDpAaF8MlAFpdDsF7JJ3NyS5QmqxHMQDQvH-MTyCDpC3bADAnfw6-khs-18fT3MQkZxaiHCnFCeCieu5HX8FLJW0Vbld4azqHzvV6iSucVFTJi1AKqwERerK9DWIfC0zose7qxVHyZkCqyCO7MtebmgcFgz8G-TMEKd3KEBdjQHW4g989DJ5vUEw0r_96qW6vxxph2br8GDqh0gRdNttZbE8iRAL2dJQGEiGkx-SEOJbw4SoVYUwpjBejBIJ-rdG4cyC5ppwrlxPXTlnAcqq9Ie16k5dyg-_YEGJgAXpTkGiS_jUefF4WFy9TLA5mJmWV1UGK991Seolocw55nIqu8uewnGZrKlzn_URF3fqCCkPEu048MFp3V5B32llPFdw6ecDZM4ixH_Y6Ex2ja_KQrD-VhNLIsQT0l0xiRsuD4NxCSVzIO3oh4AR-qxkF088JN5s0b7vYNrPpc20UUPEMmP9UTsO9TnPKrxhEPZX0qoIvEdXQwWA__4p652c_mTlKFK9F-VOq0Af-rMFWJFT2H9Cstjh4J7RyLp70DgUD5IJYLysP3eEEvknyDAOnB3IbnpMF4vePgU0vTSD-QHvvd-GxNquZ1d-hGjAfcdG70mwq736OitfXAGrZBSH7tm4yOCxSllF5t1z0TIE-LuM1UKKvYFt1H8B8TI-mhgtXwx6FhLAytzRKYOQlO5A%3D%3D&xkcb=SoBK6_M3hwflOvyN8p0JbzkdCdPP&sjdu=Xbbwb0dbVlAPs0TIG7JumKleoDYenFeEkrm-BS2RFumAMJVRHnJnPXPQ1w4iVi_Bv12mcQcytWIaRGXVXpPFAJNHnyyAVakdybx_o10fpZKWxZ_XE6bUgpYqVY9LTx2dtS2-wQGVvKfiJXE4sfR0wlJijxkADwfaIzsQvfCpeWTbIoPN703ijAiPmVd-0iRuRqKLUPp4vg4qLycQPkInGwEJ8mAfOsifjoMlENy_ja0-fFKe71EIu7Z9bTd7syIkw1kTIG9N_3xV2F-5GZy3Y5T0JNFK-0qbOVkES4SHQ0SfH4NnnAa-f9nnHhIvvjmnptIrPbrObVWz49O6dy2-oLq1ZJUrsmKgOZzFKNCYYNCUbKfW-PqPLgNFoMJ1bqVan-wqq3U_828sw_TKiEbPLGBrX0Smwsv_nfNl645h88Ku6u2wUb1_sbeLBjNQYL-D2HF0NWgGwnwCdIgq-ovHBLz2azumxzGoE7sQwq5WgQU6Ejv27DnhAx3OksPVD9Q467ScvhtHyYiswIKxB_GqzPnKFzVfX6KlLlncqFzPtAdqUh6kbE--RtzZcE3CpSZy49vbhsBa3mK5jcKTo3kAW8OQHj80D0t3tmm_5Bz3HzmhEYtbc0mGitHUCyNLpy3-KmOMQ52bBemf2QzlsBwZM_s0P_i5BUDL-dg7xq4JOcYWtazSiL6AGCmfQ3-IZ66NzjFoAZecxXabAkK3Z3yS8YOicXGVy9B8QF5rlp7N10RoPP4JgHEDSdDTCZF8Vqkr81cMtwirsl6Ces3MDG_-lU8Qog1PeQs_BLn7VE7fnL8sFLH5stKSKIMHqtDXLFkv6ca-q2HsMP3KDd9l8Thw_GraXIfi1zP_-1dJJVesjBKRddONqBtekJ0dg4wz_0QGaCgpsJqAKTEWNu1jykb1WSIDsEyBoXQ6XBTbTgoVJg9W_pQ9pnl27ulNi4MRUUOuGkAhqGRX-suSwgxeatDCu6YGnLoNIz5_owj5ofazOCNHfLb8VF1mNhHIbmk41BvwrTMMyFcwLRwY5Zl6HRqlWTvnfRX_o71GiRqG-mo84veWnqey2tja4ynd5oGGEpJW9fAHiF_dYhtRE6_ZSH5JlkPuii5VUVatLa1wS1_UqHMTfm4yTZQ3k9qHVV5U6scDQUCqNBl8S_CEc8NZr2mHRzBoow79YUEPkNKI4KY03N1pG38fj16ZXK2GuKr1y3OpndPSDTO2ELdmaqpPn6H5V1YnWYL95pWEI_Zy8NrxXVDtYzPclQ0xpJHTWANEIb5vatND9npl3p-ADGXXrn291chTmlJv9lkd31Zosvlcvtn6BtL5lqZmS4Z29H6UBAhS-m4v2dwNroX8_PDn_lwGbwDY1d5uBB3EHNFQRlK-uGR-5UJ1eDDjSJ6uQi4hk6R-Z8H9h0Q5tm1cqEFUnMMiQpxZ6pdvi2b3fmxnjLYK0Ro&vjfrom=vjs&astse=da7fff3285780cc4&assa=589", + "tier": { + "matchedPreferences": { + "longMatchedPreferences": [], + "stringMatchedPreferences": [] + }, + "type": "DEFAULT" + }, + "title": "Java Developer - AI Trainer", + "translatedAttributes": [], + "translatedCmiJobTags": [], + "truncatedCompany": "DataAnnotation", + "urgentlyHiring": false, + "viewJobLink": "/viewjob?jk=40da1d788bab69b0&from=vjs&tk=1jtgbqmn921k6001&viewtype=embedded&advn=7831672186689475&adid=461480873&ad=-6NYlbfkN0CnM4TERr6XuVBW3VTRACvDXDx9S3Pbn6a0SwhkmoLTiLCFUpITpg4QVB7SR39MoP0BLR841cgbHYPJopI9fiQa67MfGt-TS3gF6m2xSij4aAarTk90jQV5vKgeHuaG-dJ4P0bDI4Zj_HA51wzDk11uApXr5YRiDtyBwiQNLf5d9Lyctn13orSbPQeRCVH0zqauP5hiYL4ZkfdYeGe3I1Wbq855uSspMHp95AmMxgY4r8gBmfjoH2ZckgGERlYYLnGOotBKSI7i1CVIHtUDpAaF8MlAFpdDsF7JJ3NyS5QmqxHMQDQvH-MTyCDpC3bADAnfw6-khs-18fT3MQkZxaiHCnFCeCieu5HX8FLJW0Vbld4azqHzvV6iSucVFTJi1AKqwERerK9DWIfC0zose7qxVHyZkCqyCO7MtebmgcFgz8G-TMEKd3KEBdjQHW4g989DJ5vUEw0r_96qW6vxxph2br8GDqh0gRdNttZbE8iRAL2dJQGEiGkx-SEOJbw4SoVYUwpjBejBIJ-rdG4cyC5ppwrlxPXTlnAcqq9Ie16k5dyg-_YEGJgAXpTkGiS_jUefF4WFy9TLA5mJmWV1UGK991Seolocw55nIqu8uewnGZrKlzn_URF3fqCCkPEu048MFp3V5B32llPFdw6ecDZM4ixH_Y6Ex2ja_KQrD-VhNLIsQT0l0xiRsuD4NxCSVzIO3oh4AR-qxkF088JN5s0b7vYNrPpc20UUPEMmP9UTsO9TnPKrxhEPZX0qoIvEdXQwWA__4p652c_mTlKFK9F-VOq0Af-rMFWJFT2H9Cstjh4J7RyLp70DgUD5IJYLysP3eEEvknyDAOnB3IbnpMF4vePgU0vTSD-QHvvd-GxNquZ1d-hGjAfcdG70mwq736OitfXAGrZBSH7tm4yOCxSllF5t1z0TIE-LuM1UKKvYFt1H8B8TI-mhgtXwx6FhLAytzRKYOQlO5A%3D%3D&xkcb=SoBK6_M3hwflOvyN8p0JbzkdCdPP&continueUrl=%2Fjobs%3Fq%3Dsoftware%2Bengineer%26l%3DRemote", + "vjFeaturedEmployerCandidate": false + } + ], + "pageNumber": 1, + "what": "software engineer", + "where": "Remote" + } + } +} \ No newline at end of file From 37963c64f1bd13c6e365e6e19dae38b6b0213bdf Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 014/217] test: indeed parsers --- .../platforms/indeed_jobs/test_parsers.py | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py new file mode 100644 index 000000000..7b9ca6298 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py @@ -0,0 +1,139 @@ +"""Offline parser tests: synthetic mapping plus a real captured blob.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from app.proprietary.platforms.indeed_jobs.parsers import ( + extract_jobcards_blob, + job_results, + parse_job, +) + +_FIXTURE_DIR = Path(__file__).parent / "fixtures" + + +# --- synthetic mapping (always runs) --------------------------------------- + + +def _raw_job() -> dict: + return { + "jobkey": "abc123", + "displayTitle": "Senior Data Analyst", + "title": "Senior Data Analyst (fallback)", + "company": "Acme Corp", + "truncatedCompany": "Acme", + "companyOverviewLink": "/cmp/Acme-Corp", + "companyRating": 4.1, + "companyReviewCount": 320, + "formattedLocation": "New York, NY", + "jobLocationCity": "New York", + "jobLocationState": "NY", + "jobLocationPostal": "10001", + "country": "US", + "remoteLocation": False, + "remoteWorkModel": {"type": "REMOTE_ALWAYS", "text": "Remote"}, + "jobTypes": [], + "salarySnippet": {"currency": "USD", "text": "$90,000 - $120,000 a year"}, + "extractedSalary": {"min": 90000, "max": 120000, "type": "YEARLY"}, + "snippet": "
  • 5+ years SQL & Python
", + "sponsored": True, + "newJob": False, + "urgentlyHiring": True, + "expired": False, + "indeedApplyEnabled": True, + "formattedRelativeTime": "3 days ago", + "pubDate": 1_774_242_000_000, + "createDate": 1_774_276_267_415, + "thirdPartyApplyUrl": "https://ats.example.com/apply/abc123", + "taxonomyAttributes": [ + {"label": "job-types", "attributes": [{"label": "Full-time", "suid": "x"}]}, + {"label": "remote", "attributes": [{"label": "Remote", "suid": "y"}]}, + { + "label": "benefits", + "attributes": [ + {"label": "Health insurance", "suid": "a"}, + {"label": "401(k)", "suid": "b"}, + ], + }, + ], + } + + +def test_parse_job_maps_core_fields(): + item = parse_job(_raw_job()) + assert item["jobKey"] == "abc123" + assert item["title"] == "Senior Data Analyst" # displayTitle wins over title + assert item["jobUrl"] == "https://www.indeed.com/viewjob?jk=abc123" + assert item["applyUrl"] == "https://ats.example.com/apply/abc123" + assert item["company"] == "Acme Corp" + assert item["companyUrl"] == "https://www.indeed.com/cmp/Acme-Corp" + assert item["companyReviewCount"] == 320 + assert item["city"] == "New York" + assert item["isRemote"] is True + assert item["remoteType"] == "REMOTE_ALWAYS" + assert item["jobTypes"] == ["Full-time"] + assert item["benefits"] == ["Health insurance", "401(k)"] + assert item["sponsored"] is True + assert item["urgentlyHiring"] is True + + +def test_parse_job_salary_and_snippet(): + item = parse_job(_raw_job()) + sal = item["salary"] + assert sal["salaryText"] == "$90,000 - $120,000 a year" + assert sal["salaryMin"] == 90000 + assert sal["salaryMax"] == 120000 + assert sal["currency"] == "USD" + assert sal["period"] == "year" + assert sal["isEstimated"] is False + # snippet HTML is stripped + entities decoded into plain text. + assert item["descriptionText"] == "5+ years SQL & Python" + assert item["descriptionHtml"] is None + + +def test_parse_job_dates_from_epoch_ms(): + item = parse_job(_raw_job()) + assert item["datePublished"] == "2026-03-23T05:00:00.000Z" + assert item["age"] == "3 days ago" + + +def test_parse_job_respects_base_url(): + item = parse_job(_raw_job(), base_url="https://uk.indeed.com") + assert item["jobUrl"] == "https://uk.indeed.com/viewjob?jk=abc123" + assert item["companyUrl"] == "https://uk.indeed.com/cmp/Acme-Corp" + + +def test_extract_blob_anchors_on_assignment_not_first_occurrence(): + # Decoy mention precedes the real assignment; the extractor must skip it. + html = ( + '' + '' + ) + blob = extract_jobcards_blob(html) + results = job_results(blob) + assert [r["jobkey"] for r in results] == ["k1", "k2"] + + +def test_extract_blob_missing_returns_none(): + assert extract_jobcards_blob("just a moment...") is None + assert job_results(None) == [] + + +# --- fixture-pinned (real captured blob) ----------------------------------- + + +def test_fixture_blob_parses_into_items(): + fixture = _FIXTURE_DIR / "sample_jobcards.json" + blob = json.loads(fixture.read_text()) + results = job_results(blob) + assert len(results) == 3 + for raw in results: + item = parse_job(raw) + assert isinstance(item["jobKey"], str) and item["jobKey"] + assert item["title"] + assert item["jobUrl"].startswith("https://www.indeed.com/viewjob?jk=") + assert "salaryText" in item["salary"] From 91e868fc26143b1307dec9cdb505d12ccb52abea Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 015/217] test: indeed url resolver --- .../indeed_jobs/test_url_resolver.py | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/test_url_resolver.py diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_url_resolver.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_url_resolver.py new file mode 100644 index 000000000..b995b3d0b --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_url_resolver.py @@ -0,0 +1,87 @@ +"""Offline tests for Indeed URL classification and search-URL building.""" + +from __future__ import annotations + +from urllib.parse import parse_qs, urlparse + +from app.proprietary.platforms.indeed_jobs.url_resolver import ( + build_search_url, + country_domain, + resolve_url, +) + + +def test_resolve_search_url(): + r = resolve_url( + "https://www.indeed.com/jobs?q=software+engineer&l=Remote&sort=date" + ) + assert r is not None + assert r.kind == "search" + assert r.value == "software engineer" + assert r.location == "Remote" + assert r.domain == "www.indeed.com" + assert r.params.get("sort") == "date" + + +def test_resolve_company_url(): + r = resolve_url("https://www.indeed.com/cmp/Google/jobs") + assert r is not None + assert r.kind == "company" + assert r.value == "Google" + + +def test_resolve_viewjob_url(): + r = resolve_url("https://uk.indeed.com/viewjob?jk=abc123&from=serp") + assert r is not None + assert r.kind == "job" + assert r.value == "abc123" + assert r.domain == "uk.indeed.com" + + +def test_resolve_country_subdomain_host(): + r = resolve_url("https://de.indeed.com/jobs?q=entwickler") + assert r is not None + assert r.kind == "search" + assert r.domain == "de.indeed.com" + + +def test_resolve_rejects_non_indeed(): + assert resolve_url("https://www.linkedin.com/jobs?q=dev") is None + assert resolve_url("https://notindeed.com.evil.com/jobs") is None + + +def test_country_domain_map(): + assert country_domain("us") == "www.indeed.com" + assert country_domain("gb") == "uk.indeed.com" + assert country_domain("de") == "de.indeed.com" + assert country_domain("") == "www.indeed.com" + + +def test_build_search_url_basic(): + url = build_search_url( + "data analyst", + country="us", + location="New York, NY", + sort="date", + start=20, + ) + parsed = urlparse(url) + qs = parse_qs(parsed.query) + assert parsed.netloc == "www.indeed.com" + assert parsed.path == "/jobs" + assert qs["q"] == ["data analyst"] + assert qs["l"] == ["New York, NY"] + assert qs["sort"] == ["date"] + assert qs["start"] == ["20"] + + +def test_build_search_url_remote_keyword_fallback_and_jobtype(): + url = build_search_url( + "developer", country="gb", remote="remote", job_type="fulltime", from_days=7 + ) + parsed = urlparse(url) + qs = parse_qs(parsed.query) + assert parsed.netloc == "uk.indeed.com" + assert qs["q"] == ["developer remote"] + assert qs["jt"] == ["fulltime"] + assert qs["fromage"] == ["7"] From a60aba112c52ae744d9d20204548bea65188d198 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 016/217] test: indeed fetch rotation resilience --- .../indeed_jobs/test_fetch_resilience.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py new file mode 100644 index 000000000..4eae65f84 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py @@ -0,0 +1,95 @@ +"""Offline tests for the rotate-on-block fetch loop (no network, fake session).""" + +from __future__ import annotations + +import pytest + +from app.proprietary.platforms.indeed_jobs.fetch import ( + IndeedAccessBlockedError, + IndeedSession, +) + +_OK_HTML = "jobs listing" +_BLOCK_HTML = "secure.indeed.com security check" + + +class _FakePage: + def __init__(self, html: str, url: str) -> None: + self.html_content = html + self.url = url + + +class _Controller: + """Shared state across sessions the factory hands out (survives rotation).""" + + def __init__(self, target_outcomes: list[str]) -> None: + self.target_outcomes = target_outcomes + self.sessions_started = 0 + self.home_fetches: dict[str, int] = {} + self.target_index = 0 + + def factory(self) -> _FakeSession: + return _FakeSession(self) + + +class _FakeSession: + def __init__(self, ctrl: _Controller) -> None: + self._ctrl = ctrl + + async def start(self) -> None: + self._ctrl.sessions_started += 1 + + async def close(self) -> None: + pass + + async def fetch(self, url: str, **_: object) -> _FakePage: + if url.endswith("/") and "/jobs" not in url: # warm-up hit + self._ctrl.home_fetches[url] = self._ctrl.home_fetches.get(url, 0) + 1 + return _FakePage("home", url) + outcome = self._ctrl.target_outcomes[self._ctrl.target_index] + self._ctrl.target_index += 1 + if outcome == "OK": + return _FakePage(_OK_HTML, url) + if outcome == "ERROR": + raise RuntimeError("boom") + return _FakePage(_BLOCK_HTML, "https://secure.indeed.com/auth") + + +_URL = "https://www.indeed.com/jobs?q=dev" + + +@pytest.mark.asyncio +async def test_rotates_past_a_block_then_succeeds(): + ctrl = _Controller(["BLOCK", "OK"]) + session = IndeedSession(ctrl.factory) + html = await session.fetch_html(_URL) + assert html == _OK_HTML + assert session.rotations == 1 + assert ctrl.sessions_started == 2 # initial + one rotation + + +@pytest.mark.asyncio +async def test_recovers_after_a_fetch_error(): + ctrl = _Controller(["ERROR", "OK"]) + session = IndeedSession(ctrl.factory) + assert await session.fetch_html(_URL) == _OK_HTML + assert session.rotations == 1 + + +@pytest.mark.asyncio +async def test_raises_after_exhausting_rotations(): + ctrl = _Controller(["BLOCK"] * 10) + session = IndeedSession(ctrl.factory) + with pytest.raises(IndeedAccessBlockedError): + await session.fetch_html(_URL) + assert session.rotations == 3 + + +@pytest.mark.asyncio +async def test_warms_domain_once_without_rotation(): + ctrl = _Controller(["OK", "OK"]) + session = IndeedSession(ctrl.factory) + await session.fetch_html(_URL) + await session.fetch_html(_URL + "&start=10") + assert ctrl.home_fetches["https://www.indeed.com/"] == 1 + await session.close() From 063faf4bd6f92e7851ad76a725d53e68eca3d586 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:42:00 +0200 Subject: [PATCH 017/217] test: indeed scraper orchestration --- .../platforms/indeed_jobs/test_scraper.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py new file mode 100644 index 000000000..c56d64f75 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py @@ -0,0 +1,105 @@ +"""Offline orchestration tests: pagination, dedupe, and caps via a fake session.""" + +from __future__ import annotations + +import json +from urllib.parse import parse_qs, urlparse + +import pytest + +from app.proprietary.platforms.indeed_jobs.schemas import IndeedScrapeInput +from app.proprietary.platforms.indeed_jobs.scraper import iter_indeed, scrape_indeed + + +def _page_html(job_keys: list[str]) -> str: + """Wrap job keys in the ``mosaic-provider-jobcards`` assignment shape.""" + results = [ + {"jobkey": k, "displayTitle": f"Job {k}", "company": "Acme"} for k in job_keys + ] + model = {"metaData": {"mosaicProviderJobCardsModel": {"results": results}}} + return ( + f'window.mosaic.providerData["mosaic-provider-jobcards"]={json.dumps(model)};' + ) + + +class _FakeSession: + """Returns per-``start`` pages and records the URLs requested.""" + + def __init__(self, pages: dict[int, list[str]]) -> None: + self._pages = pages + self.fetched: list[str] = [] + + async def fetch_html(self, url: str) -> str: + self.fetched.append(url) + start = int(parse_qs(urlparse(url).query).get("start", ["0"])[0]) + return _page_html(self._pages.get(start, [])) + + +async def _collect(input_model, session) -> list[dict]: + return [item async for item in iter_indeed(input_model, session)] + + +@pytest.mark.asyncio +async def test_paginates_and_dedupes_across_pages(): + session = _FakeSession({0: ["k1", "k2", "k3"], 10: ["k3", "k4"], 20: []}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2", "k3", "k4"] + assert all(i["scrapedAt"] for i in items) # stamped by the orchestrator + + +@pytest.mark.asyncio +async def test_stops_when_a_page_is_all_duplicates(): + session = _FakeSession({0: ["k1", "k2"], 10: ["k1", "k2"], 20: ["k9"]}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + assert 20 not in { + int(parse_qs(urlparse(u).query).get("start", ["0"])[0]) for u in session.fetched + } + + +@pytest.mark.asyncio +async def test_respects_max_items_per_query(): + session = _FakeSession({0: ["k1", "k2", "k3", "k4"]}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=2), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + + +@pytest.mark.asyncio +async def test_global_dedupe_across_queries(): + # Both queries hit page 0 (same fake pages) and return the same keys. + session = _FakeSession({0: ["k1", "k2"]}) + items = await _collect( + IndeedScrapeInput(queries=["dev", "engineer"], maxItemsPerQuery=100), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + + +@pytest.mark.asyncio +async def test_start_urls_skip_viewjob_and_scrape_search(): + session = _FakeSession({0: ["k1"]}) + input_model = IndeedScrapeInput( + startUrls=[ + {"url": "https://www.indeed.com/jobs?q=dev"}, + {"url": "https://www.indeed.com/viewjob?jk=abc"}, + ], + maxItemsPerQuery=100, + ) + items = await _collect(input_model, session) + assert [i["jobKey"] for i in items] == ["k1"] + + +@pytest.mark.asyncio +async def test_scrape_indeed_limit_with_injected_session(): + session = _FakeSession({0: ["k1", "k2", "k3"]}) + items = await scrape_indeed( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), + limit=2, + session=session, + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] From 05bfc6a10ef521204b83fc3a68346f7402ef5b88 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 018/217] feat: add INDEED_JOB billing unit --- surfsense_backend/app/capabilities/core/types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index 67b9175fc..6b3be70d0 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -30,6 +30,7 @@ class BillingUnit(StrEnum): TIKTOK_VIDEO = "tiktok_video" TIKTOK_USER = "tiktok_user" TIKTOK_COMMENT = "tiktok_comment" + INDEED_JOB = "indeed_job" class BillableInput(Protocol): From e25d09c25e33fab2f583ffce4e01fb19153bec7e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 019/217] feat: add indeed per-job scrape rate config --- surfsense_backend/app/config/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 92e448d9a..c006c8cc6 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -736,6 +736,11 @@ class Config: # Comments are the cheapest per-item TikTok data, matching the per-comment # market (and YouTube's comment meter). TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) + # Warmed-browser listings put Indeed on par with the other browser-driven + # scrapers (Reddit, Instagram) rather than the cheaper API-backed meters. + INDEED_SCRAPE_MICROS_PER_JOB = int( + os.getenv("INDEED_SCRAPE_MICROS_PER_JOB", "3500") + ) # Retry an empty listing draw on a fresh rotating IP. Set to 1 for a static # proxy, where every retry re-hits the same exit. TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3")) From d89ca141a4b914ff342ab68c58cd2cf6bd408df4 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 020/217] feat: meter indeed_job billing unit --- surfsense_backend/app/capabilities/core/billing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 67abf4164..bc031db3f 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -40,6 +40,7 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT", + BillingUnit.INDEED_JOB: "INDEED_SCRAPE_MICROS_PER_JOB", } @@ -61,6 +62,7 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.TIKTOK_VIDEO: "video", BillingUnit.TIKTOK_USER: "profile", BillingUnit.TIKTOK_COMMENT: "comment", + BillingUnit.INDEED_JOB: "job", } From e887c1cae718fa0b7ca4ca1a57c04a2d82bdf4ae Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 021/217] feat: indeed.scrape io schemas --- .../app/capabilities/indeed/scrape/schemas.py | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 surfsense_backend/app/capabilities/indeed/scrape/schemas.py diff --git a/surfsense_backend/app/capabilities/indeed/scrape/schemas.py b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py new file mode 100644 index 000000000..a02f6f13b --- /dev/null +++ b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py @@ -0,0 +1,112 @@ +"""``indeed.scrape`` I/O contracts. + +A lean, agent-friendly surface over ``IndeedScrapeInput`` +(``app/proprietary/platforms/indeed_jobs``). The executor maps this to the full +scraper input; the scraper's ``IndeedItem`` is reused verbatim as the output +element. +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.indeed_jobs import IndeedItem +from app.proprietary.platforms.indeed_jobs.schemas import ( + IndeedJobType, + IndeedLevel, + IndeedRemote, + IndeedSort, +) + +MAX_INDEED_SOURCES = 20 +"""Per-call cap on urls + search_queries: bounds a synchronous request's fan-out.""" + +MAX_INDEED_ITEMS = 100 +"""Hard ceiling on jobs returned per call, regardless of the per-query caps.""" + + +class ScrapeInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_INDEED_SOURCES, + description=( + "Indeed URLs to scrape: a search page (/jobs?q=&l=) or a company " + "jobs page (/cmp//jobs). Provide these OR search_queries " + "(at least one source is required)." + ), + ) + search_queries: list[str] = Field( + default_factory=list, + max_length=MAX_INDEED_SOURCES, + description=( + "Job search terms; each returns up to max_items_per_query results, " + "shaped by country/location/job_type/etc." + ), + ) + country: str = Field( + default="us", + description="Country code selecting the Indeed domain, e.g. 'us', 'gb', 'de'.", + ) + location: str | None = Field( + default=None, + description="Where to search, e.g. 'Remote', 'New York, NY'.", + ) + radius: int | None = Field( + default=None, + description="Search radius in miles/km around location.", + ) + job_type: IndeedJobType | None = Field( + default=None, + description="Employment type filter: fulltime, parttime, contract, etc.", + ) + level: IndeedLevel | None = Field( + default=None, + description="Experience level filter: entry_level, mid_level, senior_level.", + ) + remote: IndeedRemote | None = Field( + default=None, + description="Work model filter: remote or hybrid.", + ) + from_days: int | None = Field( + default=None, + description="Only return jobs posted within the last N days.", + ) + sort: IndeedSort = Field( + default="relevance", + description="Result ordering: relevance or date.", + ) + max_items: int = Field( + default=25, + ge=1, + le=MAX_INDEED_ITEMS, + description="Max total jobs to return across all sources.", + ) + max_items_per_query: int = Field( + default=25, + ge=0, + description="Max jobs to pull per search/company target.", + ) + + @model_validator(mode="after") + def _require_a_source(self) -> ScrapeInput: + if not self.urls and not self.search_queries: + raise ValueError("Provide at least one of 'urls' or 'search_queries'.") + return self + + @property + def estimated_units(self) -> int: + """Worst-case billable jobs for the pre-flight gate: ``max_items`` is a + hard cross-source ceiling (le=100), so no call can exceed it.""" + return self.max_items + + +class ScrapeOutput(BaseModel): + items: list[IndeedItem] = Field( + default_factory=list, + description="One item per job posting, in emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned job = one billable unit.""" + return len(self.items) From b457599fd4577a65dc1fba61288574e1ba3c6204 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 022/217] feat: indeed.scrape executor --- .../capabilities/indeed/scrape/executor.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 surfsense_backend/app/capabilities/indeed/scrape/executor.py diff --git a/surfsense_backend/app/capabilities/indeed/scrape/executor.py b/surfsense_backend/app/capabilities/indeed/scrape/executor.py new file mode 100644 index 000000000..60df11972 --- /dev/null +++ b/surfsense_backend/app/capabilities/indeed/scrape/executor.py @@ -0,0 +1,55 @@ +"""``indeed.scrape`` executor: verb input → scraper → Indeed job items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.indeed.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.indeed_jobs import ( + IndeedAccessBlockedError, + IndeedScrapeInput, + scrape_indeed, +) + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the executor to a scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_indeed + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + actor_input = IndeedScrapeInput( + startUrls=[{"url": url} for url in payload.urls], + queries=payload.search_queries, + country=payload.country, + location=payload.location, + radius=payload.radius, + jobType=payload.job_type, + level=payload.level, + remote=payload.remote, + fromDays=payload.from_days, + sort=payload.sort, + maxItems=payload.max_items, + maxItemsPerQuery=payload.max_items_per_query, + ) + emit_progress( + "starting", "Resolving Indeed targets", total=payload.max_items, unit="job" + ) + try: + items = await scrape_fn(actor_input, limit=payload.max_items) + except IndeedAccessBlockedError as exc: + # Anonymous-only scraper; a hard block can't be retried with creds. + raise ForbiddenError( + f"Indeed refused anonymous access: {exc}", + code="INDEED_ACCESS_BLOCKED", + ) from exc + emit_progress( + "done", f"Scraped {len(items)} job(s)", current=len(items), unit="job" + ) + return ScrapeOutput(items=items) + + return execute From 3f91a006cd98967a03df3b72ab706957d3c83a2b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 023/217] feat: register indeed.scrape capability --- .../capabilities/indeed/scrape/definition.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 surfsense_backend/app/capabilities/indeed/scrape/definition.py diff --git a/surfsense_backend/app/capabilities/indeed/scrape/definition.py b/surfsense_backend/app/capabilities/indeed/scrape/definition.py new file mode 100644 index 000000000..b831cd4a7 --- /dev/null +++ b/surfsense_backend/app/capabilities/indeed/scrape/definition.py @@ -0,0 +1,23 @@ +"""``indeed.scrape`` capability registration (billed per job; see config +``INDEED_SCRAPE_MICROS_PER_JOB``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.indeed.scrape.executor import build_scrape_executor +from app.capabilities.indeed.scrape.schemas import ScrapeInput, ScrapeOutput + +INDEED_SCRAPE = Capability( + name="indeed.scrape", + description=( + "Scrape public Indeed job postings, including title, company, location, " + "salary, and description. Use urls or search_queries." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.INDEED_JOB, + docs_url="/docs/connectors/native/indeed", +) + +register_capability(INDEED_SCRAPE) From 17159172ee28e5cfefaae84e2f9f7a2f3e340cff Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 024/217] feat: indeed.scrape verb package --- surfsense_backend/app/capabilities/indeed/scrape/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 surfsense_backend/app/capabilities/indeed/scrape/__init__.py diff --git a/surfsense_backend/app/capabilities/indeed/scrape/__init__.py b/surfsense_backend/app/capabilities/indeed/scrape/__init__.py new file mode 100644 index 000000000..6b7fa588e --- /dev/null +++ b/surfsense_backend/app/capabilities/indeed/scrape/__init__.py @@ -0,0 +1,3 @@ +"""``indeed.scrape`` verb: Indeed search / company URLs → job postings.""" + +from __future__ import annotations From 8c410d8682097801f524293bbc25b34feae6e014 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 025/217] feat: indeed capability namespace --- surfsense_backend/app/capabilities/indeed/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 surfsense_backend/app/capabilities/indeed/__init__.py diff --git a/surfsense_backend/app/capabilities/indeed/__init__.py b/surfsense_backend/app/capabilities/indeed/__init__.py new file mode 100644 index 000000000..5e053244f --- /dev/null +++ b/surfsense_backend/app/capabilities/indeed/__init__.py @@ -0,0 +1,5 @@ +"""``indeed.*`` namespace: platform-native Indeed data verbs.""" + +from __future__ import annotations + +from app.capabilities.indeed.scrape import definition as _scrape # noqa: F401 From ad49e9004e33a11b8b333dafb3d187e01c189e60 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:23 +0200 Subject: [PATCH 026/217] feat: register indeed namespace on routes --- surfsense_backend/app/routes/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 85450d4b7..b5186a72f 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -3,6 +3,7 @@ from fastapi import APIRouter, Depends # Import verb namespaces for their registration side effects before the door builds. import app.capabilities.google_maps import app.capabilities.google_search +import app.capabilities.indeed import app.capabilities.instagram import app.capabilities.reddit import app.capabilities.tiktok From 4171248f512b9b343913cbd27627a1f185280281 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:31 +0200 Subject: [PATCH 027/217] test: indeed capability test package --- surfsense_backend/tests/unit/capabilities/indeed/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 surfsense_backend/tests/unit/capabilities/indeed/__init__.py diff --git a/surfsense_backend/tests/unit/capabilities/indeed/__init__.py b/surfsense_backend/tests/unit/capabilities/indeed/__init__.py new file mode 100644 index 000000000..e69de29bb From 725cb8a82b153a154d27d10469e93b2f3ab35e9f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:31 +0200 Subject: [PATCH 028/217] test: indeed.scrape test package --- .../tests/unit/capabilities/indeed/scrape/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 surfsense_backend/tests/unit/capabilities/indeed/scrape/__init__.py diff --git a/surfsense_backend/tests/unit/capabilities/indeed/scrape/__init__.py b/surfsense_backend/tests/unit/capabilities/indeed/scrape/__init__.py new file mode 100644 index 000000000..e69de29bb From d029bc0cb806312610473c82e9cfa1392c04d49b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:31 +0200 Subject: [PATCH 029/217] test: indeed.scrape registration --- .../unit/capabilities/indeed/test_registry.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 surfsense_backend/tests/unit/capabilities/indeed/test_registry.py diff --git a/surfsense_backend/tests/unit/capabilities/indeed/test_registry.py b/surfsense_backend/tests/unit/capabilities/indeed/test_registry.py new file mode 100644 index 000000000..08bc8d7cf --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/indeed/test_registry.py @@ -0,0 +1,23 @@ +"""The indeed namespace registers its verb as one Capability the doors/agent read.""" + +from __future__ import annotations + +import pytest + +from app.capabilities import ( + indeed, # noqa: F401 — importing the namespace registers its verbs +) +from app.capabilities.core.store import get_capability +from app.capabilities.core.types import BillingUnit +from app.capabilities.indeed.scrape.schemas import ScrapeInput, ScrapeOutput + +pytestmark = pytest.mark.unit + + +def test_indeed_scrape_is_registered_and_billed_per_job(): + cap = get_capability("indeed.scrape") + + assert cap.name == "indeed.scrape" + assert cap.input_schema is ScrapeInput + assert cap.output_schema is ScrapeOutput + assert cap.billing_unit is BillingUnit.INDEED_JOB From 695a7d69e0c0500836917ae57b185dff43cb49c1 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:49:31 +0200 Subject: [PATCH 030/217] test: indeed.scrape executor mapping --- .../indeed/scrape/test_executor.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 surfsense_backend/tests/unit/capabilities/indeed/scrape/test_executor.py diff --git a/surfsense_backend/tests/unit/capabilities/indeed/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/indeed/scrape/test_executor.py new file mode 100644 index 000000000..da7c48364 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/indeed/scrape/test_executor.py @@ -0,0 +1,100 @@ +"""``indeed.scrape`` executor: verb input → actor input mapping → typed items. + +Boundary mocked: the proprietary scraper (injected fake). NOT mocked: the verb's +own payload→IndeedScrapeInput mapping and the dict→IndeedItem wrapping. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.indeed.scrape.executor import build_scrape_executor +from app.capabilities.indeed.scrape.schemas import ScrapeInput, ScrapeOutput +from app.exceptions import ForbiddenError +from app.proprietary.platforms.indeed_jobs import ( + IndeedAccessBlockedError, + IndeedScrapeInput, +) + +pytestmark = pytest.mark.unit + + +class _FakeScraper: + """Records the actor input + limit it was called with; returns canned items.""" + + def __init__(self, items: list[dict]): + self._items = items + self.calls: list[tuple[IndeedScrapeInput, int | None]] = [] + + async def __call__( + self, actor_input: IndeedScrapeInput, *, limit: int | None = None + ) -> list[dict]: + self.calls.append((actor_input, limit)) + return self._items + + +async def test_maps_urls_to_start_urls_and_wraps_items(): + scraper = _FakeScraper([{"jobKey": "abc", "title": "Data Analyst"}]) + execute = build_scrape_executor(scrape_fn=scraper) + + out = await execute(ScrapeInput(urls=["https://www.indeed.com/jobs?q=dev"])) + + assert isinstance(out, ScrapeOutput) + assert len(out.items) == 1 + assert out.items[0].jobKey == "abc" + assert out.items[0].title == "Data Analyst" + + (actor_input, _limit) = scraper.calls[0] + assert [u.url for u in actor_input.startUrls] == [ + "https://www.indeed.com/jobs?q=dev" + ] + assert actor_input.queries == [] + + +async def test_maps_search_params_and_passes_limit(): + scraper = _FakeScraper([]) + execute = build_scrape_executor(scrape_fn=scraper) + + await execute( + ScrapeInput( + search_queries=["data analyst"], + country="gb", + location="Remote", + job_type="fulltime", + level="entry_level", + remote="remote", + from_days=7, + sort="date", + max_items=40, + max_items_per_query=15, + ) + ) + + (actor_input, limit) = scraper.calls[0] + assert actor_input.queries == ["data analyst"] + assert actor_input.country == "gb" + assert actor_input.location == "Remote" + assert actor_input.jobType == "fulltime" + assert actor_input.level == "entry_level" + assert actor_input.remote == "remote" + assert actor_input.fromDays == 7 + assert actor_input.sort == "date" + assert actor_input.maxItems == 40 + assert actor_input.maxItemsPerQuery == 15 + # The outer collection limit is the caller's total-item cap. + assert limit == 40 + + +async def test_access_blocked_maps_to_forbidden(): + async def _blocked(actor_input: IndeedScrapeInput, *, limit: int | None = None): + raise IndeedAccessBlockedError("all IPs refused") + + execute = build_scrape_executor(scrape_fn=_blocked) + + with pytest.raises(ForbiddenError): + await execute(ScrapeInput(search_queries=["x"])) + + +def test_requires_a_source(): + with pytest.raises(ValueError, match="at least one"): + ScrapeInput() From dc945aa14dfd124e2077bbbb8be34d3925f699ac Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:50:52 +0200 Subject: [PATCH 031/217] test: assert reddit.scrape billing unit --- .../tests/unit/capabilities/reddit/test_registry.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/reddit/test_registry.py b/surfsense_backend/tests/unit/capabilities/reddit/test_registry.py index e0bca5f26..2ac839670 100644 --- a/surfsense_backend/tests/unit/capabilities/reddit/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/reddit/test_registry.py @@ -8,15 +8,16 @@ from app.capabilities import ( reddit, # noqa: F401 — importing the namespace registers its verbs ) from app.capabilities.core.store import get_capability +from app.capabilities.core.types import BillingUnit from app.capabilities.reddit.scrape.schemas import ScrapeInput, ScrapeOutput pytestmark = pytest.mark.unit -def test_reddit_scrape_is_registered_and_free(): +def test_reddit_scrape_is_registered_and_billed_per_item(): cap = get_capability("reddit.scrape") assert cap.name == "reddit.scrape" assert cap.input_schema is ScrapeInput assert cap.output_schema is ScrapeOutput - assert cap.billing_unit is None + assert cap.billing_unit is BillingUnit.REDDIT_ITEM From ffc0675ac288281acb77efbdc097d70c80033bf0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:50:52 +0200 Subject: [PATCH 032/217] test: assert google_maps billing units --- .../tests/unit/capabilities/google_maps/test_registry.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py b/surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py index c6a007645..f9f774e8f 100644 --- a/surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/google_maps/test_registry.py @@ -8,25 +8,26 @@ from app.capabilities import ( google_maps, # noqa: F401 — importing the namespace registers its verbs ) from app.capabilities.core.store import get_capability +from app.capabilities.core.types import BillingUnit from app.capabilities.google_maps.reviews.schemas import ReviewsInput, ReviewsOutput from app.capabilities.google_maps.scrape.schemas import ScrapeInput, ScrapeOutput pytestmark = pytest.mark.unit -def test_google_maps_scrape_is_registered_and_free(): +def test_google_maps_scrape_is_registered_and_billed_per_place(): cap = get_capability("google_maps.scrape") assert cap.name == "google_maps.scrape" assert cap.input_schema is ScrapeInput assert cap.output_schema is ScrapeOutput - assert cap.billing_unit is None + assert cap.billing_unit is BillingUnit.GOOGLE_MAPS_PLACE -def test_google_maps_reviews_is_registered_and_free(): +def test_google_maps_reviews_is_registered_and_billed_per_review(): cap = get_capability("google_maps.reviews") assert cap.name == "google_maps.reviews" assert cap.input_schema is ReviewsInput assert cap.output_schema is ReviewsOutput - assert cap.billing_unit is None + assert cap.billing_unit is BillingUnit.GOOGLE_MAPS_REVIEW From 0089bb90012e2c356be498bd19be76e893b0e88e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 14 Jul 2026 21:50:52 +0200 Subject: [PATCH 033/217] test: assert youtube billing units --- .../tests/unit/capabilities/youtube/test_registry.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/youtube/test_registry.py b/surfsense_backend/tests/unit/capabilities/youtube/test_registry.py index 756b65176..5de00891f 100644 --- a/surfsense_backend/tests/unit/capabilities/youtube/test_registry.py +++ b/surfsense_backend/tests/unit/capabilities/youtube/test_registry.py @@ -8,25 +8,26 @@ from app.capabilities import ( youtube, # noqa: F401 — importing the namespace registers its verbs ) from app.capabilities.core.store import get_capability +from app.capabilities.core.types import BillingUnit from app.capabilities.youtube.comments.schemas import CommentsInput, CommentsOutput from app.capabilities.youtube.scrape.schemas import ScrapeInput, ScrapeOutput pytestmark = pytest.mark.unit -def test_youtube_scrape_is_registered_and_free(): +def test_youtube_scrape_is_registered_and_billed_per_video(): cap = get_capability("youtube.scrape") assert cap.name == "youtube.scrape" assert cap.input_schema is ScrapeInput assert cap.output_schema is ScrapeOutput - assert cap.billing_unit is None + assert cap.billing_unit is BillingUnit.YOUTUBE_VIDEO -def test_youtube_comments_is_registered_and_free(): +def test_youtube_comments_is_registered_and_billed_per_comment(): cap = get_capability("youtube.comments") assert cap.name == "youtube.comments" assert cap.input_schema is CommentsInput assert cap.output_schema is CommentsOutput - assert cap.billing_unit is None + assert cap.billing_unit is BillingUnit.YOUTUBE_COMMENT From a51630543e3625223510b16e964172bcffcd6230 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:08:08 +0530 Subject: [PATCH 034/217] refactor: remove deprecated configuration files for searxng - Deleted `limiter.toml` and `settings.yml` as they are no longer needed in the current setup. --- docker/searxng/limiter.toml | 5 --- docker/searxng/settings.yml | 90 ------------------------------------- 2 files changed, 95 deletions(-) delete mode 100644 docker/searxng/limiter.toml delete mode 100644 docker/searxng/settings.yml diff --git a/docker/searxng/limiter.toml b/docker/searxng/limiter.toml deleted file mode 100644 index dce84146f..000000000 --- a/docker/searxng/limiter.toml +++ /dev/null @@ -1,5 +0,0 @@ -[botdetection.ip_limit] -link_token = false - -[botdetection.ip_lists] -pass_ip = ["0.0.0.0/0"] diff --git a/docker/searxng/settings.yml b/docker/searxng/settings.yml deleted file mode 100644 index 0b805b6aa..000000000 --- a/docker/searxng/settings.yml +++ /dev/null @@ -1,90 +0,0 @@ -use_default_settings: - engines: - remove: - - ahmia - - torch - - qwant - - qwant news - - qwant images - - qwant videos - - mojeek - - mojeek images - - mojeek news - -server: - secret_key: "override-me-via-env" - limiter: false - image_proxy: false - method: "GET" - default_http_headers: - X-Robots-Tag: "noindex, nofollow" - -search: - formats: - - html - - json - default_lang: "auto" - autocomplete: "" - safe_search: 0 - ban_time_on_fail: 5 - max_ban_time_on_fail: 120 - suspended_times: - SearxEngineAccessDenied: 3600 - SearxEngineCaptcha: 3600 - SearxEngineTooManyRequests: 600 - cf_SearxEngineCaptcha: 7200 - cf_SearxEngineAccessDenied: 3600 - recaptcha_SearxEngineCaptcha: 7200 - -ui: - static_use_hash: true - -outgoing: - request_timeout: 12.0 - max_request_timeout: 20.0 - pool_connections: 100 - pool_maxsize: 20 - enable_http2: true - extra_proxy_timeout: 10 - retries: 1 - # Uncomment and set your residential proxy URL to route search engine requests through it. - # Format: http://:@:/ - # - # proxies: - # all://: - # - http://user:pass@proxy-host:port/ - -engines: - - name: google - disabled: false - weight: 1.2 - retry_on_http_error: [429, 503] - - name: duckduckgo - disabled: false - weight: 1.1 - retry_on_http_error: [429, 503] - - name: brave - disabled: false - weight: 1.0 - retry_on_http_error: [429, 503] - - name: bing - disabled: false - weight: 0.9 - retry_on_http_error: [429, 503] - - name: wikipedia - disabled: false - weight: 0.8 - - name: stackoverflow - disabled: false - weight: 0.7 - - name: yahoo - disabled: false - weight: 0.7 - retry_on_http_error: [429, 503] - - name: wikidata - disabled: false - weight: 0.6 - - name: currency - disabled: false - - name: ddg definitions - disabled: false From 3740171a5ce7625545cb596e38cb70ce81c976d3 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:12:30 +0530 Subject: [PATCH 035/217] feat(layout): add cookie persistence for playground sidebar state management --- .../components/layout/ui/shell/LayoutShell.tsx | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx index 7a42d5a2e..8578b7dec 100644 --- a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx +++ b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx @@ -44,6 +44,20 @@ const DocumentTabContent = dynamic( const PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE = "surfsense_playground_sidebar_collapsed"; const PLAYGROUND_SIDEBAR_COOKIE_MAX_AGE = 60 * 60 * 24 * 365; +function persistPlaygroundSidebarCollapsedCookie(isCollapsed: boolean) { + void window.cookieStore + ?.set({ + name: PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE, + value: String(isCollapsed), + path: "/", + expires: Date.now() + PLAYGROUND_SIDEBAR_COOKIE_MAX_AGE * 1000, + sameSite: "lax", + }) + .catch(() => { + // Ignore preference persistence failures. + }); +} + function MacDesktopTitleBar({ isSidebarCollapsed, onToggleSidebar, @@ -242,8 +256,7 @@ export function LayoutShell({ const handlePlaygroundSidebarToggle = () => { setIsPlaygroundSidebarCollapsed((collapsed) => { const nextCollapsed = !collapsed; - const secureAttribute = window.location.protocol === "https:" ? "; Secure" : ""; - document.cookie = `${PLAYGROUND_SIDEBAR_COLLAPSED_COOKIE}=${nextCollapsed}; Path=/; Max-Age=${PLAYGROUND_SIDEBAR_COOKIE_MAX_AGE}; SameSite=Lax${secureAttribute}`; + persistPlaygroundSidebarCollapsedCookie(nextCollapsed); return nextCollapsed; }); }; From 7889babb8e8743b7938f8eb60cef1ee9a944c345 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 15 Jul 2026 20:11:41 +0530 Subject: [PATCH 036/217] feat(sidebar): add loading skeleton for threads in AllChatsSidebar component --- .../layout/ui/sidebar/AllChatsSidebar.tsx | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx index 3063a9c50..04780bf3e 100644 --- a/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx @@ -94,7 +94,9 @@ function AllChatsContent({ workspaceId, className }: AllChatsContentProps) { const { data: threadsData, error: threadsError, + isFetching: isFetchingThreads, isLoading: isLoadingThreads, + isPlaceholderData, } = useQuery({ queryKey: ["all-threads", workspaceId], queryFn: () => fetchThreads(Number(workspaceId)), @@ -228,6 +230,7 @@ function AllChatsContent({ workspaceId, className }: AllChatsContentProps) { }, []); const isLoading = isSearchMode ? isLoadingSearch : isLoadingThreads; + const isLoadingMoreThreads = !isSearchMode && isFetchingThreads && isPlaceholderData; const error = isSearchMode ? searchError : threadsError; const selectedFilterLabel = showArchived ? "Archived" : "Active"; @@ -307,10 +310,12 @@ function AllChatsContent({ workspaceId, className }: AllChatsContentProps) { {[75, 90, 55, 80, 65, 85].map((titleWidth) => (
- - +
))} @@ -485,6 +490,21 @@ function AllChatsContent({ workspaceId, className }: AllChatsContentProps) { ); })} + {isLoadingMoreThreads ? ( +
+ {[62, 48, 56].map((titleWidth) => ( +
+ +
+ ))} +
+ ) : null} ) : isSearchMode ? (
From a466fdcf5d5dd54393a1f65cf85bd25210ee3c94 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:12:51 +0530 Subject: [PATCH 037/217] refactor(AllChatsSidebar): replace timestamp formatting with relative date display and adjust button styles --- .../layout/ui/sidebar/AllChatsSidebar.tsx | 102 ++++++++---------- surfsense_web/lib/format-date.ts | 22 ++-- 2 files changed, 56 insertions(+), 68 deletions(-) diff --git a/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx index 04780bf3e..e7cdb7d5e 100644 --- a/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx @@ -37,14 +37,13 @@ import { import { Input } from "@/components/ui/input"; import { Skeleton } from "@/components/ui/skeleton"; import { Spinner } from "@/components/ui/spinner"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { useActivateChatThread } from "@/hooks/use-activate-chat-thread"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useLongPress } from "@/hooks/use-long-press"; import { useIsMobile } from "@/hooks/use-mobile"; import { useArchiveThread, useDeleteThread, useRenameThread } from "@/hooks/use-thread-mutations"; import { fetchThreads, searchThreads, type ThreadListItem } from "@/lib/chat/thread-persistence"; -import { formatThreadTimestamp } from "@/lib/format-date"; +import { formatRelativeDate } from "@/lib/format-date"; import { cn } from "@/lib/utils"; interface AllChatsContentProps { @@ -288,7 +287,7 @@ function AllChatsContent({ workspaceId, className }: AllChatsContentProps) { placeholder={t("search_chats") || "Search chats..."} value={searchQuery} onChange={(e) => setSearchQuery(e.target.value)} - className="h-12 border-0 bg-muted pl-10 pr-9 text-base shadow-none" + className="h-10 border-0 bg-muted pl-10 pr-9 text-base shadow-none" /> {searchQuery && ( ) : ( - - - - - -

- {t("updated") || "Updated"}: {formatThreadTimestamp(thread.updatedAt)} -

- - + )}
-
- {thread.visibility === "SEARCH_SPACE" ? ( - - Shared - - ) : null} +
+
+ {thread.visibility === "SEARCH_SPACE" ? ( + + Shared + + ) : null} + + {formatRelativeDate(thread.updatedAt)} + +
setOpenDropdownId(isOpen ? thread.id : null)} @@ -436,11 +426,9 @@ function AllChatsContent({ workspaceId, className }: AllChatsContentProps) { variant="ghost" size="icon" className={cn( - "pointer-events-auto h-7 w-7 hover:bg-transparent", + "pointer-events-auto absolute right-0 h-7 w-7 hover:bg-transparent", openDropdownId === thread.id && "bg-accent hover:bg-accent", - !isMobile && - openDropdownId !== thread.id && - "opacity-0 group-hover/item:opacity-100" + openDropdownId !== thread.id && "opacity-0 group-hover/item:opacity-100" )} disabled={isBusy} > diff --git a/surfsense_web/lib/format-date.ts b/surfsense_web/lib/format-date.ts index a062b08fa..06ae77c64 100644 --- a/surfsense_web/lib/format-date.ts +++ b/surfsense_web/lib/format-date.ts @@ -5,29 +5,29 @@ import { isThisYear, isToday, isTomorrow, - isYesterday, } from "date-fns"; /** * Format a date string as a human-readable relative time * - < 1 min: "Just now" - * - < 60 min: "15m ago" - * - Today: "Today, 2:30 PM" - * - Yesterday: "Yesterday, 2:30 PM" - * - < 7 days: "3d ago" + * - < 60 min: "15 minutes ago" + * - < 24 hours: "21 hours ago" + * - < 7 days: "2 days ago" + * - Older this year: "Jan 15" * - Older: "Jan 15, 2026" */ export function formatRelativeDate(dateString: string): string { const date = new Date(dateString); const now = new Date(); - const minutesAgo = differenceInMinutes(now, date); - const daysAgo = differenceInDays(now, date); + const minutesAgo = Math.max(0, differenceInMinutes(now, date)); + const hoursAgo = Math.floor(minutesAgo / 60); + const daysAgo = Math.floor(hoursAgo / 24); if (minutesAgo < 1) return "Just now"; - if (minutesAgo < 60) return `${minutesAgo}m ago`; - if (isToday(date)) return `Today, ${format(date, "h:mm a")}`; - if (isYesterday(date)) return `Yesterday, ${format(date, "h:mm a")}`; - if (daysAgo < 7) return `${daysAgo}d ago`; + if (minutesAgo < 60) return `${minutesAgo} minute${minutesAgo === 1 ? "" : "s"} ago`; + if (hoursAgo < 24) return `${hoursAgo} hour${hoursAgo === 1 ? "" : "s"} ago`; + if (daysAgo < 7) return `${daysAgo} day${daysAgo === 1 ? "" : "s"} ago`; + if (isThisYear(date)) return format(date, "MMM d"); return format(date, "MMM d, yyyy"); } From 8aaf9cadbe948c4771ba918f20256fd803e4b616 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:16:38 +0530 Subject: [PATCH 038/217] feat(sidebar): add onChatsClick handler to sidebar components for improved navigation --- .../layout/providers/LayoutDataProvider.tsx | 4 ++++ .../components/layout/ui/shell/LayoutShell.tsx | 4 ++++ .../components/layout/ui/sidebar/MobileSidebar.tsx | 10 ++++++++++ .../components/layout/ui/sidebar/Sidebar.tsx | 14 +++++++++++++- 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 7fc862e49..682d2923e 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -631,6 +631,9 @@ export function LayoutDataProvider({ const isArtifactsPage = pathname?.endsWith("/artifacts") === true; const isPlaygroundPage = pathname?.includes("/playground") === true; const isAllChatsPage = pathname?.endsWith("/chats") === true; + const handleChatsClick = useCallback(() => { + router.push(`/dashboard/${workspaceId}/chats`); + }, [router, workspaceId]); const handleViewAllChats = useCallback(() => { router.push( isAllChatsPage ? `/dashboard/${workspaceId}/new-chat` : `/dashboard/${workspaceId}/chats` @@ -671,6 +674,7 @@ export function LayoutDataProvider({ onChatRename={handleChatRename} onChatDelete={handleChatDelete} onChatArchive={handleChatArchive} + onChatsClick={handleChatsClick} onViewAllChats={handleViewAllChats} user={{ email: user?.email || "", diff --git a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx index 8578b7dec..44a2259ee 100644 --- a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx +++ b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx @@ -106,6 +106,7 @@ interface LayoutShellProps { onChatRename?: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; onChatArchive?: (chat: ChatItem) => void; + onChatsClick?: () => void; onViewAllChats?: () => void; user: User; onSettings?: () => void; @@ -208,6 +209,7 @@ export function LayoutShell({ onChatRename, onChatDelete, onChatArchive, + onChatsClick, onViewAllChats, user, onSettings, @@ -289,6 +291,7 @@ export function LayoutShell({ onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} + onChatsClick={onChatsClick} onViewAllChats={onViewAllChats} isAllChatsActive={isAllChatsPage} user={user} @@ -388,6 +391,7 @@ export function LayoutShell({ onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} + onChatsClick={onChatsClick} onViewAllChats={onViewAllChats} isAllChatsActive={isAllChatsPage} user={user} diff --git a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx index 1d02b56f4..de1b08a6a 100644 --- a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx @@ -28,6 +28,7 @@ interface MobileSidebarProps { onChatRename?: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; onChatArchive?: (chat: ChatItem) => void; + onChatsClick?: () => void; onViewAllChats?: () => void; isAllChatsActive?: boolean; user: User; @@ -77,6 +78,7 @@ export function MobileSidebar({ onChatRename, onChatDelete, onChatArchive, + onChatsClick, onViewAllChats, isAllChatsActive = false, user, @@ -194,6 +196,14 @@ export function MobileSidebar({ onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} + onChatsClick={ + onChatsClick + ? () => { + onOpenChange(false); + onChatsClick(); + } + : undefined + } onViewAllChats={ onViewAllChats ? () => { diff --git a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx index 7862ae90e..e24523448 100644 --- a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx @@ -1,6 +1,6 @@ "use client"; -import { CreditCard, SquarePen, Zap } from "lucide-react"; +import { CreditCard, MessageCircleMore, SquarePen, Zap } from "lucide-react"; import Link from "next/link"; import { useParams } from "next/navigation"; import { useTranslations } from "next-intl"; @@ -62,6 +62,7 @@ interface SidebarProps { onChatRename?: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; onChatArchive?: (chat: ChatItem) => void; + onChatsClick?: () => void; onViewAllChats?: () => void; isAllChatsActive?: boolean; user: User; @@ -101,6 +102,7 @@ export function Sidebar({ onChatRename, onChatDelete, onChatArchive, + onChatsClick, onViewAllChats, isAllChatsActive = false, user, @@ -222,6 +224,16 @@ export function Sidebar({ onScroll={(event) => setIsSidebarNavScrolled(event.currentTarget.scrollTop > 0)} >
+ {onChatsClick && ( + + )} {automationsItem && ( Date: Wed, 15 Jul 2026 21:19:37 +0530 Subject: [PATCH 039/217] refactor(automations-loading): update skeleton loading styles for improved alignment and consistency --- .../components/automations-loading.tsx | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/automations/components/automations-loading.tsx b/surfsense_web/app/dashboard/[workspace_id]/automations/components/automations-loading.tsx index 1156be3f6..058e61598 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/automations/components/automations-loading.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/automations/components/automations-loading.tsx @@ -13,21 +13,18 @@ export function AutomationsLoadingRows() { return ( <> {ROW_KEYS.map((key) => ( - - -
- - -
+ + + - + - + - - + + ))} From 6ee95e5b2c21986dc31d3016a8c515c7da749c69 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 040/217] feat: parse indeed /viewjob detail page --- .../platforms/indeed_jobs/parsers.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py index ab2b97cf9..b61b13e6a 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py @@ -22,6 +22,9 @@ _DEFAULT_BASE = "https://www.indeed.com" _JOBCARDS_ANCHOR = 'window.mosaic.providerData["mosaic-provider-jobcards"]=' +# A /viewjob page assigns the full posting to this global. +_INITIAL_DATA_ANCHOR = "window._initialData" + # Indeed's extractedSalary.type -> our SalaryPeriod. _SALARY_PERIODS = { "HOURLY": "hour", @@ -79,6 +82,26 @@ def extract_jobcards_blob(html: str) -> dict | None: return data if isinstance(data, dict) else None +def extract_initial_data(html: str) -> dict | None: + """Decode the ``window._initialData`` assignment on a /viewjob page.""" + import json + + idx = html.find(_INITIAL_DATA_ANCHOR) + if idx == -1: + return None + brace = html.find("{", idx + len(_INITIAL_DATA_ANCHOR)) + if brace == -1: + return None + blob = _brace_match(html, brace) + if not blob: + return None + try: + data = json.loads(blob) + except ValueError: + return None + return data if isinstance(data, dict) else None + + def job_results(blob: dict | None) -> list[dict[str, Any]]: """Return the raw job records from a decoded blob.""" if not isinstance(blob, dict): @@ -226,3 +249,57 @@ def parse_job(raw: dict[str, Any], *, base_url: str = _DEFAULT_BASE) -> dict[str "datePublished": _utc_from_ms(raw.get("pubDate")), "createdAt": _utc_from_ms(raw.get("createDate")), } + + +def _detail_salary(hdr: dict[str, Any]) -> dict[str, Any] | None: + """Salary from the detail header's flat ``salaryMin/Max/Currency/Type`` fields.""" + smin = hdr.get("salaryMin") + smax = hdr.get("salaryMax") + if smin is None and smax is None: + return None + return { + "salaryText": None, + "salaryMin": smin, + "salaryMax": smax, + "currency": hdr.get("salaryCurrency"), + "period": _SALARY_PERIODS.get(hdr.get("salaryType")), + "isEstimated": False, + } + + +def parse_job_detail(html: str, *, base_url: str = _DEFAULT_BASE) -> dict[str, Any]: + """Map a /viewjob page to enrichment fields (empty dict if not a job page). + + Returns only fields the detail page actually carries, so the caller can merge + it onto a listing item without clobbering known values with blanks. The full + description (``sanitizedJobDescription``) is the field listings never have. + """ + data = extract_initial_data(html) + if not isinstance(data, dict): + return {} + jim = (data.get("jobInfoWrapperModel") or {}).get("jobInfoModel") or {} + if not isinstance(jim, dict): + return {} + hdr = jim.get("jobInfoHeaderModel") + hdr = hdr if isinstance(hdr, dict) else {} + taxo = _taxonomy(hdr) + desc_html = jim.get("sanitizedJobDescription") + desc_html = desc_html if isinstance(desc_html, str) and desc_html else None + remote_model = hdr.get("remoteWorkModel") + out: dict[str, Any] = { + "descriptionHtml": desc_html, + "descriptionText": _clean_snippet(desc_html), + "title": hdr.get("jobTitle"), + "company": hdr.get("companyName"), + "companyUrl": _abs_url(hdr.get("companyOverviewLink"), base_url), + "formattedLocation": hdr.get("formattedLocation") or data.get("jobLocation"), + "remoteType": remote_model.get("type") + if isinstance(remote_model, dict) + else None, + "jobTypes": _job_types(hdr, taxo), + "benefits": taxo.get("benefits", []), + "salary": _detail_salary(hdr), + } + if _is_remote(hdr, taxo): + out["isRemote"] = True + return {k: v for k, v in out.items() if v not in (None, [], {})} From 51c7220321103580a615ba730b463400531d6995 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 041/217] feat: indeed job-detail and enrichment flow --- .../platforms/indeed_jobs/scraper.py | 65 +++++++++++++++---- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py index 4f66f2ebf..0c9fcfb16 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py @@ -17,7 +17,12 @@ from typing import Any from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse from .fetch import IndeedSession, now_iso, open_session -from .parsers import extract_jobcards_blob, job_results, parse_job +from .parsers import ( + extract_jobcards_blob, + job_results, + parse_job, + parse_job_detail, +) from .schemas import IndeedItem, IndeedScrapeInput from .url_resolver import build_search_url, resolve_url @@ -76,24 +81,25 @@ async def _paginate( return -def _targets(input_model: IndeedScrapeInput) -> list[tuple[str, str]]: - """Resolve inputs to ``(url, domain)`` search/company targets. +def _targets(input_model: IndeedScrapeInput) -> list[tuple[str, str, str]]: + """Resolve inputs to ``(kind, url, domain)`` targets. - ``startUrls`` take precedence over ``queries``. Job (``/viewjob``) URLs are - skipped here; detail enrichment is a separate flow. + ``startUrls`` take precedence over ``queries``. ``kind`` is ``search`` for + query-built and search/company URLs, or ``job`` for a ``/viewjob`` URL. """ if input_model.startUrls: - out: list[tuple[str, str]] = [] + out: list[tuple[str, str, str]] = [] for entry in input_model.startUrls: resolved = resolve_url(entry.url) - if resolved is None or resolved.kind == "job": - logger.warning("[indeed] skipping unsupported URL: %s", entry.url) + if resolved is None: + logger.warning("[indeed] skipping unrecognized URL: %s", entry.url) continue - out.append((resolved.url, resolved.domain)) + kind = "job" if resolved.kind == "job" else "search" + out.append((kind, resolved.url, resolved.domain)) return out domain = None - urls: list[tuple[str, str]] = [] + urls: list[tuple[str, str, str]] = [] for query in input_model.queries: url = build_search_url( query, @@ -107,16 +113,49 @@ def _targets(input_model: IndeedScrapeInput) -> list[tuple[str, str]]: sort=input_model.sort, ) domain = domain or urlparse(url).hostname or "www.indeed.com" - urls.append((url, domain)) + urls.append(("search", url, domain)) return urls +async def _enrich(session: IndeedSession, item: dict[str, Any], base_url: str) -> None: + """Merge a job's /viewjob detail (full description, etc.) onto ``item`` in place. + + Best-effort: a blocked or malformed detail page leaves the listing fields as-is + rather than failing the run. + """ + job_url = item.get("jobUrl") + if not isinstance(job_url, str): + return + try: + detail = parse_job_detail(await session.fetch_html(job_url), base_url=base_url) + except Exception as exc: + logger.warning("[indeed] detail fetch failed for %s: %s", job_url, exc) + return + item.update(detail) + + +async def _job_item( + session: IndeedSession, url: str, base_url: str +) -> dict[str, Any] | None: + """Scrape a single /viewjob URL into an item from its detail page alone.""" + detail = parse_job_detail(await session.fetch_html(url), base_url=base_url) + if not detail: + return None + return _emit({"jobUrl": url, "source": "indeed", **detail}) + + async def iter_indeed( input_model: IndeedScrapeInput, session: IndeedSession ) -> AsyncIterator[dict[str, Any]]: """Stream flat job items for every target, deduped by ``jobKey`` across all.""" global_seen: set[str] = set() - for url, domain in _targets(input_model): + for kind, url, domain in _targets(input_model): + base_url = f"https://{domain}" + if kind == "job": + item = await _job_item(session, url, base_url) + if item is not None: + yield item + continue async for item in _paginate( session, url, domain=domain, max_items=input_model.maxItemsPerQuery ): @@ -125,6 +164,8 @@ async def iter_indeed( if job_key in global_seen: continue global_seen.add(job_key) + if input_model.scrapeJobDetails: + await _enrich(session, item, base_url) yield item From 81f535e9d0d3fde7fc0c509e553e3fb9b0e0db98 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 042/217] feat: add scrape_job_details to indeed.scrape input --- .../app/capabilities/indeed/scrape/schemas.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/surfsense_backend/app/capabilities/indeed/scrape/schemas.py b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py index a02f6f13b..02e29100d 100644 --- a/surfsense_backend/app/capabilities/indeed/scrape/schemas.py +++ b/surfsense_backend/app/capabilities/indeed/scrape/schemas.py @@ -75,6 +75,13 @@ class ScrapeInput(BaseModel): default="relevance", description="Result ordering: relevance or date.", ) + scrape_job_details: bool = Field( + default=False, + description=( + "Fetch each job's detail page for the full description (slower: one " + "extra page load per job)." + ), + ) max_items: int = Field( default=25, ge=1, From ec258d8387190f75e8fd133e9596e47504d0767b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 043/217] feat: map scrape_job_details in indeed executor --- surfsense_backend/app/capabilities/indeed/scrape/executor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/capabilities/indeed/scrape/executor.py b/surfsense_backend/app/capabilities/indeed/scrape/executor.py index 60df11972..85983e0da 100644 --- a/surfsense_backend/app/capabilities/indeed/scrape/executor.py +++ b/surfsense_backend/app/capabilities/indeed/scrape/executor.py @@ -33,6 +33,7 @@ def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: remote=payload.remote, fromDays=payload.from_days, sort=payload.sort, + scrapeJobDetails=payload.scrape_job_details, maxItems=payload.max_items, maxItemsPerQuery=payload.max_items_per_query, ) From 0b73581cbe501c5f74b2f11370289ec0ebd0be76 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 044/217] test: indeed viewjob detail fixture --- .../unit/platforms/indeed_jobs/fixtures/sample_viewjob.html | 1 + 1 file changed, 1 insertion(+) create mode 100644 surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_viewjob.html diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_viewjob.html b/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_viewjob.html new file mode 100644 index 000000000..ad09bf38c --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/fixtures/sample_viewjob.html @@ -0,0 +1 @@ + \ No newline at end of file From 38c98ba0e425b59508fa9e0536b84332c2a4d548 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 045/217] test: indeed detail parser --- .../platforms/indeed_jobs/test_parsers.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py index 7b9ca6298..38a6544b6 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py @@ -9,6 +9,7 @@ from app.proprietary.platforms.indeed_jobs.parsers import ( extract_jobcards_blob, job_results, parse_job, + parse_job_detail, ) _FIXTURE_DIR = Path(__file__).parent / "fixtures" @@ -137,3 +138,39 @@ def test_fixture_blob_parses_into_items(): assert item["title"] assert item["jobUrl"].startswith("https://www.indeed.com/viewjob?jk=") assert "salaryText" in item["salary"] + + +# --- detail page (parse_job_detail) ---------------------------------------- + + +def test_parse_job_detail_extracts_description_and_fields(): + html = (_FIXTURE_DIR / "sample_viewjob.html").read_text() + detail = parse_job_detail(html) + assert detail["descriptionHtml"].startswith("
") + assert "Data Analyst" in detail["descriptionText"] + assert "
" not in detail["descriptionText"] # tags stripped + assert detail["title"] + assert detail["company"] + assert detail["formattedLocation"] + assert detail["jobTypes"] == ["Full-time"] + assert detail["benefits"] == ["401(k)", "Health insurance"] + assert detail["isRemote"] is True + assert detail["remoteType"] == "HYBRID" + sal = detail["salary"] + assert (sal["salaryMin"], sal["salaryMax"], sal["period"]) == (60000, 90000, "year") + + +def test_parse_job_detail_omits_blank_fields(): + # A page with a header but no salary/description must not emit those keys, + # so a merge won't clobber listing values with blanks. + html = ( + "' + ) + detail = parse_job_detail(html) + assert detail == {"title": "Analyst"} + + +def test_parse_job_detail_not_a_job_page_returns_empty(): + assert parse_job_detail("just a moment...") == {} From d2af2402b4a3453dfdaaa1f2ddbeae96b72ba8cc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 17:54:38 +0200 Subject: [PATCH 046/217] test: indeed detail and enrichment flow --- .../platforms/indeed_jobs/test_scraper.py | 45 +++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py index c56d64f75..e5ea23452 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py @@ -22,8 +22,21 @@ def _page_html(job_keys: list[str]) -> str: ) +def _detail_html(job_key: str) -> str: + """A minimal /viewjob page carrying a full description for ``job_key``.""" + data = { + "jobInfoWrapperModel": { + "jobInfoModel": { + "sanitizedJobDescription": f"
Full description for {job_key}
", + "jobInfoHeaderModel": {"jobTitle": f"Detailed {job_key}"}, + } + } + } + return f"" + + class _FakeSession: - """Returns per-``start`` pages and records the URLs requested.""" + """Returns per-``start`` search pages (or a /viewjob detail) and records URLs.""" def __init__(self, pages: dict[int, list[str]]) -> None: self._pages = pages @@ -31,7 +44,10 @@ class _FakeSession: async def fetch_html(self, url: str) -> str: self.fetched.append(url) - start = int(parse_qs(urlparse(url).query).get("start", ["0"])[0]) + query = parse_qs(urlparse(url).query) + if "/viewjob" in url: + return _detail_html(query.get("jk", [""])[0]) + start = int(query.get("start", ["0"])[0]) return _page_html(self._pages.get(start, [])) @@ -81,7 +97,7 @@ async def test_global_dedupe_across_queries(): @pytest.mark.asyncio -async def test_start_urls_skip_viewjob_and_scrape_search(): +async def test_start_urls_scrape_search_and_job_url_detail(): session = _FakeSession({0: ["k1"]}) input_model = IndeedScrapeInput( startUrls=[ @@ -91,7 +107,28 @@ async def test_start_urls_skip_viewjob_and_scrape_search(): maxItemsPerQuery=100, ) items = await _collect(input_model, session) - assert [i["jobKey"] for i in items] == ["k1"] + assert len(items) == 2 + search_item, job_item = items + assert search_item["jobKey"] == "k1" + # The /viewjob URL is scraped from its detail page alone. + assert job_item["jobUrl"].endswith("jk=abc") + assert job_item["title"] == "Detailed abc" + assert "Full description for abc" in job_item["descriptionText"] + + +@pytest.mark.asyncio +async def test_scrape_job_details_enriches_listing_items(): + session = _FakeSession({0: ["k1", "k2"]}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100, scrapeJobDetails=True), + session, + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + for it in items: + assert it["descriptionHtml"].startswith("
Full description for") + assert "Full description for" in it["descriptionText"] + # One extra /viewjob load per listing item. + assert sum("/viewjob" in u for u in session.fetched) == 2 @pytest.mark.asyncio From 2ade6bb90318c4cd04df9ec2a07a0e88e1d6afbf Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 047/217] feat: indeed subagent package --- .../chat/multi_agent_chat/subagents/builtins/indeed/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/__init__.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/__init__.py new file mode 100644 index 000000000..ddd8a9cd3 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/__init__.py @@ -0,0 +1 @@ +"""``indeed`` builtin subagent: structured Indeed job postings.""" From 7cbd8345f9cb82f4daf6972736512515ee72b61e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 048/217] feat: indeed subagent tools package --- .../multi_agent_chat/subagents/builtins/indeed/tools/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/__init__.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/__init__.py new file mode 100644 index 000000000..e69de29bb From ada83efedf06e6458a9db36b07ddda8638163225 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 049/217] feat: wire indeed.scrape into subagent tools --- .../subagents/builtins/indeed/tools/index.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/index.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/index.py new file mode 100644 index 000000000..523579471 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/tools/index.py @@ -0,0 +1,27 @@ +"""``indeed`` sub-agent tools: the Indeed scrape capability verb.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.indeed.scrape.definition import INDEED_SCRAPE + +NAME = "indeed" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [INDEED_SCRAPE] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) From 723eb8c6cb17e096d14ae9c68f06bd599d412237 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 050/217] feat: indeed subagent routing description --- .../multi_agent_chat/subagents/builtins/indeed/description.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/description.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/description.md new file mode 100644 index 000000000..221c54fe9 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/description.md @@ -0,0 +1,2 @@ +Indeed jobs specialist: pulls structured job postings — title, company, location, salary (range, currency, period), job types, benefits, remote/hybrid flag, posting age, apply URL, and the full job description. Discovers jobs by search query (with country, location, radius, job type, experience level, remote/hybrid, and date-posted filters), or scrapes a known Indeed search, company jobs, or single job URL as-is. Optionally fetches each job's detail page for the full description. +Use whenever the task is to find or gather job listings from Indeed — openings for a role, hiring at a company, salaries for a title in a location, or remote roles in a field. Triggers include "find jobs for X", "who is hiring X", "data analyst jobs in Y", "remote X roles", and scraping a specific Indeed URL. Not for general web pages (use the web crawling specialist), Google results (use the Google Search specialist), or other job boards. From e0cc816f673477e1ad5fc6a0825341626f33a47d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 051/217] feat: indeed subagent system prompt --- .../builtins/indeed/system_prompt.md | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md new file mode 100644 index 000000000..b034b700a --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md @@ -0,0 +1,66 @@ +You are the SurfSense Indeed sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from live Indeed job data gathered with your verb, comparing against earlier results already in this conversation when the task calls for it. + + + +- `indeed_scrape` +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Finding jobs for a role: call `indeed_scrape` with `search_queries`; narrow with `location`, `country`, `job_type`, `level`, `remote`, `radius`, and `from_days`. +- Scraping a specific Indeed URL: pass a search, company jobs, or single job URL in `urls`. +- Full descriptions: set `scrape_job_details=true` to fetch each job's detail page (slower: one extra load per job). Leave it false when the listing snippet is enough. +- Controlling volume: use `max_items` for the total cap and `max_items_per_query` per search. +- Requested counts: `max_items` defaults to only 25 — when the task asks for N jobs, set `max_items` and `max_items_per_query` above N (with headroom for off-topic hits). A call that caps below the target can never satisfy it. +- Under-delivery: if the first call returns fewer on-topic results than requested, broaden it yourself — more query phrasings, wider `radius`, drop restrictive filters, larger `from_days` — before settling. Return `status=partial` only after the broadened attempt, never after a single narrow call. +- Batch multiple search terms into one call rather than many single-term calls. + +- Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, salary/rank changes). + + + +- Use only tools in ``. +- Report only results present in the tool output. Never invent titles, companies, salaries, locations, or description text. + + + +- Do not read arbitrary web pages — that belongs to the web crawling specialist. +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Google results belong to the Google Search specialist; other job boards are out of scope. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable query or URL — return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the scope you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: one entry per distinct job or delta — a single sentence each; do not paste raw payloads. Max 10 entries, unless the delegated task asks for N items: then return up to N (each backed by a real scraped result, never padded). +- `evidence.sources`: one Indeed job URL per finding when applicable, same cap as findings. List each URL once. + + From e193867e7724f372b4f87a8aa7577e77a2f3cbae Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 052/217] feat: indeed subagent builder --- .../subagents/builtins/indeed/agent.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/agent.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/agent.py new file mode 100644 index 000000000..6da835dbd --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/agent.py @@ -0,0 +1,43 @@ +"""``indeed`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Pulls structured job postings from Indeed search and company pages." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) From 238d2767c5e4392a0cd4e51424e7e3bfe4b957d8 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 053/217] feat: register indeed subagent in registry --- .../app/agents/chat/multi_agent_chat/subagents/registry.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index de37edfb9..bc368facd 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -21,6 +21,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.google_maps.agent impor from app.agents.chat.multi_agent_chat.subagents.builtins.google_search.agent import ( build_subagent as build_google_search_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.indeed.agent import ( + build_subagent as build_indeed_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.instagram.agent import ( build_subagent as build_instagram_subagent, ) @@ -85,6 +88,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "google_drive": build_google_drive_subagent, "google_maps": build_google_maps_subagent, "google_search": build_google_search_subagent, + "indeed": build_indeed_subagent, "instagram": build_instagram_subagent, "knowledge_base": build_knowledge_base_subagent, "mcp_discovery": build_mcp_discovery_subagent, From 3a9b0405c3588d18d7738821d229572a19bd8a01 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 054/217] feat: add indeed to required-connector map --- surfsense_backend/app/agents/chat/multi_agent_chat/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index 6bbd808fb..f0066484a 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -35,6 +35,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "youtube": frozenset(), "google_maps": frozenset(), "google_search": frozenset(), + "indeed": frozenset(), "reddit": frozenset(), "instagram": frozenset(), "tiktok": frozenset(), From 9e379602434344ef1513f4fe339d9ebdd4bb4319 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:46:55 +0200 Subject: [PATCH 055/217] test: pin indeed in subagent roster --- .../unit/agents/multi_agent_chat/test_subagent_composition.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index ee889cd17..cf5982780 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -32,6 +32,7 @@ _EXPECTED_SUBAGENTS = frozenset( "google_drive", "google_maps", "google_search", + "indeed", "instagram", "knowledge_base", "mcp_discovery", From 457c073c44a5d2b129abe0026198f88d4122d43e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:54:49 +0200 Subject: [PATCH 056/217] feat: add indeed MCP scraper tool --- .../features/scrapers/platforms/indeed.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 surfsense_mcp/mcp_server/features/scrapers/platforms/indeed.py diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/indeed.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/indeed.py new file mode 100644 index 000000000..6ea0007f9 --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/indeed.py @@ -0,0 +1,135 @@ +"""Indeed scraper tool.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +IndeedSort = Literal["relevance", "date"] +IndeedJobType = Literal[ + "fulltime", + "parttime", + "contract", + "internship", + "temporary", + "permanent", + "seasonal", + "freelance", +] +IndeedLevel = Literal["entry_level", "mid_level", "senior_level"] +IndeedRemote = Literal["remote", "hybrid"] + + +def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None: + """Register the Indeed tool.""" + + @mcp.tool( + name="surfsense_indeed_scrape", + title="Search or scrape Indeed jobs", + annotations=SCRAPE, + structured_output=False, + ) + async def indeed_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="Indeed URLs: a search page " + "('https://www.indeed.com/jobs?q=data+analyst'), a company jobs " + "page ('/cmp//jobs'), or a single job ('/viewjob?jk=...'). " + "Provide urls OR search_queries." + ), + ] = None, + search_queries: Annotated[ + list[str] | None, + Field( + description="Job search terms, e.g. ['data analyst', 'ml engineer']. " + "Provide search_queries OR urls." + ), + ] = None, + country: Annotated[ + str, + Field(description="Country code selecting the Indeed domain, e.g. 'us', 'gb'."), + ] = "us", + location: Annotated[ + str | None, + Field(description="Where to search, e.g. 'Remote', 'New York, NY'."), + ] = None, + radius: Annotated[ + int | None, + Field(description="Search radius in miles/km around location."), + ] = None, + job_type: Annotated[ + IndeedJobType | None, + Field(description="Employment type filter."), + ] = None, + level: Annotated[ + IndeedLevel | None, + Field(description="Experience level filter."), + ] = None, + remote: Annotated[ + IndeedRemote | None, + Field(description="Work model filter: remote or hybrid."), + ] = None, + from_days: Annotated[ + int | None, + Field(description="Only return jobs posted within the last N days."), + ] = None, + sort: Annotated[ + IndeedSort, Field(description="Result ordering: relevance or date.") + ] = "relevance", + scrape_job_details: Annotated[ + bool, + Field( + description="True fetches each job's detail page for the full " + "description (slower); False returns the listing snippet only." + ), + ] = False, + max_items: Annotated[ + int, Field(ge=1, description="Maximum jobs to return in total.") + ] = 25, + max_items_per_query: Annotated[ + int, Field(ge=0, description="Max jobs per search/company target.") + ] = 25, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Search or scrape public Indeed job postings. + + Use this for ANY Indeed job research — openings for a role, who is hiring + at a company, salaries for a title in a location, or remote roles — + instead of a generic web search. Returns jobs with title, company, + location, salary, job types, and description; set scrape_job_details for + the full description per job. + Example: search_queries=['data analyst'], location='Remote', max_items=30. + """ + return await run_scraper( + client, + context, + platform="indeed", + verb="scrape", + payload={ + "urls": urls, + "search_queries": search_queries, + "country": country, + "location": location, + "radius": radius, + "job_type": job_type, + "level": level, + "remote": remote, + "from_days": from_days, + "sort": sort, + "scrape_job_details": scrape_job_details, + "max_items": max_items, + "max_items_per_query": max_items_per_query, + }, + workspace=workspace, + response_format=response_format, + ) From b89fe1a4f08f5069108ca4e065f64c051f3b596b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:54:49 +0200 Subject: [PATCH 057/217] feat: register indeed MCP tool --- surfsense_mcp/mcp_server/features/scrapers/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index 9aabbe0e5..e4823ca5c 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -1,7 +1,7 @@ """Scraper tools: one MCP surface per SurfSense platform capability. -Web crawl, Google Search, Reddit, YouTube, and Google Maps each get a tool that -maps a natural-language request to the workspace's scraper. Two run-history tools +Web crawl, Google Search, Reddit, YouTube, Google Maps, and Indeed each get a +tool that maps a natural-language request to the workspace's scraper. Two run-history tools list and fetch past runs, so a large result truncated inline can be retrieved in full later. Each platform lives in its own module under platforms/. """ @@ -16,6 +16,7 @@ from . import run_history from .platforms import ( google_maps, google_search, + indeed, instagram, reddit, tiktok, @@ -31,6 +32,7 @@ _REGISTRARS = ( instagram, tiktok, google_maps, + indeed, run_history, ) From e88ff8cef7a22e96aba153eea88b4f8886838f83 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:54:49 +0200 Subject: [PATCH 058/217] test: expect indeed MCP tool in selfcheck --- surfsense_mcp/mcp_server/selfcheck.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_mcp/mcp_server/selfcheck.py b/surfsense_mcp/mcp_server/selfcheck.py index bb61151bc..b9f21c638 100644 --- a/surfsense_mcp/mcp_server/selfcheck.py +++ b/surfsense_mcp/mcp_server/selfcheck.py @@ -31,6 +31,7 @@ EXPECTED_TOOLS = { "surfsense_google_maps_reviews", "surfsense_instagram_scrape", "surfsense_instagram_details", + "surfsense_indeed_scrape", "surfsense_list_scraper_runs", "surfsense_get_scraper_run", # knowledge-base management From 65c757fbc60d8e8d77ce24158a59922e762aff64 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:54:49 +0200 Subject: [PATCH 059/217] feat: mention indeed in MCP server instructions --- surfsense_mcp/mcp_server/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/surfsense_mcp/mcp_server/server.py b/surfsense_mcp/mcp_server/server.py index f801aeb15..ef5f15c8a 100644 --- a/surfsense_mcp/mcp_server/server.py +++ b/surfsense_mcp/mcp_server/server.py @@ -38,7 +38,8 @@ def build_server(settings: Settings) -> tuple[FastMCP, SurfSenseClient]: "task involves Reddit (posts, comments, finding subreddits or " "communities), YouTube (videos, transcripts, comments), Instagram " "(posts, reels, profile details), TikTok (videos by hashtag, " - "search, or URL), Google Maps (places, reviews), Google Search " + "search, or URL), Google Maps (places, reviews), Indeed (job " + "postings by role, company, or location), Google Search " "results, or reading " "specific web pages. Scraper results are persisted as runs; if an " "inline result is truncated, fetch it in full with " From 5b0982f80b19a317117d5500724d2b27fc63a45f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 18:57:15 +0200 Subject: [PATCH 060/217] docs: add indeed native connector page and listings --- .../content/docs/connectors/index.mdx | 2 +- .../content/docs/connectors/native/indeed.mdx | 51 +++++++++++++++++++ .../content/docs/connectors/native/index.mdx | 7 ++- .../content/docs/connectors/native/meta.json | 1 + .../content/docs/how-to/mcp-server.mdx | 8 +-- 5 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 surfsense_web/content/docs/connectors/native/indeed.mdx diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx index f16cf79b9..5bf7d33c1 100644 --- a/surfsense_web/content/docs/connectors/index.mdx +++ b/surfsense_web/content/docs/connectors/index.mdx @@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li } title="Native Connectors" - description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" + description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, and Web Crawl — usable in chat, the API Playground, or via REST" href="/docs/connectors/native" /> /jobs`), or a single job (`/viewjob?jk=`) (max 20 sources per call, combined with queries) | +| `search_queries` | — | Job search terms, e.g. `["data analyst"]` | +| `country` | `us` | Country code selecting the Indeed domain, e.g. `us`, `gb`, `de` | +| `location` | — | Where to search, e.g. `Remote`, `New York, NY` | +| `radius` | — | Search radius in miles/km around `location` | +| `job_type` | — | `fulltime`, `parttime`, `contract`, `internship`, `temporary`, `permanent`, `seasonal`, or `freelance` | +| `level` | — | `entry_level`, `mid_level`, or `senior_level` | +| `remote` | — | `remote` or `hybrid` | +| `from_days` | — | Only return jobs posted within the last N days | +| `sort` | `relevance` | `relevance` or `date` | +| `scrape_job_details` | `false` | Fetch each job's detail page for the full description (slower: one extra page load per job) | +| `max_items` | `25` | Max total jobs returned across all sources (hard cap 100) | +| `max_items_per_query` | `25` | Max jobs per search/company target | + +## Example + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/indeed/scrape" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "search_queries": ["data analyst"], + "location": "Remote", + "remote": "remote", + "sort": "date", + "max_items": 30 + }' +``` + +The response is `{ "items": [...] }` — one item per job posting. Billing is per returned job. + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Indeed → Scrape** in your workspace. diff --git a/surfsense_web/content/docs/connectors/native/index.mdx b/surfsense_web/content/docs/connectors/native/index.mdx index 12e1a7be3..fff814b7f 100644 --- a/surfsense_web/content/docs/connectors/native/index.mdx +++ b/surfsense_web/content/docs/connectors/native/index.mdx @@ -1,6 +1,6 @@ --- title: Native Connectors -description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the web +description: SurfSense's built-in scraper APIs for Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, and the web --- import { Card, Cards } from 'fumadocs-ui/components/card'; @@ -38,6 +38,11 @@ Native connectors are SurfSense's own scraper APIs — built into the platform, description="Structured SERPs: organic results, people-also-ask, AI overviews" href="/docs/connectors/native/google-search" /> + @@ -257,14 +257,14 @@ For self-host (stdio), all settings are environment variables passed by the clie - **401 errors** — the API key is wrong or expired; create a new one. - **403 errors** — API access is disabled for the workspace; toggle **API key access** on under **API Playground → API Keys**. - **"Could not reach SurfSense"** — the backend isn't running or `SURFSENSE_BASE_URL` is wrong. -- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 24 tools register without needing a backend. +- **Server won't start** — run `uv run python -m mcp_server.selfcheck` inside `surfsense_mcp`; it verifies all 25 tools register without needing a backend. ## Tools reference | Group | Tools | |-------|-------| | Workspaces | `surfsense_list_workspaces`, `surfsense_select_workspace` | -| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_tiktok_comments`, `surfsense_tiktok_user_search`, `surfsense_tiktok_trending`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | +| Scrapers | `surfsense_reddit_scrape`, `surfsense_youtube_scrape`, `surfsense_youtube_comments`, `surfsense_instagram_scrape`, `surfsense_instagram_details`, `surfsense_tiktok_scrape`, `surfsense_tiktok_comments`, `surfsense_tiktok_user_search`, `surfsense_tiktok_trending`, `surfsense_google_maps_scrape`, `surfsense_google_maps_reviews`, `surfsense_google_search`, `surfsense_indeed_scrape`, `surfsense_web_crawl`, `surfsense_list_scraper_runs`, `surfsense_get_scraper_run` | | Knowledge base | `surfsense_search_knowledge_base`, `surfsense_list_documents`, `surfsense_get_document`, `surfsense_add_document`, `surfsense_upload_file`, `surfsense_update_document`, `surfsense_delete_document` | Usage is billed exactly like the REST API — scraper tools are metered per returned item, and every call is recorded under **API Playground → Runs**. From e19c876fbc8e984e6cd18348c127b6d8c803c98b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 19:03:51 +0200 Subject: [PATCH 061/217] feat: wire indeed connector into env, readmes, playground and marketing --- README.es.md | 5 +- README.hi.md | 5 +- README.md | 5 +- README.pt-BR.md | 5 +- README.zh-CN.md | 5 +- docker/.env.example | 1 + surfsense_backend/.env.example | 1 + surfsense_web/lib/auth-utils.ts | 1 + .../lib/connectors-marketing/google-maps.tsx | 1 + .../connectors-marketing/google-search.tsx | 1 + .../lib/connectors-marketing/indeed.tsx | 327 ++++++++++++++++++ .../lib/connectors-marketing/index.ts | 2 + .../lib/connectors-marketing/instagram.tsx | 1 + .../lib/connectors-marketing/reddit.tsx | 1 + .../lib/connectors-marketing/tiktok.tsx | 1 + .../lib/connectors-marketing/web-crawl.tsx | 1 + .../lib/connectors-marketing/youtube.tsx | 1 + surfsense_web/lib/playground/catalog.ts | 7 + .../lib/playground/platform-icons.tsx | 1 + surfsense_web/public/connectors/indeed.svg | 5 + 20 files changed, 367 insertions(+), 10 deletions(-) create mode 100644 surfsense_web/lib/connectors-marketing/indeed.tsx create mode 100644 surfsense_web/public/connectors/indeed.svg diff --git a/README.es.md b/README.es.md index 08231b5ab..3d38b8334 100644 --- a/README.es.md +++ b/README.es.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM para investigación de inteligencia competitiva -SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. +SurfSense es la **plataforma de inteligencia competitiva de código abierto para agentes de IA**, como NotebookLM pero con conectores de scraping en vivo. Tus agentes monitorean a la competencia, siguen los rankings y escuchan a tu mercado con datos en vivo de **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed y la web abierta**, a través de una única **API REST** o un **servidor MCP**. Agentes programados y activados por eventos convierten lo que encuentran en informes y alertas, y una base de conocimiento integrada mantiene cada hallazgo disponible para búsqueda con citas. > [!NOTE] > **📢 Una nota para nuestros usuarios de la alternativa a NotebookLM** @@ -107,6 +107,7 @@ Las automatizaciones ejecutan turnos completos de agente según un horario o en | **TikTok** | Videos, comentarios, hashtags y perfiles sin aprobación de la Research API | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Lugares, calificaciones y reseñas para investigar competidores locales y prospectos | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | SERPs en vivo para seguimiento de posiciones y monitoreo de mercado | [Google Search API](https://www.surfsense.com/google-search) | +| **Indeed** | Ofertas de empleo públicas con salarios y descripciones completas, por búsqueda o empresa | [Indeed Scraper API](https://www.surfsense.com/indeed) | | **Web Crawl** (rastreo web) | Cualquier página de la web abierta como contenido limpio y estructurado | [Web Crawling API](https://www.surfsense.com/web-crawl) | | **Conectores MCP externos** | Conecta cualquier servidor MCP a tus agentes, con OAuth de un clic para Notion, Slack, Jira y más | [External MCP Connectors](https://www.surfsense.com/external-mcp-connectors) | @@ -246,7 +247,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | Característica | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Datos de mercado en vivo para agentes** | No | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search y rastreo web vía API REST y MCP | +| **Datos de mercado en vivo para agentes** | No | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed y rastreo web vía API REST y MCP | | **Servidor MCP** | No | Cada conector expuesto como herramienta nativa de agente, más servidores MCP propios con aplicaciones OAuth de un clic | | **Fuentes por Notebook** | 50 (gratis) a 600 (Ultra, $249.99/mes) | Ilimitadas | | **Número de Notebooks** | 100 (gratis) a 500 (niveles de pago) | Ilimitado | diff --git a/README.hi.md b/README.hi.md index eb65ff489..2a26e2146 100644 --- a/README.hi.md +++ b/README.hi.md @@ -22,7 +22,7 @@ # SurfSense: कॉम्पिटिटिव इंटेलिजेंस रिसर्च के लिए NotebookLM -SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। +SurfSense **AI एजेंट्स के लिए ओपन सोर्स कॉम्पिटिटिव इंटेलिजेंस प्लेटफ़ॉर्म** है, बिलकुल NotebookLM जैसा, पर लाइव स्क्रैपिंग कनेक्टर्स के साथ। आपके एजेंट प्रतिस्पर्धियों पर नज़र रखते हैं, रैंकिंग ट्रैक करते हैं, और **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed और ओपन वेब** से लाइव डेटा के साथ आपके बाज़ार की बात सुनते हैं, वह भी एक ही **REST API** या **MCP सर्वर** के ज़रिए। शेड्यूल्ड और इवेंट-ट्रिगर्ड एजेंट अपनी खोजों को ब्रीफ़ और अलर्ट में बदलते हैं, और एक बिल्ट-इन नॉलेज बेस हर खोज को साइटेशन के साथ खोजने योग्य बनाए रखता है। > [!NOTE] > **📢 हमारे NotebookLM-विकल्प उपयोगकर्ताओं के लिए एक सूचना** @@ -107,6 +107,7 @@ SurfSense **AI एजेंट्स के लिए ओपन सोर्स | **TikTok** | Research API अप्रूवल के बिना वीडियो, कमेंट, हैशटैग और प्रोफ़ाइल | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | स्थानीय प्रतिस्पर्धी और लीड रिसर्च के लिए स्थान, रेटिंग और रिव्यू | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | रैंक ट्रैकिंग और मार्केट मॉनिटरिंग के लिए लाइव SERP | [Google Search API](https://www.surfsense.com/google-search) | +| **Indeed** | सार्वजनिक नौकरी लिस्टिंग, सैलरी और पूरे विवरण के साथ, सर्च या कंपनी के अनुसार | [Indeed Scraper API](https://www.surfsense.com/indeed) | | **Web Crawl** | ओपन वेब का कोई भी पेज साफ़-सुथरे, स्ट्रक्चर्ड कंटेंट के रूप में | [Web Crawling API](https://www.surfsense.com/web-crawl) | | **External MCP Connectors** | कोई भी MCP सर्वर अपने एजेंट्स से जोड़ें, Notion, Slack, Jira और अन्य के लिए वन-क्लिक OAuth के साथ | [External MCP Connectors](https://www.surfsense.com/external-mcp-connectors) | @@ -246,7 +247,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | फ़ीचर | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **एजेंट्स के लिए लाइव मार्केट डेटा** | नहीं | REST API और MCP के ज़रिए Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search और वेब क्रॉल कनेक्टर | +| **एजेंट्स के लिए लाइव मार्केट डेटा** | नहीं | REST API और MCP के ज़रिए Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed और वेब क्रॉल कनेक्टर | | **MCP सर्वर** | नहीं | हर कनेक्टर नेटिव एजेंट टूल के रूप में उपलब्ध, साथ ही वन-क्लिक OAuth ऐप्स के साथ अपने MCP सर्वर लाने की सुविधा | | **प्रति नोटबुक स्रोत** | 50 (Free) से 600 (Ultra, $249.99/माह) | असीमित | | **नोटबुक की संख्या** | 100 (Free) से 500 (सशुल्क टियर) | असीमित | diff --git a/README.md b/README.md index ba289ada4..56af65dd1 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM for Competitive Intelligence Research -SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. +SurfSense is the **open-source competitive intelligence platform for AI agents**, like NotebookLM but with live scraping connectors. Your agents monitor competitors, track rankings, and listen to your market with live data from **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, and the open web**, through one **REST API** or **MCP server**. Scheduled and event-triggered agents turn what they find into briefs and alerts, and a built-in knowledge base keeps every finding searchable with citations. > [!NOTE] > **📢 A note for our NotebookLM-alternative users** @@ -107,6 +107,7 @@ Automations run full agent turns on a schedule or in response to events, then wr | **TikTok** | Videos, comments, hashtags, and profiles without Research API approval | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Places, ratings, and reviews for local competitor and lead research | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | Live SERPs for rank tracking and market monitoring | [Google Search API](https://www.surfsense.com/google-search) | +| **Indeed** | Public job postings with salaries and full descriptions, by search or company | [Indeed Scraper API](https://www.surfsense.com/indeed) | | **Web Crawl** | Any page on the open web as clean, structured content | [Web Crawling API](https://www.surfsense.com/web-crawl) | | **External MCP Connectors** | Bring any MCP server to your agents, with one-click OAuth for Notion, Slack, Jira, and more | [External MCP Connectors](https://www.surfsense.com/external-mcp-connectors) | @@ -245,7 +246,7 @@ Still comparing us as a NotebookLM alternative? Here is the honest breakdown. | Feature | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Live market data for agents** | No | Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, and web crawl connectors via REST API and MCP | +| **Live market data for agents** | No | Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed, and web crawl connectors via REST API and MCP | | **MCP server** | No | Every connector exposed as a native agent tool, plus bring-your-own MCP servers with one-click OAuth apps | | **Sources per Notebook** | 50 (Free) to 600 (Ultra, $249.99/mo) | Unlimited | | **Number of Notebooks** | 100 (Free) to 500 (paid tiers) | Unlimited | diff --git a/README.pt-BR.md b/README.pt-BR.md index 327e3373c..496398d19 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -22,7 +22,7 @@ # SurfSense: NotebookLM para Pesquisa de Inteligência Competitiva -O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. +O SurfSense é a **plataforma open source de inteligência competitiva para agentes de IA**, como o NotebookLM, mas com conectores de scraping ao vivo. Seus agentes monitoram concorrentes, acompanham rankings e escutam o seu mercado com dados ao vivo do **Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed e da web aberta**, por meio de uma única **API REST** ou de um **servidor MCP**. Agentes agendados ou acionados por eventos transformam o que encontram em relatórios e alertas, e uma base de conhecimento integrada mantém cada descoberta pesquisável, com citações. > [!NOTE] > **📢 Um recado para nossos usuários que buscavam uma alternativa ao NotebookLM** @@ -107,6 +107,7 @@ As automações executam turnos completos de agente de forma agendada ou em resp | **TikTok** | Vídeos, comentários, hashtags e perfis sem aprovação da Research API | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | Estabelecimentos, avaliações e reviews para pesquisa local de concorrentes e leads | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | SERPs ao vivo para acompanhamento de rankings e monitoramento de mercado | [Google Search API](https://www.surfsense.com/google-search) | +| **Indeed** | Vagas públicas com salários e descrições completas, por busca ou empresa | [Indeed Scraper API](https://www.surfsense.com/indeed) | | **Web Crawl** (rastreamento web) | Qualquer página da web aberta como conteúdo limpo e estruturado | [Web Crawling API](https://www.surfsense.com/web-crawl) | | **Conectores MCP externos** | Traga qualquer servidor MCP para seus agentes, com OAuth em um clique para Notion, Slack, Jira e outros | [External MCP Connectors](https://www.surfsense.com/external-mcp-connectors) | @@ -246,7 +247,7 @@ Ainda nos comparando como alternativa ao NotebookLM? Aqui está o comparativo ho | Recurso | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **Dados de mercado ao vivo para agentes** | Não | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search e rastreamento web via API REST e MCP | +| **Dados de mercado ao vivo para agentes** | Não | Conectores de Reddit, YouTube, Instagram, TikTok, Google Maps, Google Search, Indeed e rastreamento web via API REST e MCP | | **Servidor MCP** | Não | Cada conector exposto como ferramenta nativa de agente, além de servidores MCP próprios com apps OAuth em um clique | | **Fontes por Notebook** | 50 (gratuito) a 600 (Ultra, US$ 249,99/mês) | Ilimitadas | | **Número de Notebooks** | 100 (gratuito) a 500 (planos pagos) | Ilimitado | diff --git a/README.zh-CN.md b/README.zh-CN.md index 53a95968b..c75e71a81 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -22,7 +22,7 @@ # SurfSense：面向竞争情报研究的 NotebookLM -SurfSense 是**面向 AI 智能体的开源竞争情报平台**，就像 NotebookLM，但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**，利用来自 **Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search 和开放网络**的实时数据，监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警，内置的知识库则让每一条发现都可搜索、可引用。 +SurfSense 是**面向 AI 智能体的开源竞争情报平台**，就像 NotebookLM，但配备了实时抓取连接器。你的智能体可以通过一个 **REST API** 或 **MCP 服务器**，利用来自 **Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search、Indeed 和开放网络**的实时数据，监控竞争对手、追踪排名、倾听市场动态。定时和事件触发的智能体会把发现的内容转化为简报和预警，内置的知识库则让每一条发现都可搜索、可引用。 > [!NOTE] > **📢 致我们的 NotebookLM 替代品用户** @@ -107,6 +107,7 @@ SurfSense 是**面向 AI 智能体的开源竞争情报平台**，就像 Noteboo | **TikTok** | 视频、评论、话题标签和主页，无需 Research API 审批 | [TikTok Scraper API](https://www.surfsense.com/tiktok) | | **Google Maps** | 地点、评分和评论，用于本地竞争对手和潜在客户调研 | [Google Maps Scraper API](https://www.surfsense.com/google-maps) | | **Google Search** | 实时搜索结果页，用于排名追踪和市场监控 | [Google Search API](https://www.surfsense.com/google-search) | +| **Indeed** | 公开职位信息，含薪资与完整职位描述，按搜索或公司抓取 | [Indeed Scraper API](https://www.surfsense.com/indeed) | | **Web Crawl** | 把开放网络上的任意页面转为干净、结构化的内容 | [Web Crawling API](https://www.surfsense.com/web-crawl) | | **外部 MCP 连接器** | 将任意 MCP 服务器接入你的智能体，Notion、Slack、Jira 等支持一键 OAuth | [External MCP Connectors](https://www.surfsense.com/external-mcp-connectors) | @@ -246,7 +247,7 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 | 功能 | Google NotebookLM | SurfSense | |---------|-------------------|-----------| -| **面向智能体的实时市场数据** | 无 | 通过 REST API 和 MCP 提供 Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search 和网页爬取连接器 | +| **面向智能体的实时市场数据** | 无 | 通过 REST API 和 MCP 提供 Reddit、YouTube、Instagram、TikTok、Google Maps、Google Search、Indeed 和网页爬取连接器 | | **MCP 服务器** | 无 | 每个连接器都作为原生智能体工具暴露，还可自带 MCP 服务器并使用一键 OAuth 应用 | | **每个笔记本的来源数** | 50 个（免费版）至 600 个（Ultra 版，249.99 美元/月） | 无限制 | | **笔记本数量** | 100 个（免费版）至 500 个（付费档位） | 无限制 | diff --git a/docker/.env.example b/docker/.env.example index 2aa0806a8..7f4510320 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -457,6 +457,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 # TIKTOK_MICROS_PER_COMMENT=1500 +# INDEED_SCRAPE_MICROS_PER_JOB=3500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 3d2355460..44a3af872 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -297,6 +297,7 @@ MICROS_PER_PAGE=1000 # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 # TIKTOK_MICROS_PER_COMMENT=1500 +# INDEED_SCRAPE_MICROS_PER_JOB=3500 # Browser-listing retries when a feed is empty (profile feed is withheld from # flagged IPs; each retry draws a fresh rotating exit IP). Set to 1 for a static IP. # TIKTOK_LISTING_MAX_ATTEMPTS=3 diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts index 7d9320fb0..837c3eb44 100644 --- a/surfsense_web/lib/auth-utils.ts +++ b/surfsense_web/lib/auth-utils.ts @@ -44,6 +44,7 @@ const PUBLIC_ROUTE_PREFIXES = [ "/youtube", "/google-maps", "/google-search", + "/indeed", "/web-crawl", ]; diff --git a/surfsense_web/lib/connectors-marketing/google-maps.tsx b/surfsense_web/lib/connectors-marketing/google-maps.tsx index 03b1d247b..b09301ca6 100644 --- a/surfsense_web/lib/connectors-marketing/google-maps.tsx +++ b/surfsense_web/lib/connectors-marketing/google-maps.tsx @@ -306,6 +306,7 @@ export const googleMaps: ConnectorPageContent = { { label: "Instagram API", href: "/instagram" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, { label: "Read the docs", href: "/docs" }, ], diff --git a/surfsense_web/lib/connectors-marketing/google-search.tsx b/surfsense_web/lib/connectors-marketing/google-search.tsx index 7f436f9b0..a7b69c2ad 100644 --- a/surfsense_web/lib/connectors-marketing/google-search.tsx +++ b/surfsense_web/lib/connectors-marketing/google-search.tsx @@ -261,6 +261,7 @@ export const googleSearch: ConnectorPageContent = { { label: "Google Maps API", href: "/google-maps" }, { label: "Reddit API", href: "/reddit" }, { label: "Instagram API", href: "/instagram" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, { label: "Read the docs", href: "/docs" }, ], diff --git a/surfsense_web/lib/connectors-marketing/indeed.tsx b/surfsense_web/lib/connectors-marketing/indeed.tsx new file mode 100644 index 000000000..ce717f3cc --- /dev/null +++ b/surfsense_web/lib/connectors-marketing/indeed.tsx @@ -0,0 +1,327 @@ +import { IconBriefcase } from "@tabler/icons-react"; +import type { ConnectorPageContent } from "./types"; + +export const indeed: ConnectorPageContent = { + slug: "indeed", + name: "Indeed", + icon: IconBriefcase, + + metaTitle: "Indeed Scraper API for Jobs and Hiring Data | SurfSense", + metaDescription: + "Scrape public Indeed job postings with the SurfSense Indeed Scraper API: titles, companies, salaries, and full descriptions by search or company. No Indeed API. Start free.", + keywords: [ + "indeed scraper", + "indeed scraper api", + "indeed api", + "indeed jobs api", + "scrape indeed", + "indeed job scraper", + "job posting scraper", + "salary data api", + "hiring data api", + "indeed mcp", + "labor market data", + "recruiting data tool", + ], + + h1: "Indeed Scraper API for Job Postings and Hiring Data", + heroLede: + "The SurfSense Indeed API extracts public job postings, salaries, companies, and full descriptions by search query, company page, or job URL, without Indeed's official API. Give your AI agents a live feed of who is hiring, for what, at what pay, so you track the labor market as it moves.", + + transcript: { + prompt: "Find remote data analyst roles posted this week and what they pay", + toolCall: + 'indeed.scrape({ search_queries: ["data analyst"], location: "Remote",\n remote: "remote", from_days: 7, sort: "date", max_items: 30 })', + rows: [ + { + primary: "Senior Data Analyst · Acme Corp", + secondary: "Remote (US) · $120k–$145k/year · posted 2 days ago", + tag: "salary listed", + }, + { + primary: "Data Analyst, Growth · Globex", + secondary: "Remote · $95k–$110k/year · Indeed Apply", + tag: "buying signal", + }, + { + primary: "Marketing Data Analyst · Initech", + secondary: "Remote (US) · estimated $88k–$102k · posted today", + tag: "new today", + }, + ], + resultSummary: "30 jobs · 22 with salary · surfaced in 3.4s", + }, + + extractIntro: + "Every call returns structured job items. Point the API at a search query, an Indeed search or company page, or a single job URL, and set scrape_job_details for the full description per job.", + extractFields: [ + { + label: "Job", + description: "Title, job key, listing URL, apply URL, and whether Indeed Apply is enabled.", + }, + { + label: "Company", + description: "Company name, profile URL, star rating, and review count where available.", + }, + { + label: "Location", + description: "Formatted location, city, state, postal code, country, and remote or hybrid flags.", + }, + { + label: "Salary", + description: + "Pay text, min and max bounds, currency, period, and whether the figure is an Indeed estimate.", + }, + { + label: "Description", + description: + "Listing snippet by default; the full text and HTML description with scrape_job_details.", + }, + { + label: "Signals", + description: + "Job types, benefits, sponsored, urgently hiring, new, and expired flags, plus post age.", + }, + ], + + useCasesHeading: "What teams do with the Indeed API", + useCases: [ + { + title: "Competitor hiring intelligence", + description: + "Track what your competitors are hiring for and where. A spike in sales or ML roles is a roadmap signal months before it ships. Feed the stream to an agent that flags the moves that matter.", + }, + { + title: "Salary and compensation benchmarking", + description: + "Pull real posted salaries for a title in a location and benchmark your own bands against the live market, instead of a survey that is a year stale.", + }, + { + title: "Labor market and sector research", + description: + "Measure hiring demand for a role, skill, or sector over time. Turn thousands of postings into a demand index your analysts and clients can act on.", + }, + { + title: "Recruiting and lead sourcing", + description: + "Find companies actively hiring for a role and reach them while the need is hot. Job postings are a public, timely buying signal for staffing and B2B sales.", + }, + ], + + comparison: { + heading: "An Indeed API alternative built for agents", + intro: + "Indeed retired its public Publisher jobs API and gates data behind partner programs. If you cannot get access or need clean structured jobs now, here is how SurfSense compares.", + columnLabel: "DIY Indeed scraping", + rows: [ + { + feature: "Access", + official: "Publisher API retired; partner-gated and approval-only", + surfsense: "One API key; scrape public postings without an approval process", + }, + { + feature: "Anti-bot", + official: "You fight Cloudflare, fingerprinting, and CAPTCHAs yourself", + surfsense: "Warmed, rotated sessions managed for you; no proxy plumbing", + }, + { + feature: "Pricing", + official: "Proxy, browser, and maintenance costs you own", + surfsense: "Pay per job returned, with a free tier to start", + }, + { + feature: "Descriptions", + official: "Extra page fetch and parsing you build and maintain", + surfsense: "Full description per job with one scrape_job_details flag", + }, + { + feature: "Agent-ready", + official: "No; you build the harness yourself", + surfsense: "MCP server exposes indeed.scrape as a native tool", + }, + ], + }, + + api: { + platform: "indeed", + verb: "scrape", + mcpTool: "indeed.scrape", + requestBody: { + search_queries: ["data analyst"], + location: "Remote", + remote: "remote", + from_days: 7, + sort: "date", + max_items: 30, + }, + }, + + schema: { + requestNote: + "Provide at least one source: urls or search_queries. Up to 20 sources per call.", + request: [ + { + name: "urls", + type: "string[]", + defaultValue: "[]", + description: + "Indeed URLs: a search page (/jobs?q=&l=), a company jobs page (/cmp//jobs), or a single job (/viewjob?jk=...). Max 20.", + }, + { + name: "search_queries", + type: "string[]", + defaultValue: "[]", + description: + "Job search terms. Each returns up to max_items_per_query results, shaped by the filters below. Max 20.", + }, + { + name: "country", + type: "string", + defaultValue: '"us"', + description: "Country code selecting the Indeed domain, e.g. 'us', 'gb', 'de'.", + }, + { + name: "location", + type: "string", + description: "Where to search, e.g. 'Remote', 'New York, NY'.", + }, + { + name: "radius", + type: "integer", + description: "Search radius in miles or km around location.", + }, + { + name: "job_type", + type: "string", + description: "Employment type: fulltime, parttime, contract, internship, and more.", + }, + { + name: "level", + type: "string", + description: "Experience level: entry_level, mid_level, or senior_level.", + }, + { + name: "remote", + type: "string", + description: "Work model filter: remote or hybrid.", + }, + { + name: "from_days", + type: "integer", + description: "Only return jobs posted within the last N days.", + }, + { + name: "sort", + type: "string", + defaultValue: '"relevance"', + description: "Result ordering: relevance or date.", + }, + { + name: "scrape_job_details", + type: "boolean", + defaultValue: "false", + description: + "Fetch each job's detail page for the full description. Slower: one extra page load per job.", + }, + { + name: "max_items", + type: "integer", + defaultValue: "25", + description: "Max total jobs to return across all sources. 1 to 100.", + }, + { + name: "max_items_per_query", + type: "integer", + defaultValue: "25", + description: "Max jobs to pull per search or company target.", + }, + ], + responseNote: + "The response is { items: [...] } with one flat item per job. Fields Indeed omits are null. One returned job is one billable unit.", + response: [ + { + name: "jobKey / jobUrl / applyUrl", + type: "string", + description: "Indeed job key, listing URL, and third-party apply URL.", + }, + { + name: "title", + type: "string", + description: "The job title as posted.", + }, + { + name: "company / companyUrl", + type: "string", + description: "Company name and its Indeed profile URL.", + }, + { + name: "companyRating / companyReviewCount", + type: "number / integer", + description: "Employer star rating and number of reviews, where Indeed shows them.", + }, + { + name: "formattedLocation / isRemote / remoteType", + type: "string / boolean", + description: "Location string plus remote and hybrid flags.", + }, + { + name: "salary", + type: "object", + description: + "salaryText, salaryMin, salaryMax, currency, period, and isEstimated when the pay is an Indeed estimate.", + }, + { + name: "jobTypes / benefits", + type: "string[]", + description: "Employment types and listed benefits parsed from the posting.", + }, + { + name: "descriptionText / descriptionHtml", + type: "string", + description: "Snippet by default; the full description when scrape_job_details is set.", + }, + { + name: "sponsored / urgentlyHiring / isNew / expired", + type: "boolean", + description: "Listing flags for ranking and filtering.", + }, + { + name: "age / datePublished / scrapedAt", + type: "string", + description: "Relative post age, ISO publish date, and when the job was scraped.", + }, + ], + }, + + faq: [ + { + question: "Is scraping Indeed legal?", + answer: + "SurfSense reads only public Indeed job postings, the same listings any logged-out visitor can see. It never logs in and cannot access private or applicant data. As always, review Indeed's terms and your own compliance needs before you run at scale.", + }, + { + question: "Does Indeed have an official jobs API?", + answer: + "Indeed retired its public Publisher jobs API and now gates job data behind partner and approval programs. SurfSense is an independent alternative: you call one API, or add the MCP server to your agent, and get structured public postings back.", + }, + { + question: "Can I get the full job description?", + answer: + "Yes. By default each job returns the listing snippet, which is fast. Set scrape_job_details to true and SurfSense fetches each job's detail page for the full description text and HTML, at the cost of one extra page load per job.", + }, + { + question: "What are the rate limits?", + answer: + "Each call returns up to 100 jobs across all sources, with up to 20 URLs or search queries per request. SurfSense manages the anti-bot request budget for you, so you scale reads without running proxies or a headless browser yourself.", + }, + ], + + related: [ + { label: "Reddit API", href: "/reddit" }, + { label: "YouTube API", href: "/youtube" }, + { label: "Google Maps API", href: "/google-maps" }, + { label: "SERP API", href: "/google-search" }, + { label: "Web Crawl API", href: "/web-crawl" }, + { label: "SurfSense MCP Server", href: "/mcp-server" }, + ], +}; diff --git a/surfsense_web/lib/connectors-marketing/index.ts b/surfsense_web/lib/connectors-marketing/index.ts index 826a1f261..be3948398 100644 --- a/surfsense_web/lib/connectors-marketing/index.ts +++ b/surfsense_web/lib/connectors-marketing/index.ts @@ -1,5 +1,6 @@ import { googleMaps } from "./google-maps"; import { googleSearch } from "./google-search"; +import { indeed } from "./indeed"; import { instagram } from "./instagram"; import { reddit } from "./reddit"; import { tiktok } from "./tiktok"; @@ -17,6 +18,7 @@ const CONNECTOR_LIST: ConnectorPageContent[] = [ tiktok, googleMaps, googleSearch, + indeed, webCrawl, ]; diff --git a/surfsense_web/lib/connectors-marketing/instagram.tsx b/surfsense_web/lib/connectors-marketing/instagram.tsx index 5422a0c90..999be0de8 100644 --- a/surfsense_web/lib/connectors-marketing/instagram.tsx +++ b/surfsense_web/lib/connectors-marketing/instagram.tsx @@ -288,6 +288,7 @@ export const instagram: ConnectorPageContent = { { label: "Reddit API", href: "/reddit" }, { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, ], }; diff --git a/surfsense_web/lib/connectors-marketing/reddit.tsx b/surfsense_web/lib/connectors-marketing/reddit.tsx index 2a930afe9..c76c9984f 100644 --- a/surfsense_web/lib/connectors-marketing/reddit.tsx +++ b/surfsense_web/lib/connectors-marketing/reddit.tsx @@ -315,6 +315,7 @@ export const reddit: ConnectorPageContent = { { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, ], }; diff --git a/surfsense_web/lib/connectors-marketing/tiktok.tsx b/surfsense_web/lib/connectors-marketing/tiktok.tsx index 6063fd1ae..9a765ef0e 100644 --- a/surfsense_web/lib/connectors-marketing/tiktok.tsx +++ b/surfsense_web/lib/connectors-marketing/tiktok.tsx @@ -284,6 +284,7 @@ export const tiktok: ConnectorPageContent = { { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, ], }; diff --git a/surfsense_web/lib/connectors-marketing/web-crawl.tsx b/surfsense_web/lib/connectors-marketing/web-crawl.tsx index 1cb4d5dac..1684a61e2 100644 --- a/surfsense_web/lib/connectors-marketing/web-crawl.tsx +++ b/surfsense_web/lib/connectors-marketing/web-crawl.tsx @@ -283,6 +283,7 @@ export const webCrawl: ConnectorPageContent = { { label: "Google Maps API", href: "/google-maps" }, { label: "Reddit API", href: "/reddit" }, { label: "Instagram API", href: "/instagram" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, { label: "Read the docs", href: "/docs" }, ], diff --git a/surfsense_web/lib/connectors-marketing/youtube.tsx b/surfsense_web/lib/connectors-marketing/youtube.tsx index fceaee940..431c78357 100644 --- a/surfsense_web/lib/connectors-marketing/youtube.tsx +++ b/surfsense_web/lib/connectors-marketing/youtube.tsx @@ -275,6 +275,7 @@ export const youtube: ConnectorPageContent = { { label: "Google Maps API", href: "/google-maps" }, { label: "SERP API", href: "/google-search" }, { label: "Web Crawl API", href: "/web-crawl" }, + { label: "Indeed API", href: "/indeed" }, { label: "SurfSense MCP Server", href: "/mcp-server" }, ], }; diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index 456e52008..a8825e52a 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -2,6 +2,7 @@ import type { ComponentType } from "react"; import { GoogleMapsIcon, GoogleSearchIcon, + IndeedIcon, InstagramIcon, RedditIcon, TikTokIcon, @@ -89,6 +90,12 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ icon: GoogleSearchIcon, verbs: [{ name: "google_search.scrape", verb: "scrape", label: "Scrape" }], }, + { + id: "indeed", + label: "Indeed", + icon: IndeedIcon, + verbs: [{ name: "indeed.scrape", verb: "scrape", label: "Scrape" }], + }, { id: "web", label: "Web", diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx index c1f61978b..5bae99d1b 100644 --- a/surfsense_web/lib/playground/platform-icons.tsx +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -28,4 +28,5 @@ export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram") export const TikTokIcon = brandIcon("/connectors/tiktok.svg", "TikTok"); export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps"); export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search"); +export const IndeedIcon = brandIcon("/connectors/indeed.svg", "Indeed"); export const WebIcon = brandIcon("/connectors/web.svg", "Web"); diff --git a/surfsense_web/public/connectors/indeed.svg b/surfsense_web/public/connectors/indeed.svg new file mode 100644 index 000000000..39d792bd6 --- /dev/null +++ b/surfsense_web/public/connectors/indeed.svg @@ -0,0 +1,5 @@ + + + + + From 2a3d750cbeec21a5336e396fabbe37835f202c4d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 19:08:50 +0200 Subject: [PATCH 062/217] fix: use official indeed brand mark for connector icon --- surfsense_web/public/connectors/indeed.svg | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/surfsense_web/public/connectors/indeed.svg b/surfsense_web/public/connectors/indeed.svg index 39d792bd6..9265a1934 100644 --- a/surfsense_web/public/connectors/indeed.svg +++ b/surfsense_web/public/connectors/indeed.svg @@ -1,5 +1 @@ - - - - - +Indeed From b53f00231d24b653bd969c63684c015b25e244d2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 19:57:19 +0200 Subject: [PATCH 063/217] fix: parse indeed /viewjob detail from window._rootProps --- .../platforms/indeed_jobs/parsers.py | 32 ++++++++++++++++--- .../platforms/indeed_jobs/test_parsers.py | 20 ++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py index b61b13e6a..87824e8c7 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py @@ -22,7 +22,10 @@ _DEFAULT_BASE = "https://www.indeed.com" _JOBCARDS_ANCHOR = 'window.mosaic.providerData["mosaic-provider-jobcards"]=' -# A /viewjob page assigns the full posting to this global. +# A /viewjob page carries the posting model in ``window._rootProps`` (JSON, +# under ``preloadedVJData``); older pages inlined it as ``window._initialData``. +_ROOT_PROPS_ANCHOR = "window._rootProps" +_ROOT_PROPS_KEY = "preloadedVJData" _INITIAL_DATA_ANCHOR = "window._initialData" # Indeed's extractedSalary.type -> our SalaryPeriod. @@ -82,14 +85,14 @@ def extract_jobcards_blob(html: str) -> dict | None: return data if isinstance(data, dict) else None -def extract_initial_data(html: str) -> dict | None: - """Decode the ``window._initialData`` assignment on a /viewjob page.""" +def _decode_assignment(html: str, anchor: str) -> dict | None: + """Decode the balanced JSON object assigned after ``anchor``, or ``None``.""" import json - idx = html.find(_INITIAL_DATA_ANCHOR) + idx = html.find(anchor) if idx == -1: return None - brace = html.find("{", idx + len(_INITIAL_DATA_ANCHOR)) + brace = html.find("{", idx + len(anchor)) if brace == -1: return None blob = _brace_match(html, brace) @@ -102,6 +105,25 @@ def extract_initial_data(html: str) -> dict | None: return data if isinstance(data, dict) else None +def extract_initial_data(html: str) -> dict | None: + """Return a /viewjob posting model rooted at ``jobInfoWrapperModel``. + + Prefers ``window._rootProps`` (JSON) unwrapped at ``preloadedVJData``; falls + back to a legacy inline ``window._initialData`` blob. ``window._initialData`` + is now a JS object literal that references other globals, so it is not JSON + and is skipped when the JSON parse fails. + """ + root = _decode_assignment(html, _ROOT_PROPS_ANCHOR) + if isinstance(root, dict): + vj = root.get(_ROOT_PROPS_KEY) + if isinstance(vj, dict) and vj.get("jobInfoWrapperModel"): + return vj + legacy = _decode_assignment(html, _INITIAL_DATA_ANCHOR) + if isinstance(legacy, dict) and legacy.get("jobInfoWrapperModel"): + return legacy + return None + + def job_results(blob: dict | None) -> list[dict[str, Any]]: """Return the raw job records from a decoded blob.""" if not isinstance(blob, dict): diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py index 38a6544b6..43fed6948 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py @@ -160,6 +160,26 @@ def test_parse_job_detail_extracts_description_and_fields(): assert (sal["salaryMin"], sal["salaryMax"], sal["period"]) == (60000, 90000, "year") +def test_parse_job_detail_reads_rootprops_shape(): + # Live /viewjob assigns window._rootProps (JSON) with the model under + # preloadedVJData; window._initialData is now a non-JSON JS literal that + # references other globals and must be skipped, not misparsed. + html = ( + "" + '' + ) + detail = parse_job_detail(html) + assert detail["title"] == "Data Analyst" + assert detail["company"] == "Acme" + assert detail["descriptionHtml"] == "

Full JD text

" + assert detail["descriptionText"] == "Full JD text" + + def test_parse_job_detail_omits_blank_fields(): # A page with a header but no salary/description must not emit those keys, # so a merge won't clobber listing values with blanks. From b171002d15de8e4fd57d0ba1fe9b2bc0a920c358 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 19:57:19 +0200 Subject: [PATCH 064/217] test: add live indeed scraper e2e harness --- .../scripts/e2e_indeed_scraper.py | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 surfsense_backend/scripts/e2e_indeed_scraper.py diff --git a/surfsense_backend/scripts/e2e_indeed_scraper.py b/surfsense_backend/scripts/e2e_indeed_scraper.py new file mode 100644 index 000000000..24143af4f --- /dev/null +++ b/surfsense_backend/scripts/e2e_indeed_scraper.py @@ -0,0 +1,149 @@ +"""Manual functional e2e for the Indeed scraper (app/proprietary/platforms/indeed_jobs). + +Run from the backend directory: + cd surfsense_backend + uv run python scripts/e2e_indeed_scraper.py + # or: .venv/bin/python scripts/e2e_indeed_scraper.py + +This is NOT a pytest test (it needs live network + a residential/custom proxy). +All steps share one warmed browser session: + + Step 0 — go/no-go probe: open a session, fetch a live search page, and assert + its embedded job-cards blob parses into results. If this fails the whole + approach is blocked on this IP/proxy — later steps are skipped. + Step 1 — search query -> job items; keep one discovered /viewjob URL. + Step 2 — scrape a search URL via startUrls. + Step 3 — scrape the discovered /viewjob URL and assert a full description. + Step 4 — scrape_job_details enrichment: a query with detail pages fetched. +""" + +import asyncio +import sys +from pathlib import Path + +from dotenv import load_dotenv + +# --- bootstrap: load .env and put the backend root on sys.path before app.* --- +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +from app.proprietary.platforms.indeed_jobs import ( # noqa: E402 + IndeedScrapeInput, + scrape_indeed, +) +from app.proprietary.platforms.indeed_jobs.fetch import open_session # noqa: E402 +from app.proprietary.platforms.indeed_jobs.parsers import ( # noqa: E402 + extract_jobcards_blob, + job_results, +) +from app.proprietary.platforms.indeed_jobs.url_resolver import ( # noqa: E402 + build_search_url, +) + +_QUERY = "data analyst" +_LOCATION = "Remote" + + +def _hr(title: str) -> None: + print(f"\n{'=' * 70}\n{title}\n{'=' * 70}") + + +def _check(label: str, ok: bool, detail: str = "") -> bool: + print(f" [{'PASS' if ok else 'FAIL'}] {label}{f' — {detail}' if detail else ''}") + return ok + + +async def step0_probe(sess, state: dict) -> bool: + _hr("STEP 0 — go/no-go: warmed session + parseable job cards") + url = build_search_url(_QUERY, location=_LOCATION) + html = await sess.fetch_html(url) + raws = job_results(extract_jobcards_blob(html)) + print(f" {url} -> job_cards={len(raws)}") + return _check("search page parsed job cards", len(raws) > 0, f"{len(raws)} cards") + + +async def step1_search(sess, state: dict) -> bool: + _hr("STEP 1 — search query -> items") + items = await scrape_indeed( + IndeedScrapeInput(queries=[_QUERY], location=_LOCATION, maxItems=5), + limit=5, + session=sess, + ) + for it in items[:5]: + print(f" - {it.get('jobKey')} | {it.get('title')} @ {it.get('company')}") + state["job_url"] = next( + (it["jobUrl"] for it in items if it.get("jobUrl")), None + ) + return _check("search returned jobs", len(items) > 0, f"{len(items)} jobs") + + +async def step2_search_url(sess, state: dict) -> bool: + _hr("STEP 2 — scrape a search URL (startUrls)") + url = build_search_url(_QUERY, location=_LOCATION) + items = await scrape_indeed( + IndeedScrapeInput(startUrls=[{"url": url}], maxItems=5), + limit=5, + session=sess, + ) + return _check("search URL returned jobs", len(items) > 0, f"{len(items)} jobs") + + +async def step3_viewjob(sess, state: dict) -> bool: + _hr("STEP 3 — scrape a discovered /viewjob URL (full description)") + url = state.get("job_url") + if not url: + return _check("had a discovered job URL", False) + items = await scrape_indeed( + IndeedScrapeInput(startUrls=[{"url": url}], maxItems=1), + limit=1, + session=sess, + ) + desc = items[0].get("descriptionText") if items else None + print(f" url={url}\n description_chars={len(desc or '')}") + return _check("viewjob returned a description", bool(desc), url) + + +async def step4_enrich(sess, state: dict) -> bool: + _hr("STEP 4 — search with scrape_job_details enrichment") + items = await scrape_indeed( + IndeedScrapeInput( + queries=[_QUERY], + location=_LOCATION, + maxItems=3, + scrapeJobDetails=True, + ), + limit=3, + session=sess, + ) + # descriptionHtml is set only by the detail page; listings never carry it, + # so it proves enrichment actually merged the /viewjob model. + enriched = [i for i in items if i.get("descriptionHtml")] + return _check( + "enriched items carry full descriptionHtml", + bool(enriched), + f"{len(enriched)}/{len(items)} enriched", + ) + + +async def main() -> int: + state: dict = {} + async with open_session() as sess: + results = [await step0_probe(sess, state)] + if not results[-1]: + print("\nprobe failed — Indeed is blocking this IP/proxy. Aborting.") + return 1 + results.append(await step1_search(sess, state)) + results.append(await step2_search_url(sess, state)) + results.append(await step3_viewjob(sess, state)) + results.append(await step4_enrich(sess, state)) + _hr("SUMMARY") + print(f" {sum(results)}/{len(results)} steps passed") + return 0 if all(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) From 6180b969129ac4b878dabb0fca22554c3746954e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 20:24:22 +0200 Subject: [PATCH 065/217] fix: fail fast on gated indeed pagination, keep collected pages --- .../platforms/indeed_jobs/fetch.py | 12 +++++--- .../platforms/indeed_jobs/scraper.py | 27 ++++++++++++++--- .../indeed_jobs/test_fetch_resilience.py | 10 +++++++ .../platforms/indeed_jobs/test_scraper.py | 29 +++++++++++++++++-- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py index 9f913a579..c3be9c2a8 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/fetch.py @@ -140,14 +140,18 @@ class IndeedSession: await self._timed_fetch(f"https://{domain}/", google_search=True) self._warmed.add(domain) - async def fetch_html(self, url: str) -> str: + async def fetch_html(self, url: str, *, max_rotations: int | None = None) -> str: """Return a search/company/job page's HTML through the warmed session. Rotates the IP and re-warms on a security-wall bounce or timeout; raises - :class:`IndeedAccessBlockedError` once every rotation is exhausted. + :class:`IndeedAccessBlockedError` once rotations are exhausted. ``max_rotations`` + overrides the default budget: pass ``0`` to fail fast on a systematically + gated page (e.g. anonymous pagination) instead of burning rotations on a + block no fresh IP will clear. """ if self._session is None: await self.start() + budget = _MAX_ROTATIONS if max_rotations is None else max_rotations domain = urlparse(url).hostname or "www.indeed.com" attempt = 0 while True: @@ -163,9 +167,9 @@ class IndeedSession: except Exception as e: logger.warning("[indeed] fetch failed on %s: %s", url, e) - if attempt >= _MAX_ROTATIONS: + if attempt >= budget: raise IndeedAccessBlockedError( - f"Indeed refused {url} on {attempt} rotated IPs" + f"Indeed refused {url} after {attempt + 1} attempt(s)" ) attempt += 1 await self._rotate() diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py index 0c9fcfb16..01371d281 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py @@ -2,7 +2,9 @@ :func:`iter_indeed` streams flat job items from one warmed browser session: ``startUrls`` (search/company pages) or ``queries`` are paged by the ``start`` -offset, deduped by ``jobKey``, and stopped on the first page that adds nothing. +offset, deduped by ``jobKey``, and stopped on the first page that adds nothing +or that Indeed gates (anonymous pagination is bounced past page 1, so a gated +page ends that target and keeps the pages already yielded). :func:`scrape_indeed` collects the stream under a caller ``limit``. Targets run sequentially on a single session — a browser per target would cost @@ -16,7 +18,7 @@ from collections.abc import AsyncIterator from typing import Any from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse -from .fetch import IndeedSession, now_iso, open_session +from .fetch import IndeedAccessBlockedError, IndeedSession, now_iso, open_session from .parsers import ( extract_jobcards_blob, job_results, @@ -60,7 +62,21 @@ async def _paginate( seen: set[str] = set() emitted = 0 for page in range(_MAX_PAGES): - html = await session.fetch_html(_with_start(first_url, page * _PAGE_STEP)) + try: + # Rotate to establish first-page access, but fail fast on deeper + # pages: Indeed gates anonymous pagination to secure.indeed.com, a + # block no fresh IP clears — retrying it only burns the time budget. + # ponytail: deeper pages are dropped rather than retried; lift the + # cap with an async job if full-depth pagination is ever needed. + html = await session.fetch_html( + _with_start(first_url, page * _PAGE_STEP), + max_rotations=None if page == 0 else 0, + ) + except IndeedAccessBlockedError: + if page == 0: + raise # never got in on this target; surface the block + logger.info("[indeed] pagination gated at start=%d; stopping", page * _PAGE_STEP) + return # keep the pages already yielded raws = job_results(extract_jobcards_blob(html)) if not raws: return @@ -127,7 +143,10 @@ async def _enrich(session: IndeedSession, item: dict[str, Any], base_url: str) - if not isinstance(job_url, str): return try: - detail = parse_job_detail(await session.fetch_html(job_url), base_url=base_url) + # Fail fast: enrichment is best-effort, so a gated detail page must not + # rotate IPs and eat the run's time budget for one job's description. + html = await session.fetch_html(job_url, max_rotations=0) + detail = parse_job_detail(html, base_url=base_url) except Exception as exc: logger.warning("[indeed] detail fetch failed for %s: %s", job_url, exc) return diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py index 4eae65f84..258f5154d 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_fetch_resilience.py @@ -85,6 +85,16 @@ async def test_raises_after_exhausting_rotations(): assert session.rotations == 3 +@pytest.mark.asyncio +async def test_max_rotations_zero_fails_fast(): + # A gated page (pagination) must raise on the first block without rotating. + ctrl = _Controller(["BLOCK", "OK"]) + session = IndeedSession(ctrl.factory) + with pytest.raises(IndeedAccessBlockedError): + await session.fetch_html(_URL, max_rotations=0) + assert session.rotations == 0 + + @pytest.mark.asyncio async def test_warms_domain_once_without_rotation(): ctrl = _Controller(["OK", "OK"]) diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py index e5ea23452..acf52296b 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py @@ -7,6 +7,7 @@ from urllib.parse import parse_qs, urlparse import pytest +from app.proprietary.platforms.indeed_jobs.fetch import IndeedAccessBlockedError from app.proprietary.platforms.indeed_jobs.schemas import IndeedScrapeInput from app.proprietary.platforms.indeed_jobs.scraper import iter_indeed, scrape_indeed @@ -38,16 +39,21 @@ def _detail_html(job_key: str) -> str: class _FakeSession: """Returns per-``start`` search pages (or a /viewjob detail) and records URLs.""" - def __init__(self, pages: dict[int, list[str]]) -> None: + def __init__( + self, pages: dict[int, list[str]], blocked_starts: set[int] | None = None + ) -> None: self._pages = pages + self._blocked = blocked_starts or set() self.fetched: list[str] = [] - async def fetch_html(self, url: str) -> str: + async def fetch_html(self, url: str, *, max_rotations: int | None = None) -> str: self.fetched.append(url) query = parse_qs(urlparse(url).query) if "/viewjob" in url: return _detail_html(query.get("jk", [""])[0]) start = int(query.get("start", ["0"])[0]) + if start in self._blocked: + raise IndeedAccessBlockedError(f"gated at start={start}") return _page_html(self._pages.get(start, [])) @@ -77,6 +83,25 @@ async def test_stops_when_a_page_is_all_duplicates(): } +@pytest.mark.asyncio +async def test_pagination_gate_keeps_earlier_pages(): + # Indeed gates anonymous pagination: page 0 serves, start=10 is blocked. + # The already-yielded page-0 items must survive; paging just stops. + session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}, blocked_starts={10}) + items = await _collect( + IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session + ) + assert [i["jobKey"] for i in items] == ["k1", "k2"] + + +@pytest.mark.asyncio +async def test_first_page_block_propagates(): + # A block on the very first page yielded nothing, so it surfaces as an error. + session = _FakeSession({}, blocked_starts={0}) + with pytest.raises(IndeedAccessBlockedError): + await _collect(IndeedScrapeInput(queries=["dev"]), session) + + @pytest.mark.asyncio async def test_respects_max_items_per_query(): session = _FakeSession({0: ["k1", "k2", "k3", "k4"]}) From c485fe4c43fc900ab75b16e49efc2dca5aa7fd3f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 15 Jul 2026 20:54:37 +0200 Subject: [PATCH 066/217] fix: scrape indeed first page only, steer subagent to one query --- .../builtins/indeed/system_prompt.md | 7 +- .../platforms/indeed_jobs/scraper.py | 86 ++++++------------- .../platforms/indeed_jobs/test_scraper.py | 30 ++----- 3 files changed, 36 insertions(+), 87 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md index b034b700a..36eb855d2 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/indeed/system_prompt.md @@ -14,10 +14,9 @@ Answer the delegated question from live Indeed job data gathered with your verb, - Finding jobs for a role: call `indeed_scrape` with `search_queries`; narrow with `location`, `country`, `job_type`, `level`, `remote`, `radius`, and `from_days`. - Scraping a specific Indeed URL: pass a search, company jobs, or single job URL in `urls`. - Full descriptions: set `scrape_job_details=true` to fetch each job's detail page (slower: one extra load per job). Leave it false when the listing snippet is enough. -- Controlling volume: use `max_items` for the total cap and `max_items_per_query` per search. -- Requested counts: `max_items` defaults to only 25 — when the task asks for N jobs, set `max_items` and `max_items_per_query` above N (with headroom for off-topic hits). A call that caps below the target can never satisfy it. -- Under-delivery: if the first call returns fewer on-topic results than requested, broaden it yourself — more query phrasings, wider `radius`, drop restrictive filters, larger `from_days` — before settling. Return `status=partial` only after the broadened attempt, never after a single narrow call. -- Batch multiple search terms into one call rather than many single-term calls. +- Cost model: Indeed is latency-bound. A cold session spends minutes solving Cloudflare, and each query returns only its first page (~15 jobs) — anonymous pagination is gated, so `max_items` above ~15/query buys nothing. Every extra phrasing adds wait, not depth. +- Default to ONE focused query. Only add phrasings when the role genuinely needs variety, and then put at most 2–3 in a SINGLE call's `search_queries` (they reuse one warmed session); never make separate `indeed_scrape` calls, which each pay the cold-start cost. +- Prefer returning the first page's on-topic hits promptly over exhaustive coverage. If a single call under-delivers against a large requested N, return `status=partial` noting the ~one-page-per-query ceiling — do not chase it with more calls. - Comparison requests: pull the current results, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (added, removed, salary/rank changes). diff --git a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py index 01371d281..68b70870a 100644 --- a/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py +++ b/surfsense_backend/app/proprietary/platforms/indeed_jobs/scraper.py @@ -1,14 +1,8 @@ """Orchestrator for the Indeed scraper. -:func:`iter_indeed` streams flat job items from one warmed browser session: -``startUrls`` (search/company pages) or ``queries`` are paged by the ``start`` -offset, deduped by ``jobKey``, and stopped on the first page that adds nothing -or that Indeed gates (anonymous pagination is bounced past page 1, so a gated -page ends that target and keeps the pages already yielded). -:func:`scrape_indeed` collects the stream under a caller ``limit``. - -Targets run sequentially on a single session — a browser per target would cost -far more than the sequential paging it would save. +:func:`iter_indeed` streams deduped job items from one warmed session; each +search/company target contributes its first page. :func:`scrape_indeed` collects +the stream under a caller ``limit``. Targets run sequentially to reuse the session. """ from __future__ import annotations @@ -16,9 +10,9 @@ from __future__ import annotations import logging from collections.abc import AsyncIterator from typing import Any -from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse +from urllib.parse import urlparse -from .fetch import IndeedAccessBlockedError, IndeedSession, now_iso, open_session +from .fetch import IndeedSession, now_iso, open_session from .parsers import ( extract_jobcards_blob, job_results, @@ -32,68 +26,36 @@ logger = logging.getLogger(__name__) __all__ = ["iter_indeed", "scrape_indeed"] -# Indeed's search pagination increments the ``start`` offset by 10. -_PAGE_STEP = 10 -# Indeed stops serving useful results past ~1000; cap pages well before that. -_MAX_PAGES = 10 - def _emit(partial: dict[str, Any]) -> dict[str, Any]: """Stamp ``scrapedAt`` and normalize through the output model.""" return IndeedItem(**{**partial, "scrapedAt": now_iso()}).to_output() -def _with_start(url: str, start: int) -> str: - """Return ``url`` with its ``start`` query param set (removed when 0).""" - parsed = urlparse(url) - params = [(k, v) for k, v in parse_qsl(parsed.query) if k != "start"] - if start: - params.append(("start", str(start))) - return urlunparse(parsed._replace(query=urlencode(params))) - - -async def _paginate( - session: IndeedSession, first_url: str, *, domain: str, max_items: int +async def _search_items( + session: IndeedSession, url: str, *, domain: str, max_items: int ) -> AsyncIterator[dict[str, Any]]: - """Yield items across pages of one search/company URL, deduped by ``jobKey``.""" + """Yield deduped job cards from one search/company page. + + ponytail: caps a query at its first page (~15 jobs) — anonymous Indeed gates + ``start>=10``; deeper depth needs an authenticated session or Indeed's API. + """ if max_items <= 0: return base_url = f"https://{domain}" + html = await session.fetch_html(url) seen: set[str] = set() emitted = 0 - for page in range(_MAX_PAGES): - try: - # Rotate to establish first-page access, but fail fast on deeper - # pages: Indeed gates anonymous pagination to secure.indeed.com, a - # block no fresh IP clears — retrying it only burns the time budget. - # ponytail: deeper pages are dropped rather than retried; lift the - # cap with an async job if full-depth pagination is ever needed. - html = await session.fetch_html( - _with_start(first_url, page * _PAGE_STEP), - max_rotations=None if page == 0 else 0, - ) - except IndeedAccessBlockedError: - if page == 0: - raise # never got in on this target; surface the block - logger.info("[indeed] pagination gated at start=%d; stopping", page * _PAGE_STEP) - return # keep the pages already yielded - raws = job_results(extract_jobcards_blob(html)) - if not raws: - return - added = 0 - for raw in raws: - item = parse_job(raw, base_url=base_url) - job_key = item.get("jobKey") - if isinstance(job_key, str): - if job_key in seen: - continue - seen.add(job_key) - yield _emit(item) - emitted += 1 - added += 1 - if emitted >= max_items: - return - if added == 0: # a whole page of duplicates: end of useful results + for raw in job_results(extract_jobcards_blob(html)): + item = parse_job(raw, base_url=base_url) + job_key = item.get("jobKey") + if isinstance(job_key, str): + if job_key in seen: + continue + seen.add(job_key) + yield _emit(item) + emitted += 1 + if emitted >= max_items: return @@ -175,7 +137,7 @@ async def iter_indeed( if item is not None: yield item continue - async for item in _paginate( + async for item in _search_items( session, url, domain=domain, max_items=input_model.maxItemsPerQuery ): job_key = item.get("jobKey") diff --git a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py index acf52296b..1d815ce98 100644 --- a/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py +++ b/surfsense_backend/tests/unit/platforms/indeed_jobs/test_scraper.py @@ -62,41 +62,29 @@ async def _collect(input_model, session) -> list[dict]: @pytest.mark.asyncio -async def test_paginates_and_dedupes_across_pages(): - session = _FakeSession({0: ["k1", "k2", "k3"], 10: ["k3", "k4"], 20: []}) +async def test_dedupes_within_page(): + session = _FakeSession({0: ["k1", "k2", "k2", "k3"]}) items = await _collect( IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session ) - assert [i["jobKey"] for i in items] == ["k1", "k2", "k3", "k4"] + assert [i["jobKey"] for i in items] == ["k1", "k2", "k3"] assert all(i["scrapedAt"] for i in items) # stamped by the orchestrator @pytest.mark.asyncio -async def test_stops_when_a_page_is_all_duplicates(): - session = _FakeSession({0: ["k1", "k2"], 10: ["k1", "k2"], 20: ["k9"]}) +async def test_does_not_fetch_deeper_pages(): + # First page only; ``start>=10`` must never be requested. + session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}) items = await _collect( IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session ) assert [i["jobKey"] for i in items] == ["k1", "k2"] - assert 20 not in { - int(parse_qs(urlparse(u).query).get("start", ["0"])[0]) for u in session.fetched - } + assert all("start=" not in u for u in session.fetched) @pytest.mark.asyncio -async def test_pagination_gate_keeps_earlier_pages(): - # Indeed gates anonymous pagination: page 0 serves, start=10 is blocked. - # The already-yielded page-0 items must survive; paging just stops. - session = _FakeSession({0: ["k1", "k2"], 10: ["k3"]}, blocked_starts={10}) - items = await _collect( - IndeedScrapeInput(queries=["dev"], maxItemsPerQuery=100), session - ) - assert [i["jobKey"] for i in items] == ["k1", "k2"] - - -@pytest.mark.asyncio -async def test_first_page_block_propagates(): - # A block on the very first page yielded nothing, so it surfaces as an error. +async def test_page_block_propagates(): + # Nothing yielded before the block, so it surfaces as an error. session = _FakeSession({}, blocked_starts={0}) with pytest.raises(IndeedAccessBlockedError): await _collect(IndeedScrapeInput(queries=["dev"]), session) From 7246f89c124fe7c515350777ac24c4aad393f7b3 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 01:29:18 +0530 Subject: [PATCH 067/217] refactor(chat): enhance message loading state management and update skeleton loading components for better user experience --- .../new-chat/[[...chat_id]]/page.tsx | 104 +++++------------- .../components/assistant-ui/thread.tsx | 89 ++++++++++++--- 2 files changed, 98 insertions(+), 95 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx index f5b857ce1..22298e574 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx @@ -39,7 +39,6 @@ import { TokenUsageProvider, } from "@/components/assistant-ui/token-usage-context"; import { Button } from "@/components/ui/button"; -import { Skeleton } from "@/components/ui/skeleton"; import { useSyncChatArtifacts } from "@/features/chat-artifacts"; import { type HitlDecision, @@ -104,6 +103,7 @@ const MobileArtifactsPanel = dynamic( /** Stable empty reference so idle threads don't re-render the interrupt provider. */ const EMPTY_PENDING_INTERRUPTS: PendingInterruptState[] = []; +const EMPTY_MESSAGES: ThreadMessageLike[] = []; function parseUrlChatId(id: string | string[] | undefined): number { let parsed = 0; @@ -115,61 +115,6 @@ function parseUrlChatId(id: string | string[] | undefined): number { return Number.isNaN(parsed) ? 0 : parsed; } -function ThreadMessagesSkeleton() { - return ( -
-
-
-
-
- -
- -
- - - -
- -
- -
- -
- - - -
- -
- -
-
- -
-
- -
-
-
-
- ); -} - export default function NewChatPage() { const params = useParams(); const queryClient = useQueryClient(); @@ -226,6 +171,16 @@ export default function NewChatPage() { const threadDetailQuery = useThreadDetail(activeThreadId); const threadMessagesQuery = useThreadMessages(activeThreadId); + const hydratedMessagesRef = useRef<{ + threadId: number | null; + data: typeof threadMessagesQuery.data; + }>({ threadId: null, data: undefined }); + const hasLiveStream = !!streamState && streamState.messages.length > 0; + const isActiveThreadHydrated = hydratedMessagesRef.current.threadId === activeThreadId; + const shouldHideStaleMessages = !!activeThreadId && !hasLiveStream && !isActiveThreadHydrated; + const isThreadMessagesLoading = + shouldHideStaleMessages && !threadMessagesQuery.error; + const runtimeMessages = shouldHideStaleMessages ? EMPTY_MESSAGES : displayMessages; // Live collaboration: sync session state and messages via Zero. Kept on the // page because "AI responding" reflects the currently-viewed thread. @@ -296,8 +251,8 @@ export default function NewChatPage() { // Latest displayed messages, read by the engine wrappers at call time so // history/slice seeds stay fresh without re-creating the callbacks. - const messagesRef = useRef(displayMessages); - messagesRef.current = displayMessages; + const messagesRef = useRef(runtimeMessages); + messagesRef.current = runtimeMessages; const buildCtx = useCallback( (): EngineContext => ({ @@ -309,11 +264,6 @@ export default function NewChatPage() { [workspaceId, activeThreadId] ); - const hydratedMessagesRef = useRef<{ - threadId: number | null; - data: typeof threadMessagesQuery.data; - }>({ threadId: null, data: undefined }); - // Reset thread-local runtime state on route/workspace changes. The durable // streaming overlay is preserved for any still-running thread (and the newly // viewed thread) via ``clearInactive`` so an in-flight turn survives nav. @@ -517,8 +467,11 @@ export default function NewChatPage() { // Handle new message from user const onNew = useCallback( - (message: AppendMessage) => startNewChat(buildCtx(), message), - [buildCtx] + (message: AppendMessage) => { + if (isThreadMessagesLoading) return Promise.resolve(); + return startNewChat(buildCtx(), message); + }, + [buildCtx, isThreadMessagesLoading] ); // Cancel the in-flight turn (targets the active stream's owner thread). @@ -747,11 +700,11 @@ export default function NewChatPage() { }, [buildCtx, pendingInterrupts, activeThreadId]); // Surface the thread's deliverables to the layout-level artifacts sidebar. - useSyncChatArtifacts(displayMessages); + useSyncChatArtifacts(runtimeMessages); // Create external store runtime const runtime = useExternalStoreRuntime({ - messages: displayMessages, + messages: runtimeMessages, isRunning, onNew, onEdit, @@ -764,12 +717,7 @@ export default function NewChatPage() { ? (threadDetailQuery.error ?? threadMessagesQuery.error) : null; const shouldShowThreadLoadError = - !!threadLoadError && !!activeThreadId && !currentThread && displayMessages.length === 0; - const isThreadMessagesLoading = - !!activeThreadId && - threadMessagesQuery.isPending && - displayMessages.length === 0 && - !threadMessagesQuery.error; + !!threadLoadError && !!activeThreadId && !currentThread && runtimeMessages.length === 0; if (shouldShowThreadLoadError) { return ( @@ -798,12 +746,10 @@ export default function NewChatPage() { >
- - {isThreadMessagesLoading ? ( -
- -
- ) : null} +
diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index c51bea165..634d9bd8b 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -103,6 +103,7 @@ import { useMediaQuery } from "@/hooks/use-media-query"; import { useElectronAPI } from "@/hooks/use-platform"; import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities"; import { captureDisplayToPngDataUrl } from "@/lib/chat/display-media-capture"; +import { canSubmitChat } from "@/lib/chat/can-submit-chat"; import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; import { slideoutOpenedTickAtom } from "@/lib/layout-events"; import { findPlatform, type PlaygroundPlatform } from "@/lib/playground/catalog"; @@ -147,13 +148,19 @@ function getComposerSuggestionAnchorPoint( interface ThreadProps { hasActiveThread?: boolean; + isLoadingMessages?: boolean; } -export const Thread: FC = ({ hasActiveThread = false }) => { - return ; +export const Thread: FC = ({ hasActiveThread = false, isLoadingMessages = false }) => { + return ( + + ); }; -const ThreadContent: FC = ({ hasActiveThread = false }) => { +const ThreadContent: FC = ({ + hasActiveThread = false, + isLoadingMessages = false, +}) => { return ( = ({ hasActiveThread = false }) => { footer={ hasActiveThread || !thread.isEmpty}> - + } > @@ -173,6 +180,8 @@ const ThreadContent: FC = ({ hasActiveThread = false }) => { + {isLoadingMessages ? : null} + = ({ hasActiveThread = false }) => { ); }; +const ThreadMessagesSkeletonBody: FC = () => { + return ( +
+
+ +
+ +
+ + + +
+ +
+ +
+ +
+ + + +
+ +
+ +
+
+ ); +}; + const PremiumQuotaPinnedAlert: FC = () => { const currentThreadState = useAtomValue(currentThreadAtom); const alertsByThread = useAtomValue(premiumAlertByThreadAtom); @@ -490,7 +529,11 @@ const ChatUnavailableNotice: FC<{ workspaceId: number; canConfigure: boolean }> ); }; -const Composer: FC = () => { +interface ComposerProps { + isLoadingMessages?: boolean; +} + +const Composer: FC = ({ isLoadingMessages = false }) => { const [mentionedDocuments, setMentionedDocuments] = useAtom(mentionedDocumentsAtom); const setSubmittedMentions = useSetAtom(submittedMentionsAtom); const [showDocumentPopover, setShowDocumentPopover] = useState(false); @@ -823,7 +866,7 @@ const Composer: FC = () => { ); const handleSubmit = useCallback(() => { - if (isThreadRunning || isBlockedByOtherUser) return; + if (isLoadingMessages || isThreadRunning || isBlockedByOtherUser) return; if (showDocumentPopover || showPromptPicker) return; if (clipboardInitialText) { @@ -844,6 +887,7 @@ const Composer: FC = () => { }, [ showDocumentPopover, showPromptPicker, + isLoadingMessages, isThreadRunning, isBlockedByOtherUser, clipboardInitialText, @@ -1016,15 +1060,17 @@ const Composer: FC = () => {
- {isThreadEmpty && isComposerInputEmpty ? ( + {!isLoadingMessages && isThreadEmpty && isComposerInputEmpty ? (
@@ -1087,12 +1133,16 @@ const ConnectedScraperIcons: FC<{ workspaceId: number }> = ({ workspaceId }) => interface ComposerActionProps { isBlockedByOtherUser?: boolean; + isLoadingMessages?: boolean; + isThreadRunning?: boolean; workspaceId: number; onChatModelSelected?: () => void; } const ComposerAction: FC = ({ isBlockedByOtherUser = false, + isLoadingMessages = false, + isThreadRunning = false, workspaceId, onChatModelSelected, }) => { @@ -1210,7 +1260,20 @@ const ComposerAction: FC = ({ // send that lacks a resolvable model, making this defense-in-depth. const isWorkspaceChatReady = setupStatus?.status === "ready"; - const isSendDisabled = isComposerEmpty || !isWorkspaceChatReady || isBlockedByOtherUser; + const isSendDisabled = !canSubmitChat({ + isLoadingMessages, + isThreadRunning, + isBlockedByOtherUser, + isComposerEmpty, + isWorkspaceChatReady, + }); + const sendTooltip = isLoadingMessages + ? "Loading conversation..." + : isBlockedByOtherUser + ? "Wait for AI to finish responding" + : isComposerEmpty + ? "Enter a message or add a screenshot to send" + : "Send message"; return (
@@ -1661,13 +1724,7 @@ const ComposerAction: FC = ({ !thread.isRunning}> Date: Thu, 16 Jul 2026 01:29:32 +0530 Subject: [PATCH 068/217] feat(chat): add CanSubmitChat interface and logic to determine chat submission eligibility --- surfsense_web/lib/chat/can-submit-chat.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 surfsense_web/lib/chat/can-submit-chat.ts diff --git a/surfsense_web/lib/chat/can-submit-chat.ts b/surfsense_web/lib/chat/can-submit-chat.ts new file mode 100644 index 000000000..106b298af --- /dev/null +++ b/surfsense_web/lib/chat/can-submit-chat.ts @@ -0,0 +1,23 @@ +export interface CanSubmitChatInput { + isLoadingMessages: boolean; + isThreadRunning: boolean; + isBlockedByOtherUser: boolean; + isComposerEmpty: boolean; + isWorkspaceChatReady: boolean; +} + +export function canSubmitChat({ + isLoadingMessages, + isThreadRunning, + isBlockedByOtherUser, + isComposerEmpty, + isWorkspaceChatReady, +}: CanSubmitChatInput): boolean { + return ( + !isLoadingMessages && + !isThreadRunning && + !isBlockedByOtherUser && + !isComposerEmpty && + isWorkspaceChatReady + ); +} From b100eda8c63f30178938ced147fb9b6ecaf66df6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:11:54 +0530 Subject: [PATCH 069/217] refactor(chat-header): update className handling for responsive design and improve layout consistency --- surfsense_web/components/assistant-ui/thread.tsx | 6 +++--- surfsense_web/components/new-chat/chat-header.tsx | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 634d9bd8b..f11e82952 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -102,8 +102,8 @@ import { useCommentsSync } from "@/hooks/use-comments-sync"; import { useMediaQuery } from "@/hooks/use-media-query"; import { useElectronAPI } from "@/hooks/use-platform"; import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities"; -import { captureDisplayToPngDataUrl } from "@/lib/chat/display-media-capture"; import { canSubmitChat } from "@/lib/chat/can-submit-chat"; +import { captureDisplayToPngDataUrl } from "@/lib/chat/display-media-capture"; import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; import { slideoutOpenedTickAtom } from "@/lib/layout-events"; import { findPlatform, type PlaygroundPlatform } from "@/lib/playground/catalog"; @@ -1715,10 +1715,10 @@ const ComposerAction: FC = ({ )}
-
+
!thread.isRunning}> diff --git a/surfsense_web/components/new-chat/chat-header.tsx b/surfsense_web/components/new-chat/chat-header.tsx index e83a95d6c..f315f7645 100644 --- a/surfsense_web/components/new-chat/chat-header.tsx +++ b/surfsense_web/components/new-chat/chat-header.tsx @@ -1,5 +1,6 @@ "use client"; +import { cn } from "@/lib/utils"; import { ImageModelSelector } from "./image-model-selector"; import { ModelSelector } from "./model-selector"; @@ -10,14 +11,16 @@ interface ChatHeaderProps { } export function ChatHeader({ workspaceId, className, onChatModelSelected }: ChatHeaderProps) { + const selectorClassName = cn(className, "sm:max-w-[180px] sm:min-w-0"); + return ( -
+
- +
); } From 804e5e42f3da0ed4d05b15ee54b98dfd6fa630d6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:34:27 +0530 Subject: [PATCH 070/217] refactor(citation-panel): remove showHeader prop and enhance close button functionality in CitationPanelContent --- .../assistant-ui/inline-citation.tsx | 2 +- .../citation-panel/citation-panel.tsx | 48 ++++++++----------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/surfsense_web/components/assistant-ui/inline-citation.tsx b/surfsense_web/components/assistant-ui/inline-citation.tsx index a66c22fa3..a846e8311 100644 --- a/surfsense_web/components/assistant-ui/inline-citation.tsx +++ b/surfsense_web/components/assistant-ui/inline-citation.tsx @@ -100,7 +100,7 @@ const NumericChunkCitation: FC<{ chunkId: number }> = ({ chunkId }) => { Citation
- +
diff --git a/surfsense_web/components/citation-panel/citation-panel.tsx b/surfsense_web/components/citation-panel/citation-panel.tsx index 34cc89c8b..f501281b9 100644 --- a/surfsense_web/components/citation-panel/citation-panel.tsx +++ b/surfsense_web/components/citation-panel/citation-panel.tsx @@ -8,6 +8,7 @@ import { useEffect, useMemo, useRef } from "react"; import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { MarkdownViewer } from "@/components/markdown-viewer"; import { Button } from "@/components/ui/button"; +import { Separator } from "@/components/ui/separator"; import { Spinner } from "@/components/ui/spinner"; import { documentsApiService } from "@/lib/apis/documents-api.service"; @@ -16,7 +17,6 @@ const DEFAULT_CHUNK_WINDOW = 5; interface CitationPanelContentProps { chunkId: number; onClose?: () => void; - showHeader?: boolean; } /** @@ -25,11 +25,7 @@ interface CitationPanelContentProps { * with the cited one visually highlighted and auto-scrolled into view. * The user can jump to the full document via the editor panel. */ -export const CitationPanelContent: FC = ({ - chunkId, - onClose, - showHeader = true, -}) => { +export const CitationPanelContent: FC = ({ chunkId, onClose }) => { const openEditorPanel = useSetAtom(openEditorPanelAtom); const chunkWindow = DEFAULT_CHUNK_WINDOW; @@ -85,32 +81,13 @@ export const CitationPanelContent: FC = ({ return ( <>
- {showHeader && ( -
-

Citation

-
- {onClose && ( - - )} -
-
- )} -
+

{data?.title ?? (isLoading ? "Loading…" : `Chunk #${chunkId}`)}

-
- {totalChunks > 0 && {totalChunks} chunks} +
{!isLoading && !error && data && ( )} + {onClose && ( + <> + + + + )}
From fc2366c81e87d32aeb12f0a5b2e0ff26f9eb8712 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:40:44 +0530 Subject: [PATCH 071/217] refactor(report-panel): enhance header layout and integrate Separator component for improved UI consistency --- .../components/report-panel/report-panel.tsx | 69 ++++++++++--------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/surfsense_web/components/report-panel/report-panel.tsx b/surfsense_web/components/report-panel/report-panel.tsx index 1fce9848c..076d10255 100644 --- a/surfsense_web/components/report-panel/report-panel.tsx +++ b/surfsense_web/components/report-panel/report-panel.tsx @@ -18,6 +18,7 @@ import { DropdownMenuItem, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; +import { Separator } from "@/components/ui/separator"; import { Spinner } from "@/components/ui/spinner"; import { useMediaQuery } from "@/hooks/use-media-query"; import { baseApiService } from "@/lib/apis/base-api.service"; @@ -444,39 +445,45 @@ export function ReportPanelContent({ return ( <> {showDesktopHeader ? ( - <> +
{/* Header — matches the Documents panel header pattern */} -
-

{isResume ? "Resume" : "Report"}

- {onClose && ( - - )} -
- - {!isResume && ( -
-
-

- {reportContent?.title || title} -

-
-
- {versionSwitcher} - {exportButton} - {copyButton} - {editingActions} -
+
+
+

+ {isResume ? "Resume" : reportContent?.title || title} +

- )} - +
+ {!isResume && ( + <> + {versionSwitcher} + {exportButton} + {copyButton} + {editingActions} + + )} + {onClose && ( + <> + {!isResume && ( + + )} + + + )} +
+
+
) : ( !isResume && (
From bf9d536757586a28a17a4102b7d43d3c6f3ebbf9 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:58:45 +0530 Subject: [PATCH 072/217] refactor(artifacts-panel): improve header layout and button styles for better UI consistency --- .../chat-artifacts/ui/artifacts-panel.tsx | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/surfsense_web/features/chat-artifacts/ui/artifacts-panel.tsx b/surfsense_web/features/chat-artifacts/ui/artifacts-panel.tsx index 7b3567d73..8c5b575ad 100644 --- a/surfsense_web/features/chat-artifacts/ui/artifacts-panel.tsx +++ b/surfsense_web/features/chat-artifacts/ui/artifacts-panel.tsx @@ -70,19 +70,25 @@ export function ArtifactsPanelContent({ onClose }: { onClose?: () => void }) { return ( <> -
-

Artifacts

- {onClose && ( - - )} +
+
+
+

Artifacts

+
+
+ {onClose && ( + + )} +
+
From 1385ff4789e06e7f3a51b1004df2826f7bab6087 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 14:14:10 +0530 Subject: [PATCH 073/217] refactor(connect-agent-dialog): update dialog title for clarity and improved user understanding --- surfsense_web/components/mcp/connect-agent-dialog.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_web/components/mcp/connect-agent-dialog.tsx b/surfsense_web/components/mcp/connect-agent-dialog.tsx index e46f602ec..417877b2d 100644 --- a/surfsense_web/components/mcp/connect-agent-dialog.tsx +++ b/surfsense_web/components/mcp/connect-agent-dialog.tsx @@ -43,7 +43,7 @@ export function ConnectAgentDialog({ className }: { className?: string }) { - Connect to Claude Code, Codex, OpenCode… + Connect your coding agent to SurfSense Give your coding agent access to SurfSense scrapers and your knowledge base. Create an API key under API Keys, choose your agent, then paste the config. From 38b784fbacb1f7f0a05e2cd2259a0d7963b8c6ff Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:44:49 +0530 Subject: [PATCH 074/217] feat(workspaces): implement workspace limits feature with backend integration and UI updates --- surfsense_backend/app/config/__init__.py | 3 + .../app/routes/workspaces_routes.py | 22 +++++++ .../unit/routes/test_workspaces_limits.py | 63 +++++++++++++++++++ .../atoms/workspaces/workspace-query.atoms.ts | 19 ++++-- .../layout/providers/LayoutDataProvider.tsx | 20 +++++- .../ui/dialogs/CreateWorkspaceDialog.tsx | 2 + .../layout/ui/icon-rail/IconRail.tsx | 31 ++++++--- .../layout/ui/shell/LayoutShell.tsx | 8 +++ .../layout/ui/sidebar/MobileSidebar.tsx | 12 +++- .../contracts/types/workspace.types.ts | 8 +++ .../lib/apis/workspaces-api.service.ts | 5 ++ surfsense_web/lib/query-client/cache-keys.ts | 1 + 12 files changed, 176 insertions(+), 18 deletions(-) create mode 100644 surfsense_backend/tests/unit/routes/test_workspaces_limits.py diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 92e448d9a..830a2a04e 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -850,6 +850,9 @@ class Config: # Auth AUTH_TYPE = os.getenv("AUTH_TYPE", "LOCAL") REGISTRATION_ENABLED = os.getenv("REGISTRATION_ENABLED", "TRUE").upper() == "TRUE" + # Max workspaces a user may own. The frontend reads this through the + # workspace limits route; do not duplicate this value client-side. + MAX_WORKSPACES_PER_USER = int(os.getenv("MAX_WORKSPACES_PER_USER", "100")) # Google OAuth GOOGLE_OAUTH_CLIENT_ID = os.getenv("GOOGLE_OAUTH_CLIENT_ID") diff --git a/surfsense_backend/app/routes/workspaces_routes.py b/surfsense_backend/app/routes/workspaces_routes.py index 9dd3f196c..53d4344f7 100644 --- a/surfsense_backend/app/routes/workspaces_routes.py +++ b/surfsense_backend/app/routes/workspaces_routes.py @@ -6,6 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from app.auth.context import AuthContext +from app.config import config from app.db import ( Permission, Workspace, @@ -81,6 +82,22 @@ async def create_workspace( user = auth.user try: workspace_data = workspace.model_dump() + not_deleting = ~Workspace.name.startswith("[DELETING] ") + owned_count = ( + await session.execute( + select(func.count()) + .select_from(Workspace) + .filter(Workspace.user_id == user.id, not_deleting) + ) + ).scalar_one() + if owned_count >= config.MAX_WORKSPACES_PER_USER: + raise HTTPException( + status_code=409, + detail=( + "Workspace limit reached. You can own at most " + f"{config.MAX_WORKSPACES_PER_USER} workspaces." + ), + ) # citations_enabled defaults to True (handled by Pydantic schema) # qna_custom_instructions defaults to None/empty (handled by DB) @@ -207,6 +224,11 @@ async def read_workspaces( ) from e +@router.get("/workspaces/limits") +async def read_workspace_limits(_auth: AuthContext = Depends(allow_any_principal)): + return {"max_workspaces_per_user": config.MAX_WORKSPACES_PER_USER} + + @router.get("/workspaces/{workspace_id}", response_model=WorkspaceRead) async def read_workspace( workspace_id: int, diff --git a/surfsense_backend/tests/unit/routes/test_workspaces_limits.py b/surfsense_backend/tests/unit/routes/test_workspaces_limits.py new file mode 100644 index 000000000..ab89d08bc --- /dev/null +++ b/surfsense_backend/tests/unit/routes/test_workspaces_limits.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException + +from app.routes import workspaces_routes +from app.schemas import WorkspaceCreate + +pytestmark = pytest.mark.unit + + +class _CountResult: + def __init__(self, count: int): + self.count = count + + def scalar_one(self) -> int: + return self.count + + +class _FakeSession: + def __init__(self, owned_count: int): + self.owned_count = owned_count + + async def execute(self, _statement): + return _CountResult(self.owned_count) + + +@pytest.mark.asyncio +async def test_read_workspace_limits_uses_backend_config(monkeypatch): + monkeypatch.setattr( + workspaces_routes.config, + "MAX_WORKSPACES_PER_USER", + 37, + raising=False, + ) + + result = await workspaces_routes.read_workspace_limits(_auth=SimpleNamespace()) + + assert result == {"max_workspaces_per_user": 37} + + +@pytest.mark.asyncio +async def test_create_workspace_rejects_when_owned_limit_reached(monkeypatch): + monkeypatch.setattr( + workspaces_routes.config, + "MAX_WORKSPACES_PER_USER", + 2, + raising=False, + ) + auth = SimpleNamespace(user=SimpleNamespace(id="user-1")) + session = _FakeSession(owned_count=2) + + with pytest.raises(HTTPException) as exc_info: + await workspaces_routes.create_workspace( + WorkspaceCreate(name="Extra", description=""), + session=session, + auth=auth, + ) + + assert exc_info.value.status_code == 409 + assert "at most 2 workspaces" in exc_info.value.detail diff --git a/surfsense_web/atoms/workspaces/workspace-query.atoms.ts b/surfsense_web/atoms/workspaces/workspace-query.atoms.ts index 85203cc1d..09e2aa290 100644 --- a/surfsense_web/atoms/workspaces/workspace-query.atoms.ts +++ b/surfsense_web/atoms/workspaces/workspace-query.atoms.ts @@ -6,14 +6,23 @@ import { cacheKeys } from "@/lib/query-client/cache-keys"; export const activeWorkspaceIdAtom = atom(null); -export const workspacesQueryParamsAtom = atom({ - skip: 0, - limit: 10, - owned_only: false, +export const workspaceLimitsAtom = atomWithQuery(() => { + return { + queryKey: cacheKeys.workspaces.limits, + staleTime: Infinity, + queryFn: async () => { + return workspacesApiService.getWorkspaceLimits(); + }, + }; }); export const workspacesAtom = atomWithQuery((get) => { - const queryParams = get(workspacesQueryParamsAtom); + const workspaceLimits = get(workspaceLimitsAtom).data; + const queryParams: GetWorkspacesRequest["queryParams"] = { + skip: 0, + ...(workspaceLimits ? { limit: workspaceLimits.max_workspaces_per_user } : {}), + owned_only: false, + }; return { queryKey: cacheKeys.workspaces.withQueryParams(queryParams), diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 682d2923e..a61a1a703 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -14,7 +14,7 @@ import { announcementsDialogAtom } from "@/atoms/layout/dialogs.atom"; import { removeChatTabAtom, syncChatTabAtom, type Tab } from "@/atoms/tabs/tabs.atom"; import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { deleteWorkspaceMutationAtom } from "@/atoms/workspaces/workspace-mutation.atoms"; -import { workspacesAtom } from "@/atoms/workspaces/workspace-query.atoms"; +import { workspaceLimitsAtom, workspacesAtom } from "@/atoms/workspaces/workspace-query.atoms"; import { ActionLogDialog } from "@/components/agent-action-log/action-log-dialog"; import { AnnouncementSpotlight } from "@/components/announcements/AnnouncementSpotlight"; import { AnnouncementsDialog } from "@/components/announcements/AnnouncementsDialog"; @@ -84,6 +84,7 @@ export function LayoutDataProvider({ refetch: refetchWorkspaces, isSuccess: workspacesLoaded, } = useAtomValue(workspacesAtom); + const { data: workspaceLimits } = useAtomValue(workspaceLimitsAtom); const { mutateAsync: deleteWorkspace } = useAtomValue(deleteWorkspaceMutationAtom); const currentThreadState = useAtomValue(currentThreadAtom); const resetCurrentThread = useSetAtom(resetCurrentThreadAtom); @@ -225,6 +226,13 @@ export function LayoutDataProvider({ createdAt: space.created_at, })); }, [workspacesData]); + const maxWorkspacesPerUser = workspaceLimits?.max_workspaces_per_user; + const ownedWorkspaceCount = workspaces.reduce( + (count, space) => count + (space.isOwner ? 1 : 0), + 0 + ); + const isAtWorkspaceLimit = + maxWorkspacesPerUser !== undefined && ownedWorkspaceCount >= maxWorkspacesPerUser; // Find active workspace from list, falling back to the route-scoped detail query. const activeWorkspace: Workspace | null = useMemo(() => { @@ -335,8 +343,14 @@ export function LayoutDataProvider({ ); const handleAddWorkspace = useCallback(() => { + if (isAtWorkspaceLimit) { + toast.error( + `Workspace limit reached. You can own at most ${maxWorkspacesPerUser} workspaces.` + ); + return; + } setIsCreateWorkspaceDialogOpen(true); - }, []); + }, [isAtWorkspaceLimit, maxWorkspacesPerUser]); const setAnnouncementsDialog = useSetAtom(announcementsDialogAtom); @@ -663,6 +677,8 @@ export function LayoutDataProvider({ onWorkspaceDelete={handleWorkspaceDeleteClick} onWorkspaceSettings={handleWorkspaceSettings} onAddWorkspace={handleAddWorkspace} + isAtWorkspaceLimit={isAtWorkspaceLimit} + maxWorkspacesPerUser={maxWorkspacesPerUser} workspace={activeWorkspace} navItems={navItems} onNavItemClick={handleNavItemClick} diff --git a/surfsense_web/components/layout/ui/dialogs/CreateWorkspaceDialog.tsx b/surfsense_web/components/layout/ui/dialogs/CreateWorkspaceDialog.tsx index 75192e5e2..d8577fe1d 100644 --- a/surfsense_web/components/layout/ui/dialogs/CreateWorkspaceDialog.tsx +++ b/surfsense_web/components/layout/ui/dialogs/CreateWorkspaceDialog.tsx @@ -6,6 +6,7 @@ import { useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useState } from "react"; import { useForm } from "react-hook-form"; +import { toast } from "sonner"; import * as z from "zod"; import { createWorkspaceMutationAtom } from "@/atoms/workspaces/workspace-mutation.atoms"; import { Button } from "@/components/ui/button"; @@ -87,6 +88,7 @@ export function CreateWorkspaceDialog({ open, onOpenChange }: CreateWorkspaceDia ); } catch (error) { console.error("Failed to create workspace:", error); + toast.error(error instanceof Error ? error.message : "Failed to create workspace"); setIsSubmitting(false); } }; diff --git a/surfsense_web/components/layout/ui/icon-rail/IconRail.tsx b/surfsense_web/components/layout/ui/icon-rail/IconRail.tsx index 1f700c89f..0eeac9ca7 100644 --- a/surfsense_web/components/layout/ui/icon-rail/IconRail.tsx +++ b/surfsense_web/components/layout/ui/icon-rail/IconRail.tsx @@ -20,6 +20,8 @@ interface IconRailProps { onWorkspaceDelete?: (workspace: Workspace) => void; onWorkspaceSettings?: (workspace: Workspace) => void; onAddWorkspace: () => void; + isAtWorkspaceLimit?: boolean; + maxWorkspacesPerUser?: number; isSingleRailMode?: boolean; onNewChat?: () => void; navItems?: NavItem[]; @@ -42,6 +44,8 @@ export function IconRail({ onWorkspaceDelete, onWorkspaceSettings, onAddWorkspace, + isAtWorkspaceLimit = false, + maxWorkspacesPerUser, isSingleRailMode = false, onNewChat, navItems = [], @@ -78,6 +82,10 @@ export function IconRail({ })), ] : []; + const addWorkspaceLabel = + isAtWorkspaceLimit && maxWorkspacesPerUser !== undefined + ? `Workspace limit reached: ${maxWorkspacesPerUser}` + : "Add workspace"; return (
@@ -99,18 +107,21 @@ export function IconRail({ - + + + - Add workspace + {addWorkspaceLabel} diff --git a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx index 44a2259ee..70bb317dd 100644 --- a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx +++ b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx @@ -95,6 +95,8 @@ interface LayoutShellProps { onWorkspaceDelete?: (workspace: Workspace) => void; onWorkspaceSettings?: (workspace: Workspace) => void; onAddWorkspace: () => void; + isAtWorkspaceLimit?: boolean; + maxWorkspacesPerUser?: number; workspace: Workspace | null; navItems: NavItem[]; onNavItemClick?: (item: NavItem) => void; @@ -198,6 +200,8 @@ export function LayoutShell({ onWorkspaceDelete, onWorkspaceSettings, onAddWorkspace, + isAtWorkspaceLimit = false, + maxWorkspacesPerUser, workspace, navItems, onNavItemClick, @@ -280,6 +284,8 @@ export function LayoutShell({ activeWorkspaceId={activeWorkspaceId} onWorkspaceSelect={onWorkspaceSelect} onAddWorkspace={onAddWorkspace} + isAtWorkspaceLimit={isAtWorkspaceLimit} + maxWorkspacesPerUser={maxWorkspacesPerUser} workspace={workspace} navItems={navItems} onNavItemClick={onNavItemClick} @@ -352,6 +358,8 @@ export function LayoutShell({ onWorkspaceDelete={onWorkspaceDelete} onWorkspaceSettings={onWorkspaceSettings} onAddWorkspace={onAddWorkspace} + isAtWorkspaceLimit={isAtWorkspaceLimit} + maxWorkspacesPerUser={maxWorkspacesPerUser} isSingleRailMode={false} user={user} onUserSettings={onUserSettings} diff --git a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx index de1b08a6a..c84d39891 100644 --- a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx @@ -17,6 +17,8 @@ interface MobileSidebarProps { activeWorkspaceId: number | null; onWorkspaceSelect: (id: number) => void; onAddWorkspace: () => void; + isAtWorkspaceLimit?: boolean; + maxWorkspacesPerUser?: number; workspace: Workspace | null; navItems: NavItem[]; onNavItemClick?: (item: NavItem) => void; @@ -67,6 +69,8 @@ export function MobileSidebar({ onWorkspaceSelect, onAddWorkspace, + isAtWorkspaceLimit = false, + maxWorkspacesPerUser, workspace, navItems, onNavItemClick, @@ -107,6 +111,10 @@ export function MobileSidebar({ onChatSelect(chat); onOpenChange(false); }; + const addWorkspaceLabel = + isAtWorkspaceLimit && maxWorkspacesPerUser !== undefined + ? `Workspace limit reached: ${maxWorkspacesPerUser}` + : "Add workspace"; return ( @@ -136,10 +144,12 @@ export function MobileSidebar({ variant="ghost" size="icon" onClick={onAddWorkspace} + disabled={isAtWorkspaceLimit} + title={addWorkspaceLabel} className="h-10 w-10 shrink-0 rounded-lg border-2 border-dashed border-muted-foreground/30 hover:border-muted-foreground/50" > - Add workspace + {addWorkspaceLabel}
diff --git a/surfsense_web/contracts/types/workspace.types.ts b/surfsense_web/contracts/types/workspace.types.ts index a777f810a..8a9502131 100644 --- a/surfsense_web/contracts/types/workspace.types.ts +++ b/surfsense_web/contracts/types/workspace.types.ts @@ -29,6 +29,13 @@ export const getWorkspacesRequest = z.object({ export const getWorkspacesResponse = z.array(workspace); +/** + * Workspace limits + */ +export const workspaceLimits = z.object({ + max_workspaces_per_user: z.number(), +}); + /** * Create workspace */ @@ -94,6 +101,7 @@ export const leaveWorkspaceResponse = z.object({ // Inferred types export type Workspace = z.infer; +export type WorkspaceLimits = z.infer; export type GetWorkspacesRequest = z.infer; export type GetWorkspacesResponse = z.infer; export type CreateWorkspaceRequest = z.infer; diff --git a/surfsense_web/lib/apis/workspaces-api.service.ts b/surfsense_web/lib/apis/workspaces-api.service.ts index 8e1494e00..0bd0370a5 100644 --- a/surfsense_web/lib/apis/workspaces-api.service.ts +++ b/surfsense_web/lib/apis/workspaces-api.service.ts @@ -19,11 +19,16 @@ import { updateWorkspaceApiAccessResponse, updateWorkspaceRequest, updateWorkspaceResponse, + workspaceLimits, } from "@/contracts/types/workspace.types"; import { ValidationError } from "../error"; import { baseApiService } from "./base-api.service"; class WorkspacesApiService { + getWorkspaceLimits = async () => { + return baseApiService.get(`/api/v1/workspaces/limits`, workspaceLimits); + }; + /** * Get a list of workspaces with optional filtering and pagination */ diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index 13c141f69..9fc2ef2cb 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -49,6 +49,7 @@ export const cacheKeys = { }, workspaces: { all: ["workspaces"] as const, + limits: ["workspaces", "limits"] as const, withQueryParams: (queries: GetWorkspacesRequest["queryParams"]) => ["workspaces", ...stableEntries(queries)] as const, detail: (workspaceId: string) => ["workspaces", workspaceId] as const, From 1b62352d0da7b74d451b146b328e0d91d05e2f4c Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:55:22 +0530 Subject: [PATCH 075/217] refactor(document-mention-picker): replace MessageSquare icon with MessageCircleMore for improved visual consistency --- .../components/new-chat/document-mention-picker.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/surfsense_web/components/new-chat/document-mention-picker.tsx b/surfsense_web/components/new-chat/document-mention-picker.tsx index dca3b5f7f..17a24f799 100644 --- a/surfsense_web/components/new-chat/document-mention-picker.tsx +++ b/surfsense_web/components/new-chat/document-mention-picker.tsx @@ -8,7 +8,7 @@ import { ChevronRight, Files, Folder as FolderIcon, - MessageSquare, + MessageCircleMore, Unplug, } from "lucide-react"; import { @@ -146,7 +146,7 @@ export function promoteRecentMention(workspaceId: number, mention: MentionedDocu function getMentionIcon(mention: MentionedDocumentInfo) { if (mention.kind === "folder") return ; - if (mention.kind === "thread") return ; + if (mention.kind === "thread") return ; if (mention.kind === "connector") { return getConnectorIcon(mention.connector_type, "size-4") ?? ; } @@ -517,7 +517,7 @@ export const DocumentMentionPicker = forwardRef< id: "chats", label: "Chats", subtitle: "Reference another conversation", - icon: , + icon: , type: "branch", value: { kind: "view", view: { kind: "chats" } }, }); @@ -565,7 +565,7 @@ export const DocumentMentionPicker = forwardRef< id: getMentionDocKey(mention), label: mention.title, subtitle: "Chat", - icon: , + icon: , type: "item" as const, disabled: selectedKeys.has(getMentionDocKey(mention)), value: { kind: "mention" as const, mention }, @@ -625,7 +625,7 @@ export const DocumentMentionPicker = forwardRef< id: getMentionDocKey(mention), label: mention.title, subtitle: "Chat", - icon: , + icon: , type: "item" as const, disabled: selectedKeys.has(getMentionDocKey(mention)), value: { kind: "mention" as const, mention }, From 0ca696f3fca25a1bf021882b57ccf416f4de465f Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:58:18 +0530 Subject: [PATCH 076/217] refactor(tabs): remove legacy tab migration functionality and associated tests for codebase cleanup --- surfsense_web/atoms/tabs/migrate-tabs.test.ts | 21 ------------------- surfsense_web/atoms/tabs/migrate-tabs.ts | 19 ----------------- 2 files changed, 40 deletions(-) delete mode 100644 surfsense_web/atoms/tabs/migrate-tabs.test.ts delete mode 100644 surfsense_web/atoms/tabs/migrate-tabs.ts diff --git a/surfsense_web/atoms/tabs/migrate-tabs.test.ts b/surfsense_web/atoms/tabs/migrate-tabs.test.ts deleted file mode 100644 index e0067a4d2..000000000 --- a/surfsense_web/atoms/tabs/migrate-tabs.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { migrateLegacyTabs } from "./migrate-tabs"; - -// Run with: pnpm exec tsx --test atoms/tabs/migrate-tabs.test.ts -test("maps legacy searchSpaceId to workspaceId on read", () => { - const migrated = migrateLegacyTabs({ - tabs: [{ id: "chat-new", type: "chat", searchSpaceId: 7 } as never], - activeTabId: "chat-new", - }); - const tab = migrated.tabs[0] as { workspaceId?: number }; - assert.equal(tab.workspaceId, 7); -}); - -test("leaves an already-migrated workspaceId untouched", () => { - const migrated = migrateLegacyTabs({ - tabs: [{ id: "d1", type: "document", workspaceId: 3, searchSpaceId: 9 } as never], - }); - const tab = migrated.tabs[0] as { workspaceId?: number }; - assert.equal(tab.workspaceId, 3); -}); diff --git a/surfsense_web/atoms/tabs/migrate-tabs.ts b/surfsense_web/atoms/tabs/migrate-tabs.ts deleted file mode 100644 index e95a921de..000000000 --- a/surfsense_web/atoms/tabs/migrate-tabs.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** - * One-time read-migration for persisted tabs: legacy state stored the workspace - * as `searchSpaceId`. Map it to `workspaceId` on read so already-open tabs keep - * their workspace association after the rename. Pure + dependency-free so it can - * be unit-checked without loading the atom module. - */ -export function migrateLegacyTabs }>( - state: T -): T { - return { - ...state, - tabs: state.tabs.map((t) => { - const legacy = t as { workspaceId?: number; searchSpaceId?: number }; - return legacy.workspaceId === undefined && legacy.searchSpaceId !== undefined - ? { ...t, workspaceId: legacy.searchSpaceId } - : t; - }), - }; -} From 2c4ebaeb7f414e7fa1f5b506ba6f775952fe7afc Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:09:50 +0530 Subject: [PATCH 077/217] feat(zero): publish thread title, visibility, created_by columns --- .../175_publish_thread_metadata_to_zero.py | 23 +++++++++++++++++++ surfsense_backend/app/zero_publication.py | 3 +++ 2 files changed, 26 insertions(+) create mode 100644 surfsense_backend/alembic/versions/175_publish_thread_metadata_to_zero.py diff --git a/surfsense_backend/alembic/versions/175_publish_thread_metadata_to_zero.py b/surfsense_backend/alembic/versions/175_publish_thread_metadata_to_zero.py new file mode 100644 index 000000000..d4d298a4a --- /dev/null +++ b/surfsense_backend/alembic/versions/175_publish_thread_metadata_to_zero.py @@ -0,0 +1,23 @@ +"""publish thread metadata to zero_publication + +Revision ID: 175 +Revises: 174 +""" + +from collections.abc import Sequence + +from alembic import op +from app.zero_publication import apply_publication + +revision: str = "175" +down_revision: str | None = "174" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + apply_publication(op.get_bind()) + + +def downgrade() -> None: + """No-op. Historical publication shapes are immutable.""" diff --git a/surfsense_backend/app/zero_publication.py b/surfsense_backend/app/zero_publication.py index c44a29dcd..f6dff6cdf 100644 --- a/surfsense_backend/app/zero_publication.py +++ b/surfsense_backend/app/zero_publication.py @@ -60,6 +60,9 @@ AUTOMATION_COLS = [ NEW_CHAT_THREAD_COLS = [ "id", "workspace_id", + "title", + "visibility", + "created_by_id", ] # Enough to drive the lifecycle UI by push: status, the reviewable brief, and From c0492be3ef8d8c8695a6adc69e3c7a04a9a18532 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:09:56 +0530 Subject: [PATCH 078/217] feat(zero): mirror thread title/visibility/createdById in schema --- surfsense_web/zero/schema/chat.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surfsense_web/zero/schema/chat.ts b/surfsense_web/zero/schema/chat.ts index d42a11848..81673feaa 100644 --- a/surfsense_web/zero/schema/chat.ts +++ b/surfsense_web/zero/schema/chat.ts @@ -24,6 +24,9 @@ export const newChatThreadTable = table("new_chat_threads") .columns({ id: number(), workspaceId: number().from("workspace_id"), + title: string(), + visibility: string(), + createdById: string().optional().from("created_by_id"), }) .primaryKey("id"); From c49a3d765d78572a1fe55e80545c0e90fca2a8ed Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:10:31 +0530 Subject: [PATCH 079/217] feat(zero): add threads.byIds query with per-thread authz --- surfsense_web/zero/queries/chat.ts | 10 ++++++++++ surfsense_web/zero/queries/index.ts | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/surfsense_web/zero/queries/chat.ts b/surfsense_web/zero/queries/chat.ts index 40e09a6ee..06969a4e9 100644 --- a/surfsense_web/zero/queries/chat.ts +++ b/surfsense_web/zero/queries/chat.ts @@ -29,3 +29,13 @@ export const chatSessionQueries = { .one() ), }; + +export const threadQueries = { + byIds: defineQuery(z.object({ ids: z.array(z.number()) }), ({ args: { ids }, ctx }) => + constrainToAllowedSpaces(zql.new_chat_threads, ctx) + .where("id", "IN", ids) + .where(({ or, cmp }) => + or(cmp("createdById", ctx?.userId ?? ""), cmp("visibility", "SEARCH_SPACE")) + ) + ), +}; diff --git a/surfsense_web/zero/queries/index.ts b/surfsense_web/zero/queries/index.ts index 45df8fa98..2e51bf9a8 100644 --- a/surfsense_web/zero/queries/index.ts +++ b/surfsense_web/zero/queries/index.ts @@ -1,6 +1,6 @@ import { defineQueries } from "@rocicorp/zero"; import { automationRunQueries } from "./automations"; -import { chatSessionQueries, commentQueries, messageQueries } from "./chat"; +import { chatSessionQueries, commentQueries, messageQueries, threadQueries } from "./chat"; import { connectorQueries, documentQueries } from "./documents"; import { folderQueries } from "./folders"; import { notificationQueries } from "./inbox"; @@ -15,6 +15,7 @@ export const queries = defineQueries({ messages: messageQueries, comments: commentQueries, chatSession: chatSessionQueries, + threads: threadQueries, user: userQueries, automationRuns: automationRunQueries, podcasts: podcastQueries, From 88b9632911ee7a85c2b91bd938854f190eaeed21 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:10:39 +0530 Subject: [PATCH 080/217] feat(zero): add documents.byIds query for document tabs --- surfsense_web/zero/queries/documents.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surfsense_web/zero/queries/documents.ts b/surfsense_web/zero/queries/documents.ts index d2f483989..6c110aee2 100644 --- a/surfsense_web/zero/queries/documents.ts +++ b/surfsense_web/zero/queries/documents.ts @@ -9,6 +9,9 @@ export const documentQueries = { if (!canReadSpace(ctx, workspaceId)) return denySpace(query).orderBy("createdAt", "desc"); return constrainToAllowedSpaces(query, ctx).orderBy("createdAt", "desc"); }), + byIds: defineQuery(z.object({ ids: z.array(z.number()) }), ({ args: { ids }, ctx }) => + constrainToAllowedSpaces(zql.documents, ctx).where("id", "IN", ids) + ), }; export const connectorQueries = { From a8fb161bef97f6e5fc658ebb7a20308abd7d0c20 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:10:47 +0530 Subject: [PATCH 081/217] refactor(tabs): slim Tab to pointer-only state on v2 storage key --- surfsense_web/atoms/tabs/tabs.atom.ts | 137 ++++++++++++-------------- 1 file changed, 61 insertions(+), 76 deletions(-) diff --git a/surfsense_web/atoms/tabs/tabs.atom.ts b/surfsense_web/atoms/tabs/tabs.atom.ts index 45e0c098c..fbc546c42 100644 --- a/surfsense_web/atoms/tabs/tabs.atom.ts +++ b/surfsense_web/atoms/tabs/tabs.atom.ts @@ -1,22 +1,13 @@ import { atom } from "jotai"; import { atomWithStorage, createJSONStorage } from "jotai/utils"; -import type { ChatVisibility } from "@/lib/chat/thread-persistence"; -import { migrateLegacyTabs } from "./migrate-tabs"; export type TabType = "chat" | "document"; export interface Tab { id: string; type: TabType; - title: string; - /** For chat tabs */ - chatId?: number | null; - chatUrl?: string; - visibility?: ChatVisibility; - hasComments?: boolean; - /** For document tabs */ - documentId?: number; - workspaceId?: number; + entityId: number | null; + workspaceId: number; } interface TabsState { @@ -27,9 +18,8 @@ interface TabsState { const INITIAL_CHAT_TAB: Tab = { id: "chat-new", type: "chat", - title: "New Chat", - chatId: null, - chatUrl: undefined, + entityId: null, + workspaceId: 0, }; const initialState: TabsState = { @@ -37,23 +27,14 @@ const initialState: TabsState = { activeTabId: "chat-new", }; -// Prevent race conditions where route-sync recreates a just-deleted chat tab. -const deletedChatIdsAtom = atom>(new Set()); - // Persist tabs in localStorage so they survive a hard refresh and let the user // keep tabs open across multiple workspaces (browser-like behavior). const localStorageAdapter = createJSONStorage( () => (typeof window !== "undefined" ? localStorage : undefined) as Storage ); -// Wrap getItem in place so the adapter keeps its original (sync) type while -// migrating legacy persisted state on read. -const baseGetItem = localStorageAdapter.getItem.bind(localStorageAdapter); -localStorageAdapter.getItem = (key, initialValue) => - migrateLegacyTabs(baseGetItem(key, initialValue)); - export const tabsStateAtom = atomWithStorage( - "surfsense:tabs", + "surfsense:tabs:v2", initialState, localStorageAdapter, { getOnInit: true } @@ -66,11 +47,11 @@ export const activeTabAtom = atom((get) => { return state.tabs.find((t) => t.id === state.activeTabId) ?? null; }); -function makeChatTabId(chatId: number | null): string { +export function makeChatTabId(chatId: number | null): string { return chatId ? `chat-${chatId}` : "chat-new"; } -function makeDocumentTabId(documentId: number): string { +export function makeDocumentTabId(documentId: number): string { return `doc-${documentId}`; } @@ -86,24 +67,12 @@ export const syncChatTabAtom = atom( set, { chatId, - title, - chatUrl, workspaceId, - visibility, - hasComments, }: { chatId: number | null; - title?: string; - chatUrl?: string; workspaceId: number; - visibility?: ChatVisibility; - hasComments?: boolean; } ) => { - if (chatId && get(deletedChatIdsAtom).has(chatId)) { - return; - } - const state = get(tabsStateAtom); const tabId = makeChatTabId(chatId); const existing = state.tabs.find((t) => t.id === tabId); @@ -116,11 +85,8 @@ export const syncChatTabAtom = atom( t.id === tabId ? { ...t, - title: title || t.title, - chatUrl: chatUrl || t.chatUrl, - workspaceId: workspaceId ?? t.workspaceId, - ...(visibility !== undefined ? { visibility } : {}), - ...(hasComments !== undefined ? { hasComments } : {}), + entityId: chatId, + workspaceId, } : t ), @@ -136,11 +102,13 @@ export const syncChatTabAtom = atom( set(tabsStateAtom, { ...state, activeTabId: "chat-new", - tabs: state.tabs.map((t) => (t.id === "chat-new" ? { ...t, workspaceId, chatUrl } : t)), + tabs: state.tabs.map((t) => + t.id === "chat-new" ? { ...t, entityId: null, workspaceId } : t + ), }); } else { set(tabsStateAtom, { - tabs: [...state.tabs, { ...INITIAL_CHAT_TAB, workspaceId, chatUrl }], + tabs: [...state.tabs, { ...INITIAL_CHAT_TAB, workspaceId }], activeTabId: "chat-new", }); } @@ -152,12 +120,8 @@ export const syncChatTabAtom = atom( const newTab: Tab = { id: tabId, type: "chat", - title: title || "New Chat", - chatId, - chatUrl, + entityId: chatId, workspaceId, - ...(visibility !== undefined ? { visibility } : {}), - ...(hasComments !== undefined ? { hasComments } : {}), }; let updatedTabs: Tab[]; @@ -172,29 +136,26 @@ export const syncChatTabAtom = atom( } ); -/** Update the title of the current chat tab (e.g., when a chat gets its first response). */ +/** Promote the lazy "new chat" tab once the server creates its thread. */ export const updateChatTabTitleAtom = atom( null, - (get, set, { chatId, title }: { chatId: number; title: string }) => { + (get, set, { chatId }: { chatId: number; title?: string }) => { const state = get(tabsStateAtom); const tabId = makeChatTabId(chatId); const hasExactTab = state.tabs.some((t) => t.id === tabId); - // During lazy thread creation, title updates can arrive before "chat-new" - // is swapped to chat-{id}. In that case, promote the active "chat-new" tab. + // During lazy thread creation, title updates can arrive before "chat-new" is + // swapped to chat-{id}. In that case, promote the pointer only; title comes + // from Zero. if (!hasExactTab && state.activeTabId === "chat-new") { set(tabsStateAtom, { ...state, activeTabId: tabId, - tabs: state.tabs.map((t) => (t.id === "chat-new" ? { ...t, id: tabId, chatId, title } : t)), + tabs: state.tabs.map((t) => + t.id === "chat-new" ? { ...t, id: tabId, entityId: chatId } : t + ), }); - return; } - - set(tabsStateAtom, { - ...state, - tabs: state.tabs.map((t) => (t.id === tabId ? { ...t, title } : t)), - }); } ); @@ -204,7 +165,7 @@ export const openDocumentTabAtom = atom( ( get, set, - { documentId, workspaceId, title }: { documentId: number; workspaceId: number; title?: string } + { documentId, workspaceId }: { documentId: number; workspaceId: number; title?: string } ) => { const state = get(tabsStateAtom); const tabId = makeDocumentTabId(documentId); @@ -214,7 +175,7 @@ export const openDocumentTabAtom = atom( set(tabsStateAtom, { ...state, activeTabId: tabId, - tabs: state.tabs.map((t) => (t.id === tabId ? { ...t, title: title || t.title } : t)), + tabs: state.tabs.map((t) => (t.id === tabId ? { ...t, workspaceId } : t)), }); return; } @@ -222,8 +183,7 @@ export const openDocumentTabAtom = atom( const newTab: Tab = { id: tabId, type: "document", - title: title || `Document ${documentId}`, - documentId, + entityId: documentId, workspaceId, }; @@ -254,11 +214,12 @@ export const closeTabAtom = atom(null, (get, set, tabId: string) => { // Don't close the last tab — always keep at least one if (remaining.length === 0) { + const closedTab = state.tabs[idx]; set(tabsStateAtom, { - tabs: [INITIAL_CHAT_TAB], + tabs: [{ ...INITIAL_CHAT_TAB, workspaceId: closedTab.workspaceId }], activeTabId: "chat-new", }); - return INITIAL_CHAT_TAB; + return { ...INITIAL_CHAT_TAB, workspaceId: closedTab.workspaceId }; } let newActiveId = state.activeTabId; @@ -279,18 +240,16 @@ export const removeChatTabAtom = atom(null, (get, set, chatId: number) => { const idx = state.tabs.findIndex((t) => t.id === tabId); if (idx === -1) return null; - const deletedChatIds = get(deletedChatIdsAtom); - set(deletedChatIdsAtom, new Set([...deletedChatIds, chatId])); - const remaining = state.tabs.filter((t) => t.id !== tabId); // Always keep at least one tab available. if (remaining.length === 0) { + const removedTab = state.tabs[idx]; set(tabsStateAtom, { - tabs: [INITIAL_CHAT_TAB], + tabs: [{ ...INITIAL_CHAT_TAB, workspaceId: removedTab.workspaceId }], activeTabId: "chat-new", }); - return INITIAL_CHAT_TAB; + return { ...INITIAL_CHAT_TAB, workspaceId: removedTab.workspaceId }; } let newActiveId = state.activeTabId; @@ -303,8 +262,34 @@ export const removeChatTabAtom = atom(null, (get, set, chatId: number) => { return remaining.find((t) => t.id === newActiveId) ?? null; }); -/** Reset tabs when switching workspaces. */ -export const resetTabsAtom = atom(null, (_get, set) => { - set(tabsStateAtom, { ...initialState }); - set(deletedChatIdsAtom, new Set()); +/** Remove unresolved chat pointers after Zero confirms the queried rows are complete. */ +export const pruneMissingChatTabsAtom = atom(null, (get, set, missingChatIds: Set) => { + if (missingChatIds.size === 0) return; + + const state = get(tabsStateAtom); + const firstMissingIdx = state.tabs.findIndex( + (t) => t.type === "chat" && t.entityId !== null && missingChatIds.has(t.entityId) + ); + if (firstMissingIdx === -1) return; + + const remaining = state.tabs.filter( + (t) => !(t.type === "chat" && t.entityId !== null && missingChatIds.has(t.entityId)) + ); + + if (remaining.length === 0) { + set(tabsStateAtom, { + tabs: [INITIAL_CHAT_TAB], + activeTabId: "chat-new", + }); + return; + } + + const activeWasPruned = state.tabs.some( + (t) => t.id === state.activeTabId && t.type === "chat" && t.entityId !== null && missingChatIds.has(t.entityId) + ); + const newActiveId = activeWasPruned + ? remaining[Math.min(firstMissingIdx, remaining.length - 1)].id + : state.activeTabId; + + set(tabsStateAtom, { tabs: remaining, activeTabId: newActiveId }); }); From 873ec0bcd789641ce787f0051695030e289ab827 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:10:53 +0530 Subject: [PATCH 082/217] feat(tabs): add useResolvedTabs join hook resolving titles from zero --- surfsense_web/hooks/use-resolved-tabs.test.ts | 53 +++++++ surfsense_web/hooks/use-resolved-tabs.ts | 129 ++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 surfsense_web/hooks/use-resolved-tabs.test.ts create mode 100644 surfsense_web/hooks/use-resolved-tabs.ts diff --git a/surfsense_web/hooks/use-resolved-tabs.test.ts b/surfsense_web/hooks/use-resolved-tabs.test.ts new file mode 100644 index 000000000..34a51df86 --- /dev/null +++ b/surfsense_web/hooks/use-resolved-tabs.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + getMissingCompleteChatIds, + resolveTabPointers, + type ResolvedTab, +} from "./use-resolved-tabs"; + +// Run with: pnpm exec tsx --test hooks/use-resolved-tabs.test.ts +test("does not prune unresolved chat tabs before Zero completes", () => { + const missing = getMissingCompleteChatIds({ + tabs: [{ id: "chat-42", type: "chat", entityId: 42, workspaceId: 7 }], + threadRows: [], + resultType: "unknown", + }); + + assert.equal(missing.size, 0); +}); + +test("prunes unresolved chat tabs only after Zero completes", () => { + const missing = getMissingCompleteChatIds({ + tabs: [ + { id: "chat-42", type: "chat", entityId: 42, workspaceId: 7 }, + { id: "chat-43", type: "chat", entityId: 43, workspaceId: 7 }, + ], + threadRows: [{ id: 43, title: "Present", visibility: "PRIVATE" }], + resultType: "complete", + }); + + assert.deepEqual([...missing], [42]); +}); + +test("merges pointer tabs with synced row titles", () => { + const resolved = resolveTabPointers({ + tabs: [ + { id: "chat-42", type: "chat", entityId: 42, workspaceId: 7 }, + { id: "doc-9", type: "document", entityId: 9, workspaceId: 7 }, + ], + threadRows: [{ id: 42, title: "Live chat title", visibility: "SEARCH_SPACE" }], + documentRows: [{ id: 9, title: "Live document title" }], + }); + + assert.deepEqual( + resolved.map((tab): Pick => ({ + id: tab.id, + title: tab.title, + })), + [ + { id: "chat-42", title: "Live chat title" }, + { id: "doc-9", title: "Live document title" }, + ] + ); +}); diff --git a/surfsense_web/hooks/use-resolved-tabs.ts b/surfsense_web/hooks/use-resolved-tabs.ts new file mode 100644 index 000000000..6b321cbd3 --- /dev/null +++ b/surfsense_web/hooks/use-resolved-tabs.ts @@ -0,0 +1,129 @@ +"use client"; + +import { useQuery } from "@rocicorp/zero/react"; +import { useAtomValue, useSetAtom } from "jotai"; +import { useEffect, useMemo } from "react"; +import { pruneMissingChatTabsAtom, type Tab, tabsAtom } from "@/atoms/tabs/tabs.atom"; +import type { ChatVisibility } from "@/lib/chat/thread-persistence"; +import { queries } from "@/zero/queries"; + +interface ThreadRow { + id: number; + title: string; + visibility: string; +} + +interface DocumentRow { + id: number; + title: string; +} + +export interface ResolvedTab extends Tab { + title: string; + chatUrl?: string; + visibility?: ChatVisibility; +} + +function uniqueEntityIds(tabs: Tab[], type: Tab["type"]): number[] { + const ids = new Set(); + for (const tab of tabs) { + if (tab.type === type && tab.entityId !== null) { + ids.add(tab.entityId); + } + } + return [...ids]; +} + +function rowById(rows: readonly T[] | undefined): Map { + return new Map((rows ?? []).map((row) => [row.id, row])); +} + +export function getChatUrl(workspaceId: number, threadId: number | null): string { + return threadId + ? `/dashboard/${workspaceId}/new-chat/${threadId}` + : `/dashboard/${workspaceId}/new-chat`; +} + +export function resolveTabPointers({ + tabs, + threadRows, + documentRows, +}: { + tabs: Tab[]; + threadRows?: readonly ThreadRow[]; + documentRows?: readonly DocumentRow[]; +}): ResolvedTab[] { + const threads = rowById(threadRows); + const documents = rowById(documentRows); + + return tabs.map((tab) => { + if (tab.type === "document") { + const title = + tab.entityId === null ? "Document" : (documents.get(tab.entityId)?.title ?? `Document ${tab.entityId}`); + return { ...tab, title }; + } + + const row = tab.entityId === null ? undefined : threads.get(tab.entityId); + return { + ...tab, + title: row?.title || (tab.entityId === null ? "New Chat" : `Chat ${tab.entityId}`), + chatUrl: getChatUrl(tab.workspaceId, tab.entityId), + ...(row?.visibility !== undefined ? { visibility: row.visibility as ChatVisibility } : {}), + }; + }); +} + +export function getMissingCompleteChatIds({ + tabs, + threadRows, + resultType, +}: { + tabs: Tab[]; + threadRows?: readonly ThreadRow[]; + resultType: string; +}): Set { + if (resultType !== "complete") return new Set(); + + const threadIds = new Set((threadRows ?? []).map((row) => row.id)); + return new Set( + tabs + .filter( + (tab): tab is Tab & { type: "chat"; entityId: number } => + tab.type === "chat" && tab.entityId !== null && !threadIds.has(tab.entityId) + ) + .map((tab) => tab.entityId) + ); +} + +export function useResolvedTabs(): ResolvedTab[] { + const tabs = useAtomValue(tabsAtom); + const pruneMissingChatTabs = useSetAtom(pruneMissingChatTabsAtom); + + const chatIds = useMemo(() => uniqueEntityIds(tabs, "chat"), [tabs]); + const documentIds = useMemo(() => uniqueEntityIds(tabs, "document"), [tabs]); + const [threadRows, threadResult] = useQuery( + queries.threads.byIds({ ids: chatIds.length > 0 ? chatIds : [-1] }) + ); + const [documentRows] = useQuery( + queries.documents.byIds({ ids: documentIds.length > 0 ? documentIds : [-1] }) + ); + + const missingChatIds = useMemo( + () => + getMissingCompleteChatIds({ + tabs, + threadRows, + resultType: threadResult.type, + }), + [tabs, threadRows, threadResult.type] + ); + + useEffect(() => { + pruneMissingChatTabs(missingChatIds); + }, [missingChatIds, pruneMissingChatTabs]); + + return useMemo( + () => resolveTabPointers({ tabs, threadRows, documentRows }), + [tabs, threadRows, documentRows] + ); +} From e3884399201024200b077396b8f5d29cfd0629ec Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:11:05 +0530 Subject: [PATCH 083/217] refactor(tabs): render TabBar from resolved tabs --- .../components/layout/ui/tabs/TabBar.tsx | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/surfsense_web/components/layout/ui/tabs/TabBar.tsx b/surfsense_web/components/layout/ui/tabs/TabBar.tsx index 869c9cee2..fc92643b8 100644 --- a/surfsense_web/components/layout/ui/tabs/TabBar.tsx +++ b/surfsense_web/components/layout/ui/tabs/TabBar.tsx @@ -3,19 +3,14 @@ import { useAtomValue, useSetAtom } from "jotai"; import { Plus, X } from "lucide-react"; import { Fragment, useCallback, useEffect, useRef, useState } from "react"; -import { - activeTabIdAtom, - closeTabAtom, - switchTabAtom, - type Tab, - tabsAtom, -} from "@/atoms/tabs/tabs.atom"; +import { activeTabIdAtom, closeTabAtom, switchTabAtom } from "@/atoms/tabs/tabs.atom"; import { Button } from "@/components/ui/button"; +import { getChatUrl, type ResolvedTab, useResolvedTabs } from "@/hooks/use-resolved-tabs"; import { cn } from "@/lib/utils"; interface TabBarProps { - onTabSwitch?: (tab: Tab) => void; - onTabPrefetch?: (tab: Tab) => void; + onTabSwitch?: (tab: ResolvedTab) => void; + onTabPrefetch?: (tab: ResolvedTab) => void; onNewChat?: () => void; leftActions?: React.ReactNode; rightActions?: React.ReactNode; @@ -43,7 +38,7 @@ export function TabBar({ rightActions, className, }: TabBarProps) { - const tabs = useAtomValue(tabsAtom); + const tabs = useResolvedTabs(); const activeTabId = useAtomValue(activeTabIdAtom); const switchTab = useSetAtom(switchTabAtom); const closeTab = useSetAtom(closeTabAtom); @@ -65,7 +60,7 @@ export function TabBar({ ); const handleTabClick = useCallback( - (tab: Tab) => { + (tab: ResolvedTab) => { if (tab.id === activeTabId) return; switchTab(tab.id); onTabSwitch?.(tab); @@ -74,7 +69,7 @@ export function TabBar({ ); const handleTabPrefetch = useCallback( - (tab: Tab) => { + (tab: ResolvedTab) => { if (tab.type === "chat") { onTabPrefetch?.(tab); } @@ -87,10 +82,21 @@ export function TabBar({ e.stopPropagation(); const fallback = closeTab(tabId); if (fallback) { - onTabSwitch?.(fallback); + const resolvedFallback = + tabs.find((tab) => tab.id === fallback.id) ?? + (fallback.type === "chat" + ? { + ...fallback, + title: "New Chat", + chatUrl: getChatUrl(fallback.workspaceId, fallback.entityId), + } + : undefined); + if (resolvedFallback) { + onTabSwitch?.(resolvedFallback); + } } }, - [closeTab, onTabSwitch] + [closeTab, onTabSwitch, tabs] ); // React to tab list growth via a MutationObserver so the scroll catches the From fa21566cdfa47619540026d94767b6f9c788de86 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:11:18 +0530 Subject: [PATCH 084/217] refactor(tabs): resolve document tab via entityId and live title --- .../layout/ui/shell/LayoutShell.tsx | 88 +++++++++++++++++-- 1 file changed, 79 insertions(+), 9 deletions(-) diff --git a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx index 70bb317dd..42ef141b2 100644 --- a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx +++ b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx @@ -3,10 +3,11 @@ import { useAtomValue } from "jotai"; import dynamic from "next/dynamic"; import { useMemo, useState } from "react"; -import { activeTabAtom, type Tab } from "@/atoms/tabs/tabs.atom"; +import { activeTabIdAtom } from "@/atoms/tabs/tabs.atom"; import { Logo } from "@/components/Logo"; import { Spinner } from "@/components/ui/spinner"; import { TooltipProvider } from "@/components/ui/tooltip"; +import { type ResolvedTab, useResolvedTabs } from "@/hooks/use-resolved-tabs"; import { useIsMobile } from "@/hooks/use-mobile"; import { useElectronAPI } from "@/hooks/use-platform"; import { cn } from "@/lib/utils"; @@ -123,6 +124,7 @@ interface LayoutShellProps { defaultCollapsed?: boolean; isChatPage?: boolean; isAllChatsPage?: boolean; + showTabs?: boolean; useWorkspacePanel?: boolean; workspacePanelViewportClassName?: string; workspacePanelContentClassName?: string; @@ -130,8 +132,8 @@ interface LayoutShellProps { className?: string; notifications?: NotificationsDropdownData; isLoadingChats?: boolean; - onTabSwitch?: (tab: Tab) => void; - onTabPrefetch?: (tab: Tab) => void; + onTabSwitch?: (tab: ResolvedTab) => void; + onTabPrefetch?: (tab: ResolvedTab) => void; playgroundSidebar?: React.ReactNode; initialPlaygroundSidebarCollapsed?: boolean; } @@ -141,19 +143,85 @@ function MainContentPanel({ onTabSwitch, onTabPrefetch, onNewChat, + showTabs = true, showRightPanelExpandButton = true, showTopBorder = false, children, }: { isChatPage: boolean; - onTabSwitch?: (tab: Tab) => void; - onTabPrefetch?: (tab: Tab) => void; + onTabSwitch?: (tab: ResolvedTab) => void; + onTabPrefetch?: (tab: ResolvedTab) => void; onNewChat?: () => void; + showTabs?: boolean; showRightPanelExpandButton?: boolean; showTopBorder?: boolean; children: React.ReactNode; }) { - const activeTab = useAtomValue(activeTabAtom); + if (!showTabs) { + return ( + + {children} + + ); + } + + return ( + + {children} + + ); +} + +function UntabbedMainContentPanel({ + isChatPage, + showTopBorder, + children, +}: { + isChatPage: boolean; + showTopBorder: boolean; + children: React.ReactNode; +}) { + return ( +
+
+
+
+ {children} +
+
+
+ ); +} + +function TabbedMainContentPanel({ + isChatPage, + onTabSwitch, + onTabPrefetch, + onNewChat, + showRightPanelExpandButton, + showTopBorder, + children, +}: { + isChatPage: boolean; + onTabSwitch?: (tab: ResolvedTab) => void; + onTabPrefetch?: (tab: ResolvedTab) => void; + onNewChat?: () => void; + showRightPanelExpandButton: boolean; + showTopBorder: boolean; + children: React.ReactNode; +}) { + const activeTabId = useAtomValue(activeTabIdAtom); + const tabs = useResolvedTabs(); + const activeTab = tabs.find((tab) => tab.id === activeTabId) ?? null; const isDocumentTab = activeTab?.type === "document"; return ( @@ -170,11 +238,11 @@ function MainContentPanel({
- {isDocumentTab && activeTab.documentId && activeTab.workspaceId ? ( + {isDocumentTab && activeTab.entityId && activeTab.workspaceId ? (
@@ -228,6 +296,7 @@ export function LayoutShell({ defaultCollapsed = false, isChatPage = false, isAllChatsPage = false, + showTabs = true, useWorkspacePanel = false, workspacePanelViewportClassName, workspacePanelContentClassName, @@ -476,6 +545,7 @@ export function LayoutShell({ onTabSwitch={onTabSwitch} onTabPrefetch={onTabPrefetch} onNewChat={onNewChat} + showTabs={showTabs} showRightPanelExpandButton={!isMacDesktop} showTopBorder={isMacDesktop} > From 1465b907db48d12a3ee79d1b74a1bcd975a9fb03 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:11:23 +0530 Subject: [PATCH 085/217] refactor(tabs): rework fallback navigation for pointer tabs --- .../providers/FreeLayoutDataProvider.tsx | 1 + .../layout/providers/LayoutDataProvider.tsx | 41 +++++++------------ .../layout/ui/sidebar/AllChatsSidebar.tsx | 21 +++------- 3 files changed, 21 insertions(+), 42 deletions(-) diff --git a/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx b/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx index 059872bac..85b5227fa 100644 --- a/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx @@ -85,6 +85,7 @@ export function FreeLayoutDataProvider({ children }: FreeLayoutDataProviderProps onLogout={() => router.push("/register")} pageUsage={pageUsage} isChatPage + showTabs={false} isLoadingChats={false} > {children} diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index a61a1a703..6858794ca 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -11,7 +11,7 @@ import { toast } from "sonner"; import { currentThreadAtom, resetCurrentThreadAtom } from "@/atoms/chat/current-thread.atom"; import { statusInboxItemsAtom } from "@/atoms/inbox/status-inbox.atom"; import { announcementsDialogAtom } from "@/atoms/layout/dialogs.atom"; -import { removeChatTabAtom, syncChatTabAtom, type Tab } from "@/atoms/tabs/tabs.atom"; +import { removeChatTabAtom, syncChatTabAtom } from "@/atoms/tabs/tabs.atom"; import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { deleteWorkspaceMutationAtom } from "@/atoms/workspaces/workspace-mutation.atoms"; import { workspaceLimitsAtom, workspacesAtom } from "@/atoms/workspaces/workspace-query.atoms"; @@ -43,6 +43,7 @@ import { Spinner } from "@/components/ui/spinner"; import { useActivateChatThread } from "@/hooks/use-activate-chat-thread"; import { useAnnouncements } from "@/hooks/use-announcements"; import { useInbox } from "@/hooks/use-inbox"; +import { getChatUrl, type ResolvedTab } from "@/hooks/use-resolved-tabs"; import { useArchiveThread, useDeleteThread, useRenameThread } from "@/hooks/use-thread-mutations"; import { notificationsApiService } from "@/lib/apis/notifications-api.service"; import { workspacesApiService } from "@/lib/apis/workspaces-api.service"; @@ -273,19 +274,11 @@ export function LayoutDataProvider({ // Sync current chat route with tab state useEffect(() => { const chatId = currentChatId ?? null; - const chatUrl = chatId - ? `/dashboard/${workspaceId}/new-chat/${chatId}` - : `/dashboard/${workspaceId}/new-chat`; - const thread = threadsData?.threads?.find((t) => t.id === chatId); syncChatTab({ chatId, - // Avoid overwriting live SSE-updated tab titles with fallback values. - title: chatId ? (thread?.title ?? undefined) : "New Chat", - chatUrl, workspaceId: Number(workspaceId), - ...(thread?.visibility !== undefined ? { visibility: thread.visibility } : {}), }); - }, [currentChatId, workspaceId, threadsData?.threads, syncChatTab]); + }, [currentChatId, workspaceId, syncChatTab]); const chats = useMemo(() => { if (!threadsData?.threads) return []; @@ -440,26 +433,24 @@ export function LayoutDataProvider({ }, [workspaceToLeave, refetchWorkspaces, workspaceId, router, t]); const handleTabSwitch = useCallback( - (tab: Tab) => { + (tab: ResolvedTab) => { if (tab.type === "chat") { activateChatThread({ - id: tab.chatId ?? null, - title: tab.title, + id: tab.entityId, url: tab.chatUrl, - workspaceId: tab.workspaceId ?? workspaceId, + workspaceId: tab.workspaceId, ...(tab.visibility !== undefined ? { visibility: tab.visibility } : {}), - ...(tab.hasComments !== undefined ? { hasComments: tab.hasComments } : {}), }); } // Document tabs are handled in-place by LayoutShell — no navigation needed }, - [activateChatThread, workspaceId] + [activateChatThread] ); const handleTabPrefetch = useCallback( - (tab: Tab) => { + (tab: ResolvedTab) => { if (tab.type === "chat") { - prefetchChatThread(tab.chatId); + prefetchChatThread(tab.entityId); } }, [prefetchChatThread] @@ -573,16 +564,12 @@ export function LayoutDataProvider({ const fallbackTab = removeChatTab(chatToDelete.id); if (currentChatId === chatToDelete.id) { resetCurrentThread(); - if (fallbackTab?.type === "chat" && fallbackTab.chatUrl) { + if (fallbackTab?.type === "chat") { + const fallbackWorkspaceId = fallbackTab.workspaceId || Number(workspaceId); activateChatThread({ - id: fallbackTab.chatId ?? null, - title: fallbackTab.title, - url: fallbackTab.chatUrl, - workspaceId: fallbackTab.workspaceId ?? workspaceId, - ...(fallbackTab.visibility !== undefined ? { visibility: fallbackTab.visibility } : {}), - ...(fallbackTab.hasComments !== undefined - ? { hasComments: fallbackTab.hasComments } - : {}), + id: fallbackTab.entityId, + url: getChatUrl(fallbackWorkspaceId, fallbackTab.entityId), + workspaceId: fallbackWorkspaceId, }); } else { const isOutOfSync = currentThreadState.id !== null && !params?.chat_id; diff --git a/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx index e7cdb7d5e..bc861f443 100644 --- a/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/AllChatsSidebar.tsx @@ -41,6 +41,7 @@ import { useActivateChatThread } from "@/hooks/use-activate-chat-thread"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useLongPress } from "@/hooks/use-long-press"; import { useIsMobile } from "@/hooks/use-mobile"; +import { getChatUrl } from "@/hooks/use-resolved-tabs"; import { useArchiveThread, useDeleteThread, useRenameThread } from "@/hooks/use-thread-mutations"; import { fetchThreads, searchThreads, type ThreadListItem } from "@/lib/chat/thread-persistence"; import { formatRelativeDate } from "@/lib/format-date"; @@ -145,22 +146,12 @@ function AllChatsContent({ workspaceId, className }: AllChatsContentProps) { if (currentChatId === threadId) { setTimeout(() => { - if ( - fallbackTab?.type === "chat" && - fallbackTab.chatUrl && - fallbackTab.chatId !== undefined - ) { + if (fallbackTab?.type === "chat") { + const fallbackWorkspaceId = fallbackTab.workspaceId || Number(workspaceId); activateChatThread({ - id: fallbackTab.chatId ?? null, - title: fallbackTab.title, - url: fallbackTab.chatUrl, - workspaceId: fallbackTab.workspaceId ?? workspaceId, - ...(fallbackTab.visibility !== undefined - ? { visibility: fallbackTab.visibility } - : {}), - ...(fallbackTab.hasComments !== undefined - ? { hasComments: fallbackTab.hasComments } - : {}), + id: fallbackTab.entityId, + url: getChatUrl(fallbackWorkspaceId, fallbackTab.entityId), + workspaceId: fallbackWorkspaceId, }); return; } From 9f1f727d8906adbac3949671a07b0bd6d81ba171 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:11:29 +0530 Subject: [PATCH 086/217] refactor(tabs): drop snapshot args from tab sync --- .../[workspace_id]/new-chat/[[...chat_id]]/page.tsx | 4 ---- surfsense_web/hooks/use-activate-chat-thread.ts | 6 +----- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx index 22298e574..dff264c43 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/new-chat/[[...chat_id]]/page.tsx @@ -299,11 +299,7 @@ export default function NewChatPage() { setCurrentThread(thread); syncChatTab({ chatId: thread.id, - title: thread.title, - chatUrl: `/dashboard/${thread.workspace_id ?? workspaceId}/new-chat/${thread.id}`, workspaceId: thread.workspace_id ?? workspaceId, - visibility: thread.visibility, - hasComments: thread.has_comments ?? false, }); } }, [activeThreadId, workspaceId, syncChatTab, threadDetailQuery.data]); diff --git a/surfsense_web/hooks/use-activate-chat-thread.ts b/surfsense_web/hooks/use-activate-chat-thread.ts index 2b68ef76f..c0712908f 100644 --- a/surfsense_web/hooks/use-activate-chat-thread.ts +++ b/surfsense_web/hooks/use-activate-chat-thread.ts @@ -45,17 +45,13 @@ export function useActivateChatThread() { ); const activateChatThread = useCallback( - ({ id, title, url, workspaceId, visibility, hasComments }: ActivateChatThreadInput) => { + ({ id, url, workspaceId, visibility, hasComments }: ActivateChatThreadInput) => { const numericWorkspaceId = getWorkspaceId(workspaceId); const chatUrl = url ?? getChatUrl(workspaceId, id); syncChatTab({ chatId: id, - title: id ? title : (title ?? "New Chat"), - chatUrl, workspaceId: numericWorkspaceId, - ...(visibility !== undefined ? { visibility } : {}), - ...(hasComments !== undefined ? { hasComments } : {}), }); setCurrentThreadMetadata({ From 7b4e4792bffa43b455e3e99fe23f4da5d47f209e Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:16:09 +0530 Subject: [PATCH 087/217] fix(chat): update default chat title to "New Chat" in LayoutDataProvider and useResolvedTabs --- .../components/layout/providers/LayoutDataProvider.tsx | 2 +- surfsense_web/hooks/use-resolved-tabs.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 6858794ca..511c095e5 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -285,7 +285,7 @@ export function LayoutDataProvider({ return threadsData.threads.map((thread) => ({ id: thread.id, - name: thread.title || `Chat ${thread.id}`, + name: thread.title || "New Chat", url: `/dashboard/${workspaceId}/new-chat/${thread.id}`, visibility: thread.visibility, isOwnThread: thread.is_own_thread, diff --git a/surfsense_web/hooks/use-resolved-tabs.ts b/surfsense_web/hooks/use-resolved-tabs.ts index 6b321cbd3..1b072857a 100644 --- a/surfsense_web/hooks/use-resolved-tabs.ts +++ b/surfsense_web/hooks/use-resolved-tabs.ts @@ -66,7 +66,7 @@ export function resolveTabPointers({ const row = tab.entityId === null ? undefined : threads.get(tab.entityId); return { ...tab, - title: row?.title || (tab.entityId === null ? "New Chat" : `Chat ${tab.entityId}`), + title: row?.title || "New Chat", chatUrl: getChatUrl(tab.workspaceId, tab.entityId), ...(row?.visibility !== undefined ? { visibility: row.visibility as ChatVisibility } : {}), }; From c415e68bfc853e34b318d0479963fba6727095ef Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:41:29 +0530 Subject: [PATCH 088/217] refactor(chat): remove loading component and enhance footer visibility logic in ChatViewport --- .../[workspace_id]/new-chat/loading.tsx | 62 ------------------- .../components/assistant-ui/chat-viewport.tsx | 14 ++++- .../components/assistant-ui/thread.tsx | 5 +- 3 files changed, 15 insertions(+), 66 deletions(-) delete mode 100644 surfsense_web/app/dashboard/[workspace_id]/new-chat/loading.tsx diff --git a/surfsense_web/app/dashboard/[workspace_id]/new-chat/loading.tsx b/surfsense_web/app/dashboard/[workspace_id]/new-chat/loading.tsx deleted file mode 100644 index 108671662..000000000 --- a/surfsense_web/app/dashboard/[workspace_id]/new-chat/loading.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import { Skeleton } from "@/components/ui/skeleton"; - -export default function Loading() { - return ( -
-
-
-
- {/* User message */} -
- -
- - {/* Assistant message */} -
- - - -
- - {/* User message */} -
- -
- - {/* Assistant message */} -
- - - -
- - {/* User message */} -
- -
-
- - {/* Input bar */} -
-
- -
-
-
-
- ); -} diff --git a/surfsense_web/components/assistant-ui/chat-viewport.tsx b/surfsense_web/components/assistant-ui/chat-viewport.tsx index 83308b642..cb0c57442 100644 --- a/surfsense_web/components/assistant-ui/chat-viewport.tsx +++ b/surfsense_web/components/assistant-ui/chat-viewport.tsx @@ -22,9 +22,19 @@ const ChatScrollToBottom: FC = () => ( export interface ChatViewportProps { children: ReactNode; footer?: ReactNode; + /** + * Keep the footer (composer) pinned even when the thread has no messages — + * needed while an existing thread's messages are still loading, so the + * bottom composer stays visible above the loading skeleton. + */ + footerAlwaysVisible?: boolean; } -export const ChatViewport: FC = ({ children, footer }) => ( +export const ChatViewport: FC = ({ + children, + footer, + footerAlwaysVisible = false, +}) => ( = ({ children, footer }) => ( /> {children} {footer ? ( - !thread.isEmpty}> + footerAlwaysVisible || !thread.isEmpty}> = ({ }} > hasActiveThread || !thread.isEmpty}> + <> - + } > !hasActiveThread && thread.isEmpty}> From 3ec25bd9a49b7f83a9f63881830c1a872f0bbe54 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:41:40 +0530 Subject: [PATCH 089/217] refactor(chat): enhance Composer component to include active thread state and update footer logic --- surfsense_web/components/assistant-ui/thread.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index fc8118fe7..d10841c5c 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -173,7 +173,7 @@ const ThreadContent: FC = ({ footer={ <> - + } > @@ -531,10 +531,14 @@ const ChatUnavailableNotice: FC<{ workspaceId: number; canConfigure: boolean }> }; interface ComposerProps { + hasActiveThread?: boolean; isLoadingMessages?: boolean; } -const Composer: FC = ({ isLoadingMessages = false }) => { +const Composer: FC = ({ + hasActiveThread = false, + isLoadingMessages = false, +}) => { const [mentionedDocuments, setMentionedDocuments] = useAtom(mentionedDocumentsAtom); const setSubmittedMentions = useSetAtom(submittedMentionsAtom); const [showDocumentPopover, setShowDocumentPopover] = useState(false); @@ -1068,7 +1072,7 @@ const Composer: FC = ({ isLoadingMessages = false }) => { />
{!isLoadingMessages && isThreadEmpty && isComposerInputEmpty ? ( From 5cc05d90f0de4e4c093f0917381d726817f7dc4e Mon Sep 17 00:00:00 2001 From: Sanjay Santhanam <51058514+Sanjays2402@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:46:31 -0700 Subject: [PATCH 090/217] fix: avoid blocking audio writes TTS responses were written synchronously from the async presentation pipeline, blocking concurrent requests. Offload byte writes to a worker thread and add a regression test that keeps the event loop responsive. --- .../app/agents/video_presentation/nodes.py | 4 +-- surfsense_backend/app/utils/file_io.py | 7 +++++ .../agents/video_presentation/test_file_io.py | 29 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/app/utils/file_io.py create mode 100644 surfsense_backend/tests/unit/agents/video_presentation/test_file_io.py diff --git a/surfsense_backend/app/agents/video_presentation/nodes.py b/surfsense_backend/app/agents/video_presentation/nodes.py index dca89059f..e52d1f4ce 100644 --- a/surfsense_backend/app/agents/video_presentation/nodes.py +++ b/surfsense_backend/app/agents/video_presentation/nodes.py @@ -17,6 +17,7 @@ from app.config import config as app_config from app.services.kokoro_tts_service import get_kokoro_tts_service from app.services.llm_service import get_agent_llm from app.utils.content_utils import extract_text_content, strip_markdown_fences +from app.utils.file_io import write_bytes from .configuration import Configuration from .prompts import ( @@ -137,8 +138,7 @@ async def create_slide_audio(state: State, config: RunnableConfig) -> dict[str, kwargs["api_base"] = app_config.TTS_SERVICE_API_BASE response = await aspeech(**kwargs) - with open(chunk_path, "wb") as f: - f.write(response.content) + await write_bytes(chunk_path, response.content) return chunk_path diff --git a/surfsense_backend/app/utils/file_io.py b/surfsense_backend/app/utils/file_io.py new file mode 100644 index 000000000..2e5700955 --- /dev/null +++ b/surfsense_backend/app/utils/file_io.py @@ -0,0 +1,7 @@ +import asyncio +from pathlib import Path + + +async def write_bytes(path: str, content: bytes) -> None: + """Write bytes without blocking the event loop.""" + await asyncio.to_thread(Path(path).write_bytes, content) diff --git a/surfsense_backend/tests/unit/agents/video_presentation/test_file_io.py b/surfsense_backend/tests/unit/agents/video_presentation/test_file_io.py new file mode 100644 index 000000000..c85cef19c --- /dev/null +++ b/surfsense_backend/tests/unit/agents/video_presentation/test_file_io.py @@ -0,0 +1,29 @@ +import asyncio +import threading + +import pytest + +from app.utils.file_io import write_bytes + +pytestmark = pytest.mark.unit + + +@pytest.mark.asyncio +async def test_write_bytes_does_not_block_event_loop(monkeypatch, tmp_path): + write_started = threading.Event() + release_write = threading.Event() + + def blocking_write(_path, _content): + write_started.set() + release_write.wait(timeout=1) + return 5 + + monkeypatch.setattr("pathlib.Path.write_bytes", blocking_write) + + task = asyncio.create_task(write_bytes(str(tmp_path / "audio.mp3"), b"audio")) + while not write_started.is_set(): + await asyncio.sleep(0) + + assert not task.done() + release_write.set() + await task From 0d2c8df5bd6b74a2a1c4e9d1ee97dddeae38e39a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:21 +0200 Subject: [PATCH 091/217] feat(walmart): add billing units and rate config --- surfsense_backend/app/capabilities/core/billing.py | 4 ++++ surfsense_backend/app/capabilities/core/types.py | 2 ++ surfsense_backend/app/config/__init__.py | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index c2c7f6b5b..1d08a8bc7 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -41,6 +41,8 @@ _PLATFORM_RATE_KEYS: dict[BillingUnit, str] = { BillingUnit.TIKTOK_VIDEO: "TIKTOK_MICROS_PER_VIDEO", BillingUnit.TIKTOK_USER: "TIKTOK_MICROS_PER_USER", BillingUnit.TIKTOK_COMMENT: "TIKTOK_MICROS_PER_COMMENT", + BillingUnit.WALMART_PRODUCT: "WALMART_MICROS_PER_PRODUCT", + BillingUnit.WALMART_REVIEW: "WALMART_MICROS_PER_REVIEW", } @@ -63,6 +65,8 @@ _UNIT_NOUNS: dict[BillingUnit, str] = { BillingUnit.TIKTOK_VIDEO: "video", BillingUnit.TIKTOK_USER: "profile", BillingUnit.TIKTOK_COMMENT: "comment", + BillingUnit.WALMART_PRODUCT: "product", + BillingUnit.WALMART_REVIEW: "review", } diff --git a/surfsense_backend/app/capabilities/core/types.py b/surfsense_backend/app/capabilities/core/types.py index 2e8d3b6cc..ba63a0908 100644 --- a/surfsense_backend/app/capabilities/core/types.py +++ b/surfsense_backend/app/capabilities/core/types.py @@ -31,6 +31,8 @@ class BillingUnit(StrEnum): TIKTOK_VIDEO = "tiktok_video" TIKTOK_USER = "tiktok_user" TIKTOK_COMMENT = "tiktok_comment" + WALMART_PRODUCT = "walmart_product" + WALMART_REVIEW = "walmart_review" class BillableInput(Protocol): diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 20497bb15..d90aa2542 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -736,6 +736,12 @@ class Config: # Comments are the cheapest per-item TikTok data, matching the per-comment # market (and YouTube's comment meter). TIKTOK_MICROS_PER_COMMENT = int(os.getenv("TIKTOK_MICROS_PER_COMMENT", "1500")) + # Walmart products come from server-rendered JSON behind residential proxies, + # priced alongside Amazon's per-product meter. + WALMART_MICROS_PER_PRODUCT = int(os.getenv("WALMART_MICROS_PER_PRODUCT", "3500")) + # Reviews are 10 per page (many light requests per product), priced on the + # cheaper per-review market like the Google Maps review meter. + WALMART_MICROS_PER_REVIEW = int(os.getenv("WALMART_MICROS_PER_REVIEW", "1500")) # Retry an empty listing draw on a fresh rotating IP. Set to 1 for a static # proxy, where every retry re-hits the same exit. TIKTOK_LISTING_MAX_ATTEMPTS = int(os.getenv("TIKTOK_LISTING_MAX_ATTEMPTS", "3")) From a72cc3436cff02a341c1059c96b071a1b06242ec Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:21 +0200 Subject: [PATCH 092/217] feat(walmart): add product and review schemas --- .../proprietary/platforms/walmart/schemas.py | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/schemas.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/schemas.py b/surfsense_backend/app/proprietary/platforms/walmart/schemas.py new file mode 100644 index 000000000..ea25f2eff --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/schemas.py @@ -0,0 +1,188 @@ +# ruff: noqa: N815 +"""Input/output models for the public Walmart scraper. + +Two verbs share this module: ``walmart.scrape`` (products + listings) and +``walmart.reviews`` (deep paginated reviews). Walmart is a Next.js app that +ships its data as JSON in a ``', re.DOTALL) +_APP_DATA_RE = re.compile(r'', re.DOTALL) + + +def extract_next_data(html: str | None) -> dict[str, Any] | None: + """Return the parsed Next.js state object, or ``None`` when absent/invalid. + + Tries ``__NEXT_DATA__`` first, then the ``__APP_DATA__`` fallback so a single + Walmart layout experiment does not blank the whole extractor. + """ + if not html: + return None + for pattern in (_NEXT_DATA_RE, _APP_DATA_RE): + match = pattern.search(html) + if not match: + continue + try: + data = json.loads(match.group(1)) + except (ValueError, TypeError): + logger.warning("Walmart hidden JSON present but did not parse") + continue + if isinstance(data, dict): + return data + return None + + +def dig(obj: Any, *keys: str | int) -> Any: + """Walk nested dict/list keys, returning ``None`` on any miss. + + Tolerates the layout drift between Walmart's ``initialData`` variants without + a cascade of ``if key in ...`` guards at every call site. + """ + current = obj + for key in keys: + if isinstance(key, int): + if not isinstance(current, list) or not -len(current) <= key < len(current): + return None + current = current[key] + else: + if not isinstance(current, dict) or key not in current: + return None + current = current[key] + return current + + +def initial_data(next_data: dict[str, Any]) -> dict[str, Any] | None: + """The ``props.pageProps.initialData`` node shared by every page type.""" + node = dig(next_data, "props", "pageProps", "initialData") + return node if isinstance(node, dict) else None From 34b6885ba220988140135f96e38d33f5cf1afc26 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:21 +0200 Subject: [PATCH 095/217] feat(walmart): add fetch with proxy rotation and block detection --- .../proprietary/platforms/walmart/fetch.py | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/fetch.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/fetch.py b/surfsense_backend/app/proprietary/platforms/walmart/fetch.py new file mode 100644 index 000000000..851b6a674 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/fetch.py @@ -0,0 +1,171 @@ +"""Network access for public Walmart pages. + +Mirrors the Amazon fetch layer: every request goes through the configured +residential proxy (pinned to the US, since Walmart geo-locks inventory), and any +response that looks like an anti-bot interstitial is retried on a fresh proxy +exit. Ordinary HTTP failures are returned to the caller for domain-specific +handling. + +Walmart runs Akamai (edge/TLS) + PerimeterX/HUMAN (behavioral JS). Two Walmart +specifics differ from Amazon: + +* Walmart serves CAPTCHA with an HTTP ``200`` body ("Robot or human?"), so block + detection scans the body, never the status alone. +* ``412`` is PerimeterX's rejection code and is treated as blocked → rotate. + +``ponytail:`` MVP hits only the server-rendered ``__NEXT_DATA__`` pages, which +TLS impersonation + residential proxies clear without seeding PerimeterX cookies. +If block rates on those pages climb, the upgrade path is a warmed sticky session +(seed ``_px3``/``_pxhd``/``ACID`` from a homepage fetch, reuse exit + cookies) — +the same shape as Amazon's ``get_location_session``. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from typing import Any + +from scrapling.fetchers import AsyncFetcher + +from app.utils.proxy import get_geo_proxy_url, get_sticky_proxy_url + +logger = logging.getLogger(__name__) + +_MAX_IP_ATTEMPTS = 6 +_REQUEST_TIMEOUT_S = 30 +_HEADERS = { + "Accept-Language": "en-US,en;q=0.9", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", +} +_BLOCK_MARKERS = ( + "robot or human", + "px-captcha", + "/blocked", + "verify you are a human", + "access to this page has been denied", +) + + +@dataclass(frozen=True) +class FetchResult: + """The response details needed by scraper flows.""" + + status: int + html: str + url: str + cookies: dict[str, str] + headers: dict[str, str] = field(default_factory=dict) + + +async def gather_bounded[T]( + factories: list[Callable[[], Awaitable[T]]], *, concurrency: int +) -> list[T]: + """Run async factories concurrently while preserving input order.""" + if not factories: + return [] + semaphore = asyncio.Semaphore(max(1, concurrency)) + + async def run(factory: Callable[[], Awaitable[T]]) -> T: + async with semaphore: + return await factory() + + return await asyncio.gather(*(run(factory) for factory in factories)) + + +def is_blocked( + html: str | None, status: int, headers: dict[str, str] | None = None +) -> bool: + """Return whether a response is a Walmart anti-bot interstitial. + + ``412`` is PerimeterX's rejection; ``429``/``503`` are throttles. Walmart also + serves CAPTCHA with a ``200`` body, so the body is scanned regardless of + status. + """ + if status in {412, 429, 503}: + return True + text = (html or "")[:200_000].lower() + return any(marker in text for marker in _BLOCK_MARKERS) + + +def _response_url(page: Any, fallback: str) -> str: + value = getattr(page, "url", None) + return str(value) if value else fallback + + +def _response_cookies(page: Any) -> dict[str, str]: + cookies = getattr(page, "cookies", None) + return dict(cookies) if isinstance(cookies, dict) else {} + + +def _response_headers(page: Any) -> dict[str, str]: + headers = getattr(page, "headers", None) + return dict(headers) if isinstance(headers, dict) else {} + + +def _selected_proxy( + proxy: str | None, country: str, attempt: int, url: str +) -> str | None: + if proxy is not None: + return proxy + if attempt > 1: + session_id = f"walmart-{attempt}-{abs(hash((url, time.time_ns()))):x}" + return get_sticky_proxy_url(session_id, country) + return get_geo_proxy_url(country) + + +async def fetch_page( + url: str, + *, + cookies: dict[str, str] | None = None, + proxy: str | None = None, + country: str = "us", + rotate_on_block: bool = True, +) -> FetchResult | None: + """Fetch a page and retry blocked responses with fresh proxy exits.""" + attempts = _MAX_IP_ATTEMPTS if rotate_on_block else 1 + for attempt in range(1, attempts + 1): + selected_proxy = _selected_proxy(proxy, country, attempt, url) + started = time.perf_counter() + try: + page = await AsyncFetcher.get( + url, + headers={**_HEADERS}, + cookies=cookies or {}, + proxy=selected_proxy, + stealthy_headers=True, + timeout=_REQUEST_TIMEOUT_S, + ) + except Exception as exc: + logger.warning("Walmart request failed for %s: %s", url, exc) + if proxy is not None: + return None + continue + + status = int(getattr(page, "status", 0) or 0) + html = getattr(page, "html_content", None) or "" + response_headers = _response_headers(page) + logger.info( + "[walmart][perf] status=%s attempt=%s fetch_ms=%.1f url=%s", + status, + attempt, + (time.perf_counter() - started) * 1000, + url, + ) + if rotate_on_block and is_blocked(html, status, response_headers): + logger.info( + "Walmart blocked proxy attempt %s/%s for %s", attempt, attempts, url + ) + continue + return FetchResult( + status=status, + html=html, + url=_response_url(page, url), + cookies=_response_cookies(page), + headers=response_headers, + ) + logger.warning("Walmart exhausted %s proxy attempts for %s", attempts, url) + return None From 661bb8c1872210bd28bc04a2efad83cad8307717 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:22 +0200 Subject: [PATCH 096/217] feat(walmart): add JSON parsers --- .../proprietary/platforms/walmart/parsers.py | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/parsers.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/parsers.py b/surfsense_backend/app/proprietary/platforms/walmart/parsers.py new file mode 100644 index 000000000..9a0d070a1 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/parsers.py @@ -0,0 +1,231 @@ +"""Pure parsers for Walmart's hidden ``__NEXT_DATA__`` JSON. + +Product, listing, and review data all live in the same Next.js state tree; these +functions navigate it defensively (via :func:`.next_data.dig`) and normalize the +fields into the stable output shape. Missing sections yield ``None``/``[]`` so an +isolated schema change never discards an otherwise usable record. +""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import urljoin + +from .next_data import dig, initial_data + +_WALMART_ORIGIN = "https://www.walmart.com" + + +# --------------------------------------------------------------------------- # +# Shared helpers. # +# --------------------------------------------------------------------------- # + + +def _price(node: Any) -> dict[str, Any] | None: + """Normalize a Walmart price node into ``{value, currency}``.""" + if not isinstance(node, dict): + return None + value = node.get("price") + currency = node.get("currencyUnit") + if value is None and currency is None: + return None + return {"value": value, "currency": currency} + + +def _seller(raw: dict[str, Any]) -> dict[str, Any] | None: + name = raw.get("sellerName") or raw.get("sellerDisplayName") + seller_id = raw.get("sellerId") + if not name and not seller_id: + return None + is_walmart = bool(name) and "walmart" in name.lower() + return { + "id": seller_id, + "name": name, + "type": "WALMART" if is_walmart else "MARKETPLACE", + } + + +def _absolute(url: str | None) -> str | None: + return urljoin(_WALMART_ORIGIN, url) if url else None + + +# --------------------------------------------------------------------------- # +# Listing / search cards. # +# --------------------------------------------------------------------------- # + + +def _listing_card(raw: dict[str, Any]) -> dict[str, Any] | None: + """Normalize one search/category result item into a product card.""" + item_id = raw.get("usItemId") or raw.get("id") + name = raw.get("name") + if not item_id or not name: + return None + price_info = raw.get("priceInfo") or {} + availability = raw.get("availabilityStatusV2") or {} + image = raw.get("imageInfo") or {} + return { + "usItemId": str(item_id), + "name": name, + "brand": raw.get("brand"), + "url": _absolute(raw.get("canonicalUrl")), + "price": _price(price_info.get("currentPrice")) + or ({"value": raw.get("price")} if raw.get("price") is not None else None), + "listPrice": _price(price_info.get("wasPrice")), + "stars": raw.get("averageRating"), + "reviewsCount": raw.get("numberOfReviews"), + "seller": _seller(raw), + "availabilityStatus": availability.get("value") + or ("OUT_OF_STOCK" if raw.get("isOutOfStock") else None), + "inStock": (availability.get("value") == "IN_STOCK") + if availability.get("value") + else (raw.get("isOutOfStock") is False if "isOutOfStock" in raw else None), + "thumbnailImage": image.get("thumbnailUrl") or raw.get("image"), + "sponsored": raw.get("isSponsoredFlag"), + } + + +def parse_listing_page(next_data: dict[str, Any]) -> list[dict[str, Any]]: + """Extract normalized product cards from a search/category/browse page.""" + data = initial_data(next_data) + if data is None: + return [] + stacks = dig(data, "searchResult", "itemStacks") + if not isinstance(stacks, list): + return [] + cards: list[dict[str, Any]] = [] + seen: set[str] = set() + for stack in stacks: + for raw in (stack or {}).get("items") or []: + if not isinstance(raw, dict) or raw.get("__typename") not in ( + None, + "Product", + ): + continue + card = _listing_card(raw) + if card and card["usItemId"] not in seen: + seen.add(card["usItemId"]) + cards.append(card) + return cards + + +# --------------------------------------------------------------------------- # +# Product detail. # +# --------------------------------------------------------------------------- # + + +def _images(image_info: dict[str, Any]) -> list[str]: + images: list[str] = [] + for entry in image_info.get("allImages") or []: + url = entry.get("url") if isinstance(entry, dict) else None + if url and url not in images: + images.append(url) + return images + + +def _reviews_sample( + reviews: dict[str, Any] | None, limit: int = 10 +) -> dict[str, Any] | None: + if not isinstance(reviews, dict): + return None + customer = reviews.get("customerReviews") or [] + return { + "averageOverallRating": reviews.get("averageOverallRating"), + "totalReviewCount": reviews.get("totalReviewCount"), + "aspects": reviews.get("aspects") or [], + "topReviews": [ + normalize_review(r) for r in customer[:limit] if isinstance(r, dict) + ], + } + + +def parse_product( + next_data: dict[str, Any], *, url: str, include_reviews_sample: bool = True +) -> dict[str, Any]: + """Extract normalized product fields from a product detail page.""" + data = initial_data(next_data) + product = dig(data, "data", "product") if data else None + if not isinstance(product, dict): + return {} + idml = dig(data, "data", "idml") if data else None + reviews = dig(data, "data", "reviews") if data else None + price_info = product.get("priceInfo") or {} + image_info = product.get("imageInfo") or {} + availability = product.get("availabilityStatus") + + fields: dict[str, Any] = { + "usItemId": str(product.get("usItemId") or product.get("id") or "") or None, + "name": product.get("name"), + "brand": product.get("brand"), + "url": url, + "price": _price(price_info.get("currentPrice")), + "listPrice": _price(price_info.get("wasPrice")), + "currency": dig(price_info, "currentPrice", "currencyUnit"), + "availabilityStatus": availability, + "inStock": availability == "IN_STOCK" if availability else None, + "stars": product.get("averageRating"), + "reviewsCount": product.get("numberOfReviews"), + "seller": _seller(product), + "manufacturerName": product.get("manufacturerName"), + "shortDescription": product.get("shortDescription"), + "longDescription": (idml or {}).get("longDescription") + if isinstance(idml, dict) + else None, + "thumbnailImage": image_info.get("thumbnailUrl"), + "images": _images(image_info), + "category": dig(product, "category", "path"), + "variants": product.get("variantCriteria") or [], + } + if include_reviews_sample: + fields["reviewsSample"] = _reviews_sample(reviews) + return {key: value for key, value in fields.items() if value is not None} + + +# --------------------------------------------------------------------------- # +# Reviews (deep pagination). # +# --------------------------------------------------------------------------- # + + +def normalize_review(raw: dict[str, Any]) -> dict[str, Any]: + """Normalize one Walmart ``customerReviews`` record into a ``ReviewItem`` dict.""" + badges = raw.get("badges") or [] + verified = any( + isinstance(b, dict) and b.get("id") == "VerifiedPurchaser" for b in badges + ) + photos = raw.get("photos") or raw.get("media") or [] + images: list[str] = [] + for photo in photos: + if not isinstance(photo, dict): + continue + url = photo.get("normalUrl") or dig(photo, "sizes", "normal", "url") + if url and url not in images: + images.append(url) + responses = raw.get("clientResponses") or [] + seller_response = None + if responses and isinstance(responses[0], dict): + seller_response = responses[0].get("response") or responses[0].get( + "responseText" + ) + return { + "reviewId": raw.get("reviewId"), + "rating": raw.get("rating"), + "title": raw.get("reviewTitle"), + "text": raw.get("reviewText"), + "submissionTime": raw.get("reviewSubmissionTime"), + "author": raw.get("userNickname"), + "verifiedPurchase": verified, + "positiveFeedback": raw.get("positiveFeedback"), + "negativeFeedback": raw.get("negativeFeedback"), + "images": images, + "syndicated": bool(raw.get("syndicationSource")), + "sellerResponse": seller_response, + } + + +def parse_reviews_page(next_data: dict[str, Any]) -> list[dict[str, Any]]: + """Extract normalized reviews from one ``/reviews/product/{id}`` page.""" + data = initial_data(next_data) + reviews = dig(data, "data", "reviews") if data else None + customer = reviews.get("customerReviews") if isinstance(reviews, dict) else None + if not isinstance(customer, list): + return [] + return [normalize_review(r) for r in customer if isinstance(r, dict)] From d446aa32ec083c0b481837f9eb0fbdc55b5880ea Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:22 +0200 Subject: [PATCH 097/217] feat(walmart): add scrape and review orchestration --- .../proprietary/platforms/walmart/README.md | 52 ++++ .../proprietary/platforms/walmart/__init__.py | 27 ++ .../proprietary/platforms/walmart/scraper.py | 288 ++++++++++++++++++ 3 files changed, 367 insertions(+) create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/README.md create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/walmart/scraper.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/README.md b/surfsense_backend/app/proprietary/platforms/walmart/README.md new file mode 100644 index 000000000..982ee5400 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/README.md @@ -0,0 +1,52 @@ +# Walmart Scraper + +Two verbs read Walmart's public, anonymous pages: `walmart.scrape` (products + +search/category/browse listings, per-product billing) and `walmart.reviews` +(deep paginated reviews, per-review billing). + +## Scope + +The scraper reads public pages available to anonymous visitors — no login, no +account cookies. Data is extracted from the Next.js `__NEXT_DATA__` JSON blob +embedded in each page (with an `__APP_DATA__` fallback), not from the rendered +DOM, because Walmart obfuscates CSS classes and A/B-tests layout constantly. + +`walmart.scrape` returns a free sample of on-page reviews (rating distribution, +aspects, top reviews) under `reviewsSample`. `walmart.reviews` fetches the full +review history from the public `/reviews/product/{usItemId}` page, which +robots.txt permits (unlike `/search`). + +## Architecture + +- `schemas.py` defines the stable input, product, review, and error models. +- `url_resolver.py` classifies product (`/ip/`) vs listing (`/search`, `/cp/`, + `/browse/`) URLs and extracts the numeric `usItemId`. +- `next_data.py` extracts and navigates the hidden Next.js JSON state. +- `fetch.py` owns proxy-aware HTTP access (US-pinned), block detection, and + retries. +- `parsers.py` contains pure, defensive JSON parsers. +- `scraper.py` coordinates discovery, enrichment, pagination, concurrency, + limits, and in-stream error items. + +## Anti-bot + +Walmart runs Akamai (edge/TLS) + PerimeterX/HUMAN (behavioral JS). Requests go +through US residential proxies with TLS-impersonated headers; blocked responses +(body markers, `412`/`429`/`503`, or the `200`-OK CAPTCHA body) rotate to a +fresh proxy exit. + +Known ceilings and upgrade paths (see `fetch.py` / `scraper.py` `ponytail:` +notes): reviews page at 10/page; search capped at Walmart's 25-page limit; +session warming (seed `_px3`/`_pxhd` on a sticky exit) is the next lever if +block rates on the SSR pages climb, and the `/orchestra/*` GraphQL API is +deliberately avoided (rotating persisted-query hashes make it brittle). + +## Verification + +Offline fixtures cover the parsers and both flows: + + uv run pytest tests/unit/platforms/walmart/ + +A manual live check (requires network + residential proxy): + + uv run python scripts/e2e_walmart_scraper.py diff --git a/surfsense_backend/app/proprietary/platforms/walmart/__init__.py b/surfsense_backend/app/proprietary/platforms/walmart/__init__.py new file mode 100644 index 000000000..83b7cf277 --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/__init__.py @@ -0,0 +1,27 @@ +"""Platform-native Walmart scraper (products, listings, reviews).""" + +from .schemas import ( + ErrorItem, + ProductItem, + ReviewItem, + WalmartReviewsInput, + WalmartScrapeInput, +) +from .scraper import ( + iter_products, + iter_reviews, + scrape_products, + scrape_reviews, +) + +__all__ = [ + "ErrorItem", + "ProductItem", + "ReviewItem", + "WalmartReviewsInput", + "WalmartScrapeInput", + "iter_products", + "iter_reviews", + "scrape_products", + "scrape_reviews", +] diff --git a/surfsense_backend/app/proprietary/platforms/walmart/scraper.py b/surfsense_backend/app/proprietary/platforms/walmart/scraper.py new file mode 100644 index 000000000..e883efdbc --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/walmart/scraper.py @@ -0,0 +1,288 @@ +"""Orchestrate public Walmart product discovery, enrichment, and reviews. + +Two streaming cores: :func:`iter_products` dispatches each start URL to a product +or listing flow, and :func:`iter_reviews` paginates the public reviews page per +item id. Network and parsing concerns stay isolated in their own modules so +markup changes and retry policy can be tested independently. + +The failure model is in-stream error items (dicts with an ``error`` key), never +exceptions — identical to the Amazon scraper. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse + +from .fetch import fetch_page, gather_bounded +from .next_data import extract_next_data +from .parsers import parse_listing_page, parse_product, parse_reviews_page +from .schemas import ( + ErrorItem, + ProductItem, + ReviewItem, + WalmartReviewsInput, + WalmartScrapeInput, +) +from .url_resolver import ResolvedUrl, extract_item_id, resolve_url + +__all__ = ["iter_products", "iter_reviews", "scrape_products", "scrape_reviews"] + +_DETAIL_CONCURRENCY = 6 +_SEARCH_PAGE_LIMIT = 25 # Walmart caps result pagination at 25 pages. +_REVIEWS_PAGE_LIMIT = 500 # 10 reviews/page → 5000-review safety ceiling. +_DEFAULT_ITEMS_PER_START_URL = 40 + +# Friendly sort → Walmart's ``sort`` query value on the reviews page. +_REVIEW_SORT = { + "most-recent": "submission-desc", + "most-helpful": "helpful", + "rating-high": "rating-desc", + "rating-low": "rating-asc", +} + + +def _error( + code: str, description: str, *, input_url: str | None, url: str | None = None +) -> dict[str, Any]: + return ErrorItem( + error=code, + errorDescription=description, + input=input_url, + url=url or input_url, + ).model_dump() + + +def _page_url(url: str, page: int) -> str: + parsed = urlparse(url) + query = dict(parse_qsl(parsed.query, keep_blank_values=True)) + query["page"] = str(page) + return urlunparse(parsed._replace(query=urlencode(query))) + + +def _product_url(item_id: str) -> str: + return f"https://www.walmart.com/ip/{item_id}" + + +def _reviews_url(item_id: str, page: int, sort: str) -> str: + query = urlencode( + {"page": page, "sort": sort, "entryPoint": "viewAllReviewsBottom"} + ) + return f"https://www.walmart.com/reviews/product/{item_id}?{query}" + + +# --------------------------------------------------------------------------- # +# Product / listing flows. # +# --------------------------------------------------------------------------- # + + +async def _product_flow( + resolved: ResolvedUrl, input_model: WalmartScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Fetch and parse one product detail page.""" + url = _product_url(resolved.item_id) if resolved.item_id else resolved.url + response = await fetch_page(url, country=input_model.country) + if response is None: + yield _error( + "product_not_found", + "The product page could not be loaded after retrying proxy exits.", + input_url=resolved.url, + ) + return + if response.status in {404, 410}: + yield _error( + "product_not_found", + "The product page was not found.", + input_url=resolved.url, + ) + return + + next_data = extract_next_data(response.html) + fields = ( + parse_product( + next_data, + url=response.url, + include_reviews_sample=input_model.includeReviewsSample, + ) + if next_data + else {} + ) + if not fields.get("name"): + yield _error( + "product_not_found", + "The response did not contain a recognizable product.", + input_url=resolved.url, + url=response.url, + ) + return + fields["input"] = resolved.url + yield ProductItem(**fields).to_output() + + +async def _listing_flow( + resolved: ResolvedUrl, input_model: WalmartScrapeInput +) -> AsyncIterator[dict[str, Any]]: + """Page through a search/category/browse listing and optionally enrich.""" + cap = ( + input_model.maxItemsPerStartUrl + if input_model.maxItemsPerStartUrl is not None + else _DEFAULT_ITEMS_PER_START_URL + ) + max_pages = min(input_model.maxSearchPagesPerStartUrl, _SEARCH_PAGE_LIMIT) + seen: set[str] = set() + emitted = 0 + for page in range(1, max_pages + 1): + response = await fetch_page( + _page_url(resolved.url, page), country=input_model.country + ) + next_data = extract_next_data(response.html) if response else None + cards = parse_listing_page(next_data) if next_data else [] + cards = [c for c in cards if c["usItemId"] not in seen] + for card in cards: + seen.add(card["usItemId"]) + if not cards: + if page == 1: + yield _error( + "no_results_found", + "The listing page did not contain any products.", + input_url=resolved.url, + ) + return + cards = cards[: max(0, cap - emitted)] + + if not input_model.includeDetails: + for card in cards: + card["input"] = resolved.url + yield ProductItem(**card).to_output() + emitted += 1 + else: + + async def load_card(card: dict[str, Any]) -> list[dict[str, Any]]: + product = resolve_url(card["url"]) if card.get("url") else None + if product is None or product.item_id is None: + card["input"] = resolved.url + return [ProductItem(**card).to_output()] + return [item async for item in _product_flow(product, input_model)] + + for batch in await gather_bounded( + [lambda card=card: load_card(card) for card in cards], + concurrency=_DETAIL_CONCURRENCY, + ): + for item in batch: + if "error" not in item and emitted >= cap: + return + yield item + if "error" not in item: + emitted += 1 + if emitted >= cap: + return + + +_FLOWS = {"product": _product_flow, "listing": _listing_flow} + + +async def iter_products( + input_model: WalmartScrapeInput, +) -> AsyncIterator[dict[str, Any]]: + """Yield product items for every start URL. + + Each URL is classified and dispatched to its per-kind flow. An unrecognized + URL yields an ``invalid_url`` error item. + """ + for url in input_model.startUrls: + resolved = resolve_url(url) + if resolved is None: + yield _error( + "invalid_url", + "Start URL was malformed or not a recognized Walmart URL.", + input_url=url, + ) + continue + async for item in _FLOWS[resolved.kind](resolved, input_model): + yield item + + +async def scrape_products( + input_model: WalmartScrapeInput, *, limit: int | None = None +) -> list[dict[str, Any]]: + """Collect :func:`iter_products` into a list, honoring an optional ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + async for item in iter_products(input_model): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="product") + if limit is not None and len(results) >= limit: + break + return results + + +# --------------------------------------------------------------------------- # +# Reviews flow (deep pagination). # +# --------------------------------------------------------------------------- # + + +async def _reviews_flow( + item_id: str, input_model: WalmartReviewsInput +) -> AsyncIterator[dict[str, Any]]: + """Paginate the public reviews page until empty or ``maxReviews`` reached. + + Walmart does not expose a reliable total up front, so the loop stops on the + first empty page rather than trusting a page count. + """ + sort = _REVIEW_SORT[input_model.sort] + emitted = 0 + for page in range(1, _REVIEWS_PAGE_LIMIT + 1): + response = await fetch_page( + _reviews_url(item_id, page, sort), country=input_model.country + ) + next_data = extract_next_data(response.html) if response else None + reviews = parse_reviews_page(next_data) if next_data else [] + if not reviews: + if page == 1 and emitted == 0: + yield _error( + "reviews_not_found", + "The product has no public reviews or could not be loaded.", + input_url=item_id, + ) + return + for raw in reviews: + raw["usItemId"] = item_id + raw["input"] = item_id + yield ReviewItem(**raw).to_output() + emitted += 1 + if emitted >= input_model.maxReviews: + return + + +async def iter_reviews( + input_model: WalmartReviewsInput, +) -> AsyncIterator[dict[str, Any]]: + """Yield review items for every requested item id (or resolvable URL).""" + for raw_id in input_model.itemIds: + item_id = extract_item_id(raw_id) + if item_id is None: + yield _error( + "invalid_url", + "Could not extract a Walmart item id from the input.", + input_url=raw_id, + ) + continue + async for item in _reviews_flow(item_id, input_model): + yield item + + +async def scrape_reviews( + input_model: WalmartReviewsInput, *, limit: int | None = None +) -> list[dict[str, Any]]: + """Collect :func:`iter_reviews` into a list, honoring an optional ``limit``.""" + from app.capabilities.core.progress import emit_progress + + results: list[dict[str, Any]] = [] + async for item in iter_reviews(input_model): + results.append(item) + emit_progress("scraping", current=len(results), total=limit, unit="review") + if limit is not None and len(results) >= limit: + break + return results From 9b9091ae97cf9385510e7e43671547f54137ac5d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:22 +0200 Subject: [PATCH 098/217] feat(walmart): add scrape and reviews capabilities --- .../app/capabilities/walmart/__init__.py | 6 ++ .../capabilities/walmart/reviews/__init__.py | 3 + .../walmart/reviews/definition.py | 24 +++++++ .../capabilities/walmart/reviews/executor.py | 40 +++++++++++ .../capabilities/walmart/reviews/schemas.py | 72 +++++++++++++++++++ .../capabilities/walmart/scrape/__init__.py | 3 + .../capabilities/walmart/scrape/definition.py | 22 ++++++ .../capabilities/walmart/scrape/executor.py | 46 ++++++++++++ .../capabilities/walmart/scrape/schemas.py | 64 +++++++++++++++++ 9 files changed, 280 insertions(+) create mode 100644 surfsense_backend/app/capabilities/walmart/__init__.py create mode 100644 surfsense_backend/app/capabilities/walmart/reviews/__init__.py create mode 100644 surfsense_backend/app/capabilities/walmart/reviews/definition.py create mode 100644 surfsense_backend/app/capabilities/walmart/reviews/executor.py create mode 100644 surfsense_backend/app/capabilities/walmart/reviews/schemas.py create mode 100644 surfsense_backend/app/capabilities/walmart/scrape/__init__.py create mode 100644 surfsense_backend/app/capabilities/walmart/scrape/definition.py create mode 100644 surfsense_backend/app/capabilities/walmart/scrape/executor.py create mode 100644 surfsense_backend/app/capabilities/walmart/scrape/schemas.py diff --git a/surfsense_backend/app/capabilities/walmart/__init__.py b/surfsense_backend/app/capabilities/walmart/__init__.py new file mode 100644 index 000000000..1031f4230 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/__init__.py @@ -0,0 +1,6 @@ +"""``walmart.*`` namespace: platform-native Walmart data verbs.""" + +from __future__ import annotations + +from app.capabilities.walmart.reviews import definition as _reviews # noqa: F401 +from app.capabilities.walmart.scrape import definition as _scrape # noqa: F401 diff --git a/surfsense_backend/app/capabilities/walmart/reviews/__init__.py b/surfsense_backend/app/capabilities/walmart/reviews/__init__.py new file mode 100644 index 000000000..473d60402 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/__init__.py @@ -0,0 +1,3 @@ +"""Walmart deep review scraping capability.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/walmart/reviews/definition.py b/surfsense_backend/app/capabilities/walmart/reviews/definition.py new file mode 100644 index 000000000..6984ee031 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/definition.py @@ -0,0 +1,24 @@ +"""``walmart.reviews`` capability registration (billed per review; see config +``WALMART_MICROS_PER_REVIEW``).""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.walmart.reviews.executor import build_reviews_executor +from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput + +WALMART_REVIEWS = Capability( + name="walmart.reviews", + description=( + "Fetch deep paginated public Walmart product reviews with ratings, text, " + "authors, verified-purchase flags, images, and seller responses. Use " + "product urls or item ids." + ), + input_schema=ReviewsInput, + output_schema=ReviewsOutput, + executor=build_reviews_executor(), + billing_unit=BillingUnit.WALMART_REVIEW, + docs_url="/docs/connectors/native/walmart", +) + +register_capability(WALMART_REVIEWS) diff --git a/surfsense_backend/app/capabilities/walmart/reviews/executor.py b/surfsense_backend/app/capabilities/walmart/reviews/executor.py new file mode 100644 index 000000000..35f88e11a --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/executor.py @@ -0,0 +1,40 @@ +"""``walmart.reviews`` executor: verb input → scraper → review items.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput +from app.proprietary.platforms.walmart import WalmartReviewsInput, scrape_reviews + +ReviewsFn = Callable[..., Awaitable[list[dict]]] + + +def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor: + """Bind the executor to a reviews scraper fn (defaults to the proprietary actor).""" + scrape_fn = scrape_fn or scrape_reviews + + async def execute(payload: ReviewsInput) -> ReviewsOutput: + input_model = WalmartReviewsInput( + itemIds=payload.sources(), + maxReviews=payload.max_reviews, + sort=payload.sort_by, + ) + emit_progress( + "starting", + "Fetching Walmart reviews", + total=payload.estimated_units, + unit="review", + ) + items = await scrape_fn(input_model, limit=payload.estimated_units) + emit_progress( + "done", + f"Scraped {sum('error' not in item for item in items)} review(s)", + current=len(items), + unit="review", + ) + return ReviewsOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/walmart/reviews/schemas.py b/surfsense_backend/app/capabilities/walmart/reviews/schemas.py new file mode 100644 index 000000000..faf56ed5b --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/reviews/schemas.py @@ -0,0 +1,72 @@ +"""``walmart.reviews`` I/O contracts. + +A lean surface over ``WalmartReviewsInput``; the scraper's ``ReviewItem`` is +reused verbatim as the output element. Accepts product URLs or bare item ids — +both resolve to a ``usItemId`` the reviews page is keyed on. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.walmart import ReviewItem + +MAX_WALMART_REVIEW_SOURCES = 20 + + +class ReviewsInput(BaseModel): + urls: list[str] = Field( + default_factory=list, + max_length=MAX_WALMART_REVIEW_SOURCES, + description=( + "Walmart product URLs (/ip/...) or reviews URLs to fetch reviews for. " + "Provide these OR item_ids (at least one is required)." + ), + ) + item_ids: list[str] = Field( + default_factory=list, + max_length=MAX_WALMART_REVIEW_SOURCES, + description="Walmart numeric item ids (usItemId) to fetch reviews for.", + ) + max_reviews: int = Field( + default=200, + ge=1, + le=5000, + description="Max reviews to return per product (10 per page).", + ) + sort_by: Literal["most-recent", "most-helpful", "rating-high", "rating-low"] = ( + Field(default="most-recent", description="Review ordering.") + ) + + @model_validator(mode="after") + def _require_a_source(self) -> ReviewsInput: + if not (self.urls or self.item_ids): + raise ValueError("Provide at least one of 'urls' or 'item_ids'.") + return self + + def sources(self) -> list[str]: + """URLs and item ids merged; the scraper resolves each to a usItemId.""" + return [*self.urls, *self.item_ids] + + @property + def estimated_units(self) -> int: + """Worst-case billable reviews: up to ``max_reviews`` per source.""" + return (len(self.urls) + len(self.item_ids)) * self.max_reviews + + +class ReviewsOutput(BaseModel): + items: list[ReviewItem] = Field( + default_factory=list, + description="One item per review, in the scraper's emission order.", + ) + + @property + def billable_units(self) -> int: + """One returned review = one billable unit; error items are not billed.""" + return sum( + 1 + for item in self.items + if not (item.model_extra and item.model_extra.get("error")) + ) diff --git a/surfsense_backend/app/capabilities/walmart/scrape/__init__.py b/surfsense_backend/app/capabilities/walmart/scrape/__init__.py new file mode 100644 index 000000000..bb3e8f794 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/__init__.py @@ -0,0 +1,3 @@ +"""Walmart product scraping capability.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/walmart/scrape/definition.py b/surfsense_backend/app/capabilities/walmart/scrape/definition.py new file mode 100644 index 000000000..6165a8992 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/definition.py @@ -0,0 +1,22 @@ +"""Registration for the ``walmart.scrape`` capability.""" + +from __future__ import annotations + +from app.capabilities.core import BillingUnit, Capability, register_capability +from app.capabilities.walmart.scrape.executor import build_scrape_executor +from app.capabilities.walmart.scrape.schemas import ScrapeInput, ScrapeOutput + +WALMART_SCRAPE = Capability( + name="walmart.scrape", + description=( + "Scrape public Walmart product details, search/category listings, " + "prices, sellers, variants, availability, and a sample of on-page reviews." + ), + input_schema=ScrapeInput, + output_schema=ScrapeOutput, + executor=build_scrape_executor(), + billing_unit=BillingUnit.WALMART_PRODUCT, + docs_url="/docs/connectors/native/walmart", +) + +register_capability(WALMART_SCRAPE) diff --git a/surfsense_backend/app/capabilities/walmart/scrape/executor.py b/surfsense_backend/app/capabilities/walmart/scrape/executor.py new file mode 100644 index 000000000..4c4b073ee --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/executor.py @@ -0,0 +1,46 @@ +"""Executor for the ``walmart.scrape`` capability.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +from app.capabilities.core import Executor +from app.capabilities.core.progress import emit_progress +from app.capabilities.walmart.scrape.schemas import ( + MAX_WALMART_RESULTS, + ScrapeInput, + ScrapeOutput, +) +from app.proprietary.platforms.walmart import WalmartScrapeInput, scrape_products + +ScrapeFn = Callable[..., Awaitable[list[dict]]] + + +def build_scrape_executor(scrape_fn: ScrapeFn | None = None) -> Executor: + """Bind the capability input mapping to a replaceable scraper function.""" + scrape_fn = scrape_fn or scrape_products + + async def execute(payload: ScrapeInput) -> ScrapeOutput: + input_model = WalmartScrapeInput( + startUrls=payload.start_urls(), + maxItemsPerStartUrl=payload.max_items, + includeDetails=payload.include_details, + includeReviewsSample=payload.include_reviews_sample, + ) + emit_progress( + "starting", + "Scraping Walmart products", + total=payload.estimated_units, + unit="product", + ) + items = await scrape_fn(input_model, limit=MAX_WALMART_RESULTS) + emit_progress( + "done", + f"Scraped {sum('error' not in item for item in items)} product(s)", + current=len(items), + total=payload.estimated_units, + unit="product", + ) + return ScrapeOutput(items=items) + + return execute diff --git a/surfsense_backend/app/capabilities/walmart/scrape/schemas.py b/surfsense_backend/app/capabilities/walmart/scrape/schemas.py new file mode 100644 index 000000000..2d42dc371 --- /dev/null +++ b/surfsense_backend/app/capabilities/walmart/scrape/schemas.py @@ -0,0 +1,64 @@ +"""Input and output contracts for ``walmart.scrape``.""" + +from __future__ import annotations + +from urllib.parse import quote_plus + +from pydantic import BaseModel, Field, model_validator + +from app.proprietary.platforms.walmart import ProductItem + +MAX_WALMART_SOURCES = 20 +MAX_WALMART_RESULTS = 1000 + + +class ScrapeInput(BaseModel): + """Agent-facing controls for public Walmart product discovery and enrichment.""" + + urls: list[str] = Field(default_factory=list, max_length=MAX_WALMART_SOURCES) + search_terms: list[str] = Field( + default_factory=list, max_length=MAX_WALMART_SOURCES + ) + max_items: int = Field(default=10, ge=1, le=100) + include_details: bool = True + include_reviews_sample: bool = True + + @model_validator(mode="after") + def _require_source(self) -> ScrapeInput: + if not (self.urls or self.search_terms): + raise ValueError("Provide at least one URL or search term.") + if len(self.urls) + len(self.search_terms) > MAX_WALMART_SOURCES: + raise ValueError( + f"Provide no more than {MAX_WALMART_SOURCES} combined sources." + ) + return self + + def start_urls(self) -> list[str]: + """Direct URLs plus a search URL synthesized per search term.""" + searches = [ + f"https://www.walmart.com/search?q={quote_plus(term)}" + for term in self.search_terms + ] + return [*self.urls, *searches] + + @property + def estimated_units(self) -> int: + """Worst-case returned products within the hard per-run ceiling.""" + search_products = len(self.search_terms) * self.max_items + direct_products = len(self.urls) * self.max_items + return min(search_products + direct_products, MAX_WALMART_RESULTS) + + +class ScrapeOutput(BaseModel): + """Products and structured per-input errors in emission order.""" + + items: list[ProductItem] = Field(default_factory=list) + + @property + def billable_units(self) -> int: + """Count successful products; error items are never billed.""" + return sum( + 1 + for item in self.items + if not (item.model_extra and item.model_extra.get("error")) + ) From a3f8a7e7413e1405ae793a890c7f07ec27056b67 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:22 +0200 Subject: [PATCH 099/217] feat(walmart): register routes and MCP tools --- surfsense_backend/app/routes/__init__.py | 1 + .../mcp_server/features/scrapers/__init__.py | 7 +- .../features/scrapers/platforms/walmart.py | 140 ++++++++++++++++++ 3 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 509233e15..e76d3ea1d 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -7,6 +7,7 @@ import app.capabilities.google_search import app.capabilities.instagram import app.capabilities.reddit import app.capabilities.tiktok +import app.capabilities.walmart import app.capabilities.web import app.capabilities.youtube # noqa: F401 from app.automations.api import router as automations_router diff --git a/surfsense_mcp/mcp_server/features/scrapers/__init__.py b/surfsense_mcp/mcp_server/features/scrapers/__init__.py index e6f1055cd..524250060 100644 --- a/surfsense_mcp/mcp_server/features/scrapers/__init__.py +++ b/surfsense_mcp/mcp_server/features/scrapers/__init__.py @@ -1,7 +1,8 @@ """Scraper tools: one MCP surface per SurfSense platform capability. -Web crawl, Google Search, Reddit, YouTube, and Google Maps each get a tool that -maps a natural-language request to the workspace's scraper. Two run-history tools +Web crawl, Google Search, Reddit, YouTube, Google Maps, Amazon, and Walmart each +get a tool that maps a natural-language request to the workspace's scraper. Two +run-history tools list and fetch past runs, so a large result truncated inline can be retrieved in full later. Each platform lives in its own module under platforms/. """ @@ -20,6 +21,7 @@ from .platforms import ( instagram, reddit, tiktok, + walmart, web, youtube, ) @@ -33,6 +35,7 @@ _REGISTRARS = ( tiktok, google_maps, amazon, + walmart, run_history, ) diff --git a/surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py b/surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py new file mode 100644 index 000000000..0c11af653 --- /dev/null +++ b/surfsense_mcp/mcp_server/features/scrapers/platforms/walmart.py @@ -0,0 +1,140 @@ +"""Walmart scraper tools: products/listings and deep reviews.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from ....core.client import SurfSenseClient +from ....core.rendering import ResponseFormatParam +from ....core.workspace_context import WorkspaceContext, WorkspaceParam +from ..annotations import SCRAPE +from ..capability import run_scraper + +ReviewSort = Literal["most-recent", "most-helpful", "rating-high", "rating-low"] + + +def register(mcp: FastMCP, client: SurfSenseClient, context: WorkspaceContext) -> None: + """Register the Walmart product and review tools.""" + + @mcp.tool( + name="surfsense_walmart_scrape", + title="Scrape Walmart products", + annotations=SCRAPE, + structured_output=False, + ) + async def walmart_scrape( + urls: Annotated[ + list[str] | None, + Field( + description="Walmart product (/ip/), search (/search), category " + "(/cp/), or browse (/browse/) URLs. Provide urls OR search_terms." + ), + ] = None, + search_terms: Annotated[ + list[str] | None, + Field( + description="Search phrases run on walmart.com, e.g. ['air fryer']. " + "Provide search_terms OR urls." + ), + ] = None, + max_items: Annotated[ + int, + Field(ge=1, le=100, description="Max products per search term or listing URL."), + ] = 10, + include_details: Annotated[ + bool, + Field( + description="Fetch full product detail pages. False returns faster " + "card-only results from listings." + ), + ] = True, + include_reviews_sample: Annotated[ + bool, + Field( + description="Include the free on-page review sample (rating " + "distribution, aspects, top reviews) on detail pages." + ), + ] = True, + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Scrape public Walmart product data by URL or search term. + + Use this for product research: title, price, list price, rating and + review count, availability, seller (Walmart 1P vs marketplace), images, + description, variants, and a sample of on-page reviews. Only public, + anonymous data — no login. For a product's full review history use + surfsense_walmart_reviews instead. + Example: search_terms=['air fryer'], max_items=5. + """ + return await run_scraper( + client, + context, + platform="walmart", + verb="scrape", + payload={ + "urls": urls, + "search_terms": search_terms, + "max_items": max_items, + "include_details": include_details, + "include_reviews_sample": include_reviews_sample, + }, + workspace=workspace, + response_format=response_format, + ) + + @mcp.tool( + name="surfsense_walmart_reviews", + title="Fetch Walmart reviews", + annotations=SCRAPE, + structured_output=False, + ) + async def walmart_reviews( + urls: Annotated[ + list[str] | None, + Field( + description="Walmart product URLs (/ip/...). Provide urls OR item_ids." + ), + ] = None, + item_ids: Annotated[ + list[str] | None, + Field( + description="Walmart numeric item ids (usItemId) from " + "surfsense_walmart_scrape." + ), + ] = None, + max_reviews: Annotated[ + int, + Field(ge=1, le=5000, description="Max reviews per product (10 per page)."), + ] = 200, + sort_by: Annotated[ + ReviewSort, Field(description="Review ordering.") + ] = "most-recent", + workspace: WorkspaceParam = None, + response_format: ResponseFormatParam = "markdown", + ) -> str: + """Fetch deep paginated customer reviews for Walmart products. + + Use this to read the full review history on specific products; get urls + or item_ids from surfsense_walmart_scrape first if you only have a name. + Returns rating, title, text, author, verified-purchase flag, images, and + seller response per review. + Example: item_ids=['212092810'], sort_by='most-helpful', max_reviews=100. + """ + return await run_scraper( + client, + context, + platform="walmart", + verb="reviews", + payload={ + "urls": urls, + "item_ids": item_ids, + "max_reviews": max_reviews, + "sort_by": sort_by, + }, + workspace=workspace, + response_format=response_format, + ) From 7b5e09c528c2aebbe1ba861624ca48bd00c8e648 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:19:22 +0200 Subject: [PATCH 100/217] test(walmart): add parser and flow tests with e2e harness --- .../scripts/e2e_walmart_scraper.py | 80 ++++++++++ .../tests/unit/platforms/walmart/__init__.py | 0 .../platforms/walmart/fixtures/blocked.html | 4 + .../platforms/walmart/fixtures/listing.html | 3 + .../platforms/walmart/fixtures/product.html | 3 + .../platforms/walmart/fixtures/reviews.html | 3 + .../unit/platforms/walmart/test_flows.py | 138 ++++++++++++++++++ .../unit/platforms/walmart/test_parsers.py | 90 ++++++++++++ 8 files changed, 321 insertions(+) create mode 100644 surfsense_backend/scripts/e2e_walmart_scraper.py create mode 100644 surfsense_backend/tests/unit/platforms/walmart/__init__.py create mode 100644 surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html create mode 100644 surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html create mode 100644 surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html create mode 100644 surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html create mode 100644 surfsense_backend/tests/unit/platforms/walmart/test_flows.py create mode 100644 surfsense_backend/tests/unit/platforms/walmart/test_parsers.py diff --git a/surfsense_backend/scripts/e2e_walmart_scraper.py b/surfsense_backend/scripts/e2e_walmart_scraper.py new file mode 100644 index 000000000..28ed3a344 --- /dev/null +++ b/surfsense_backend/scripts/e2e_walmart_scraper.py @@ -0,0 +1,80 @@ +"""Manual end-to-end check for the public Walmart scraper. + +Run from the backend directory: + + uv run python scripts/e2e_walmart_scraper.py + +The script requires live network access and the configured residential proxy +(US exit). It is intentionally excluded from pytest. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path + +from dotenv import load_dotenv + +_BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_BACKEND_ROOT)) +for _candidate in (_BACKEND_ROOT / ".env", _BACKEND_ROOT.parent / ".env"): + if _candidate.exists(): + load_dotenv(_candidate) + break + +from app.proprietary.platforms.walmart import ( # noqa: E402 + WalmartReviewsInput, + WalmartScrapeInput, + scrape_products, + scrape_reviews, +) + +_PRODUCT_URL = "https://www.walmart.com/ip/212092810" +_SEARCH_URL = "https://www.walmart.com/search?q=air+fryer" + + +def _check(label: str, passed: bool) -> bool: + print(f"[{'PASS' if passed else 'FAIL'}] {label}") + return passed + + +async def main() -> int: + ok = True + + product_items = await scrape_products( + WalmartScrapeInput(startUrls=[_PRODUCT_URL]), limit=1 + ) + product = product_items[0] if product_items else {} + print(json.dumps(product, indent=2, ensure_ascii=False)[:2500]) + ok &= _check( + "product detail has id and name", + bool(product.get("usItemId") and product.get("name")), + ) + + search_items = await scrape_products( + WalmartScrapeInput( + startUrls=[_SEARCH_URL], maxItemsPerStartUrl=3, includeDetails=False + ), + limit=3, + ) + ok &= _check( + "search returns product cards", + bool(search_items) and any("name" in item for item in search_items), + ) + + item_id = product.get("usItemId") or "212092810" + reviews = await scrape_reviews( + WalmartReviewsInput(itemIds=[item_id], maxReviews=15), limit=15 + ) + ok &= _check( + "reviews returned with rating and text", + bool(reviews) and any(r.get("rating") and r.get("text") for r in reviews), + ) + + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/surfsense_backend/tests/unit/platforms/walmart/__init__.py b/surfsense_backend/tests/unit/platforms/walmart/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html new file mode 100644 index 000000000..a9e9d606e --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/blocked.html @@ -0,0 +1,4 @@ +Robot or human? +
Activate and hold the button to confirm that you're human.
+

Robot or human?

+ diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html new file mode 100644 index 000000000..0ebb7e682 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/listing.html @@ -0,0 +1,3 @@ +laptop - Walmart.com + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html new file mode 100644 index 000000000..3db17b152 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/product.html @@ -0,0 +1,3 @@ +Midea AC + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html b/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html new file mode 100644 index 000000000..b0d27e5b6 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/fixtures/reviews.html @@ -0,0 +1,3 @@ +Reviews + + diff --git a/surfsense_backend/tests/unit/platforms/walmart/test_flows.py b/surfsense_backend/tests/unit/platforms/walmart/test_flows.py new file mode 100644 index 000000000..045aa99d5 --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/test_flows.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from pathlib import Path + +from app.proprietary.platforms.walmart import ( + WalmartReviewsInput, + WalmartScrapeInput, + scrape_products, + scrape_reviews, + scraper, +) +from app.proprietary.platforms.walmart.fetch import FetchResult + +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _fixture(name: str) -> str: + return (_FIXTURES / name).read_text(encoding="utf-8") + + +def _response(url: str, html: str, status: int = 200) -> FetchResult: + return FetchResult(status=status, html=html, url=url, cookies={}) + + +async def test_product_flow_emits_parsed_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, _fixture("product.html")) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"]) + ) + + assert len(items) == 1 + assert items[0]["usItemId"] == "212092810" + assert items[0]["name"].startswith("Midea") + + +async def test_product_flow_maps_not_found_to_error_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, "", status=404) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://www.walmart.com/ip/212092810"]) + ) + + assert items[0]["error"] == "product_not_found" + + +async def test_listing_flow_card_only_honors_cap(monkeypatch): + calls: list[str] = [] + + async def fetch_page(url: str, **_kwargs): + calls.append(url) + html = _fixture("listing.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput( + startUrls=["https://www.walmart.com/search?q=laptop"], + maxItemsPerStartUrl=1, + includeDetails=False, + ) + ) + + assert len(items) == 1 + assert items[0]["usItemId"] == "791595618" + assert len(calls) == 1 + + +async def test_listing_flow_enriches_with_detail_pages(monkeypatch): + async def fetch_page(url: str, **_kwargs): + if "/ip/" in url: + return _response(url, _fixture("product.html")) + html = _fixture("listing.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_products( + WalmartScrapeInput( + startUrls=["https://www.walmart.com/search?q=laptop"], + maxItemsPerStartUrl=2, + includeDetails=True, + ) + ) + + # Both cards enrich to the same product fixture (detail fetch wins). + assert all(item["longDescription"] for item in items) + + +async def test_invalid_url_yields_error_item(monkeypatch): + items = await scrape_products( + WalmartScrapeInput(startUrls=["https://example.com/not-walmart"]) + ) + assert items[0]["error"] == "invalid_url" + + +async def test_reviews_flow_paginates_until_empty(monkeypatch): + calls: list[str] = [] + + async def fetch_page(url: str, **_kwargs): + calls.append(url) + html = _fixture("reviews.html") if "page=1" in url else "" + return _response(url, html) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews( + WalmartReviewsInput(itemIds=["212092810"], maxReviews=100) + ) + + assert len(items) == 2 + assert items[0]["reviewId"] == "296013686" + # page=1 returned records, page=2 empty → stop. Two fetches total. + assert len(calls) == 2 + + +async def test_reviews_flow_honors_max_reviews(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, _fixture("reviews.html")) + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews( + WalmartReviewsInput(itemIds=["212092810"], maxReviews=1) + ) + + assert len(items) == 1 + + +async def test_reviews_flow_maps_empty_to_error_item(monkeypatch): + async def fetch_page(url: str, **_kwargs): + return _response(url, "") + + monkeypatch.setattr(scraper, "fetch_page", fetch_page) + items = await scrape_reviews(WalmartReviewsInput(itemIds=["212092810"])) + + assert items[0]["error"] == "reviews_not_found" diff --git a/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py b/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py new file mode 100644 index 000000000..da66064ab --- /dev/null +++ b/surfsense_backend/tests/unit/platforms/walmart/test_parsers.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from pathlib import Path + +from app.proprietary.platforms.walmart.fetch import is_blocked +from app.proprietary.platforms.walmart.next_data import extract_next_data +from app.proprietary.platforms.walmart.parsers import ( + parse_listing_page, + parse_product, + parse_reviews_page, +) +from app.proprietary.platforms.walmart.url_resolver import extract_item_id, resolve_url + +_FIXTURES = Path(__file__).parent / "fixtures" + + +def _fixture(name: str) -> str: + return (_FIXTURES / name).read_text(encoding="utf-8") + + +def test_product_parser_extracts_core_fields_and_review_sample(): + data = extract_next_data(_fixture("product.html")) + item = parse_product(data, url="https://www.walmart.com/ip/212092810") + + assert item["usItemId"] == "212092810" + assert item["name"].startswith("Midea") + assert item["price"] == {"value": 149.0, "currency": "USD"} + assert item["listPrice"] == {"value": 199.0, "currency": "USD"} + assert item["stars"] == 4.5 + assert item["reviewsCount"] == 6287 + assert item["inStock"] is True + assert item["seller"] == {"id": "F55CT", "name": "Walmart.com", "type": "WALMART"} + assert item["longDescription"] == "

A powerful window unit.

" + assert item["images"] == [ + "https://i5.walmartimages.com/1.jpg", + "https://i5.walmartimages.com/2.jpg", + ] + assert item["reviewsSample"]["totalReviewCount"] == 6287 + assert item["reviewsSample"]["topReviews"][0]["verifiedPurchase"] is True + + +def test_listing_parser_normalizes_cards_and_marketplace_seller(): + data = extract_next_data(_fixture("listing.html")) + cards = parse_listing_page(data) + + assert [c["usItemId"] for c in cards] == ["791595618", "999001"] + first = cards[0] + assert first["url"] == "https://www.walmart.com/ip/Acer-Chromebook/791595618" + assert first["price"] == {"value": 79.99, "currency": "USD"} + assert first["seller"]["type"] == "MARKETPLACE" + assert first["inStock"] is True + # Fallback price + out-of-stock derivation for the second card. + assert cards[1]["price"] == {"value": 499.0} + assert cards[1]["inStock"] is False + + +def test_reviews_parser_extracts_all_records(): + data = extract_next_data(_fixture("reviews.html")) + reviews = parse_reviews_page(data) + + assert len(reviews) == 2 + assert reviews[0]["reviewId"] == "296013686" + assert reviews[0]["author"] == "JohnPaul" + assert reviews[0]["verifiedPurchase"] is True + assert reviews[0]["images"] == ["https://i5.walmartimages.com/r.jpg"] + assert reviews[0]["sellerResponse"] == "Thanks for the review!" + assert reviews[1]["verifiedPurchase"] is False + + +def test_block_detection_handles_status_and_body(): + assert is_blocked(_fixture("blocked.html"), 200) + assert is_blocked("", 412) + assert is_blocked("", 429) + assert not is_blocked(_fixture("product.html"), 200) + + +def test_missing_next_data_yields_none(): + assert extract_next_data("no script here") is None + assert parse_product(extract_next_data("") or {}, url="x") == {} + + +def test_url_resolver_classifies_and_extracts_ids(): + assert resolve_url("https://www.walmart.com/ip/Foo/123456789").kind == "product" + assert ( + extract_item_id("https://www.walmart.com/reviews/product/212092810") + == "212092810" + ) + assert resolve_url("https://www.walmart.com/search?q=tv").kind == "listing" + assert resolve_url("https://www.walmart.com/cp/tvs/3944").kind == "listing" + assert resolve_url("https://example.com/ip/1") is None From f6a8a2a566708ca22b74e30bdd832edeb9898a67 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:45:27 +0200 Subject: [PATCH 101/217] feat(walmart): add chat subagent with scrape and reviews tools --- .../subagents/builtins/walmart/__init__.py | 1 + .../subagents/builtins/walmart/agent.py | 43 ++++++++++++ .../subagents/builtins/walmart/description.md | 2 + .../builtins/walmart/system_prompt.md | 68 +++++++++++++++++++ .../builtins/walmart/tools/__init__.py | 0 .../subagents/builtins/walmart/tools/index.py | 28 ++++++++ 6 files changed, 142 insertions(+) create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/__init__.py create mode 100644 surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py new file mode 100644 index 000000000..8126071cf --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/__init__.py @@ -0,0 +1 @@ +"""``walmart`` builtin subagent: structured public Walmart product data and reviews.""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py new file mode 100644 index 000000000..794b050a2 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/agent.py @@ -0,0 +1,43 @@ +"""``walmart`` route: ``SurfSenseSubagentSpec`` builder for deepagents.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +from .tools.index import NAME, RULESET, load_tools + + +def build_subagent( + *, + dependencies: dict[str, Any], + model: BaseChatModel | None = None, + middleware_stack: dict[str, Any] | None = None, + mcp_tools: list[BaseTool] | None = None, +) -> SurfSenseSubagentSpec: + tools = [*load_tools(dependencies=dependencies), *(mcp_tools or [])] + description = ( + read_md_file(__package__, "description").strip() + or "Scrapes public Walmart product data and reviews for a URL or search term." + ) + system_prompt = read_md_file(__package__, "system_prompt").strip() + return pack_subagent( + name=NAME, + description=description, + system_prompt=system_prompt, + tools=tools, + ruleset=RULESET, + dependencies=dependencies, + model=model, + middleware_stack=middleware_stack, + ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md new file mode 100644 index 000000000..88c6b7c28 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/description.md @@ -0,0 +1,2 @@ +Walmart product specialist: scrapes public Walmart listings and returns structured product data — title, item id (usItemId), brand, price and list price, star rating and review count, availability, images, features, seller, and product variants — plus deep paginated customer reviews (rating, text, author, verified-purchase flag, images, and seller responses). Works from a search term (e.g. "air fryer") or from Walmart product (/ip/...), search, category, or browse URLs, and can pull many reviews per product by item id or URL. Only public, anonymous US Walmart data — no login or seller account. +Use it for product research, price tracking, catalog enrichment by item id, and review mining. Triggers include "find X on Walmart", "Walmart price of X", "reviews for this Walmart product", "compare these Walmart products", and "look up this Walmart URL/item id". Not for general web search (use the Google Search specialist), reading an arbitrary non-Walmart page (use the web crawling specialist), or other marketplaces. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md new file mode 100644 index 000000000..bf30fe2bb --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/system_prompt.md @@ -0,0 +1,68 @@ +You are the SurfSense Walmart sub-agent. +You receive delegated instructions from a supervisor agent and return structured results for supervisor synthesis. + + +Answer the delegated question from public Walmart product data and reviews gathered with your verbs, comparing against earlier results already in this conversation when the task calls for it. + + + +- `walmart_scrape` — product details and search/category listings +- `walmart_reviews` — deep paginated reviews for a product +- `read_run` / `search_run` (free readers for stored scrape output) + + + +- Discovering products: call `walmart_scrape` with `search_terms` (e.g. ["air fryer"]). +- Specific products: pass Walmart product URLs (/ip/...) or search/category/browse URLs in `urls`. +- Faster listings: set `include_details=false` to return card-only results without opening each product page. +- Sampled reviews: `walmart_scrape` returns a small on-page review sample by default (`include_reviews_sample=true`); disable it when reviews are irrelevant. +- Deep review mining: use `walmart_reviews` with product `urls` or numeric `item_ids` (usItemId); raise `max_reviews` and set `sort_by` (most-recent, most-helpful, rating-high, rating-low) as the task needs. Reviews are billed per review, so keep `max_reviews` to what the task actually requires. +- Batch multiple URLs or search terms into one call rather than many single-source calls. + +- Comparison requests: pull the current products, compare against prior values already in this conversation's earlier tool results, and report concrete deltas (price up/down, rating change, stock changes). + + + +- Use only tools in ``. +- Report only results present in the tool output. Never invent titles, item ids, prices, ratings, or reviews. +- `walmart_scrape`: provide at least one of `urls` or `search_terms`. +- `walmart_reviews`: provide at least one of `urls` or `item_ids`. + + + +- Do not perform general web search — that is the Google Search specialist's job. +- Do not read or extract an arbitrary non-Walmart page — return the URL for the web crawling specialist. +- Do not generate deliverables or perform connector mutations; return findings for the supervisor to act on. +- Only public, anonymous Walmart data — never anything behind a login or seller account. + + + +- Report uncertainty explicitly when evidence is incomplete or conflicting. +- Never present unverified claims as facts. + + + +- Underspecified request — no usable search term, URL, or item id — return `status=blocked` with the missing fields. +- Tool failure: return `status=error` with a concise recovery `next_step`. +- No useful evidence: return `status=blocked` with a narrower query or the scope you still need. + + + +Return **only** one JSON object (no markdown/prose): +{ + "status": "success" | "partial" | "blocked" | "error", + "action_summary": string, + "evidence": { + "findings": string[], + "sources": string[], + "confidence": "high" | "medium" | "low" + }, + "next_step": string | null, + "missing_fields": string[] | null, + "assumptions": string[] | null +} + +Route-specific rules: +- `evidence.findings`: max 10 entries, each a single sentence stating one distinct product, review theme, or delta. Do not paste raw payloads. +- `evidence.sources`: max 10 URLs, one per finding when applicable. List each URL once. + diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py new file mode 100644 index 000000000..ac3e5ed81 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/walmart/tools/index.py @@ -0,0 +1,28 @@ +"""``walmart`` sub-agent tools: the Walmart product scrape and reviews verbs.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset +from app.capabilities.core.access.agent import build_capability_tools +from app.capabilities.walmart.reviews.definition import WALMART_REVIEWS +from app.capabilities.walmart.scrape.definition import WALMART_SCRAPE + +NAME = "walmart" + +RULESET = Ruleset(origin=NAME, rules=[]) + +_CI_VERBS = [WALMART_SCRAPE, WALMART_REVIEWS] + + +def load_tools( + *, dependencies: dict[str, Any] | None = None, **kwargs: Any +) -> list[BaseTool]: + d = {**(dependencies or {}), **kwargs} + return build_capability_tools( + workspace_id=d.get("workspace_id"), + capabilities=_CI_VERBS, + ) From cabc5b5c86b0b602bae2a52ef99a1cbfd93ce624 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:45:27 +0200 Subject: [PATCH 102/217] feat(walmart): register subagent in roster and connector map --- .../app/agents/chat/multi_agent_chat/constants.py | 1 + .../app/agents/chat/multi_agent_chat/subagents/registry.py | 4 ++++ .../unit/agents/multi_agent_chat/test_subagent_composition.py | 1 + 3 files changed, 6 insertions(+) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py index 333d4d274..2c44dbefe 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py @@ -39,6 +39,7 @@ SUBAGENT_TO_REQUIRED_CONNECTOR_MAP: dict[str, frozenset[str]] = { "reddit": frozenset(), "instagram": frozenset(), "tiktok": frozenset(), + "walmart": frozenset(), "mcp_discovery": frozenset( { "SLACK_CONNECTOR", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 7b8b8a6f1..48de3e245 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -42,6 +42,9 @@ from app.agents.chat.multi_agent_chat.subagents.builtins.reddit.agent import ( from app.agents.chat.multi_agent_chat.subagents.builtins.tiktok.agent import ( build_subagent as build_tiktok_subagent, ) +from app.agents.chat.multi_agent_chat.subagents.builtins.walmart.agent import ( + build_subagent as build_walmart_subagent, +) from app.agents.chat.multi_agent_chat.subagents.builtins.web_crawler.agent import ( build_subagent as build_web_crawler_subagent, ) @@ -96,6 +99,7 @@ SUBAGENT_BUILDERS_BY_NAME: dict[str, SubagentBuilder] = { "onedrive": build_onedrive_subagent, "reddit": build_reddit_subagent, "tiktok": build_tiktok_subagent, + "walmart": build_walmart_subagent, "web_crawler": build_web_crawler_subagent, "youtube": build_youtube_subagent, } diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py index f578f3457..bc12f1308 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -40,6 +40,7 @@ _EXPECTED_SUBAGENTS = frozenset( "onedrive", "reddit", "tiktok", + "walmart", "web_crawler", "youtube", } From 56e8fa13e6ebbb88a9dda9ac62f8ff969ccfee6c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:45:27 +0200 Subject: [PATCH 103/217] feat(walmart): route product-review questions to marketplace specialists --- .../main_agent/system_prompt/prompts/identity/private.md | 2 +- .../main_agent/system_prompt/prompts/identity/team.md | 2 +- .../main_agent/system_prompt/prompts/kb_first.md | 3 ++- .../main_agent/system_prompt/prompts/routing.md | 4 +++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md index 82444634a..56e933253 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md @@ -6,7 +6,7 @@ are changing, and what is being published across the open web — and to put that research to work alongside their own knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google +- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web crawler return structured, current platform data (posts, comments, transcripts, videos, products, reviews, SERPs, full page content). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md index 38cec63dc..a16a8bfd2 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md @@ -6,7 +6,7 @@ are changing, and what is being published across the open web — and to put that research to work alongside the team's shared knowledge base. You do this by dispatching **specialist subagents** via the `task` tool: -- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Google +- **Live web data** — Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web crawler return structured, current platform data (posts, comments, transcripts, videos, products, reviews, SERPs, full page content). diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md index 3bae85262..60192fb39 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md @@ -2,7 +2,8 @@ CRITICAL — ground factual answers in what you actually receive this turn: - **live platform data** via the market specialists — `task(reddit, ...)`, `task(youtube, ...)`, `task(instagram, ...)`, - `task(tiktok, ...)`, `task(amazon, ...)`, `task(google_maps, ...)`, + `task(tiktok, ...)`, `task(amazon, ...)`, `task(walmart, ...)`, + `task(google_maps, ...)`, `task(google_search, ...)`, `task(web_crawler, ...)`. Anything about competitors, markets, rankings, reviews, or audience sentiment is answered from what these return **this turn**, never from your training data: your diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md index fb818cabc..7e2a0f983 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md @@ -32,7 +32,9 @@ about a brand, product, or topic is answered from the platform where they say it — `task(reddit, …)` for community discussion and threads, `task(youtube, …)` for video content, transcripts, and comment sections, `task(tiktok, …)` for short-form video trends by hashtag or search, -`task(google_maps, …)` for customer reviews of physical businesses. Web +`task(google_maps, …)` for customer reviews of physical businesses, +`task(amazon, …)` / `task(walmart, …)` for product ratings and customer +reviews of retail products (Walmart pages the full review history). Web search only finds articles *about* the conversation; the platform specialists return the conversation itself, structured and current. For competitive questions ("what are people saying about X", "how is Y From d7c30cac2f4a1622059199179a2117b99bed5783 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:45:27 +0200 Subject: [PATCH 104/217] refactor(walmart): align reviews executor docstring --- surfsense_backend/app/capabilities/walmart/reviews/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_backend/app/capabilities/walmart/reviews/executor.py b/surfsense_backend/app/capabilities/walmart/reviews/executor.py index 35f88e11a..ba1ee0367 100644 --- a/surfsense_backend/app/capabilities/walmart/reviews/executor.py +++ b/surfsense_backend/app/capabilities/walmart/reviews/executor.py @@ -13,7 +13,7 @@ ReviewsFn = Callable[..., Awaitable[list[dict]]] def build_reviews_executor(scrape_fn: ReviewsFn | None = None) -> Executor: - """Bind the executor to a reviews scraper fn (defaults to the proprietary actor).""" + """Bind the capability input mapping to a replaceable reviews scraper function.""" scrape_fn = scrape_fn or scrape_reviews async def execute(payload: ReviewsInput) -> ReviewsOutput: From b5cddcfb2b251d5a0f88ea929b62da6b4bfe1863 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:49:10 +0200 Subject: [PATCH 105/217] docs(walmart): document billing rates in env examples --- docker/.env.example | 4 +++- surfsense_backend/.env.example | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index cd9789fae..a92939e4a 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -441,7 +441,7 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000 # Debit the credit wallet per *item returned* by the platform-native scrapers -# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps scraping +# (Reddit, Google Search, Google Maps, Amazon, Walmart, YouTube). Default FALSE keeps scraping # effectively free for self-hosted installs. Each rate is micro-USD per item, # config-driven: = round(USD_per_1000_items * 1_000). Defaults sit # at/above Apify's first-party actor rates (we charge no subscription tiers, @@ -458,6 +458,8 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # TIKTOK_MICROS_PER_VIDEO=3500 # TIKTOK_MICROS_PER_USER=2500 # TIKTOK_MICROS_PER_COMMENT=1500 +# WALMART_MICROS_PER_PRODUCT=3500 +# WALMART_MICROS_PER_REVIEW=1500 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 404bd3b44..3b4a297b9 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -277,8 +277,8 @@ MICROS_PER_PAGE=1000 # WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE=3000 # Debit the credit wallet per *item returned* by the platform-native scrapers -# (Reddit, Google Search, Google Maps, Amazon, YouTube). Default FALSE keeps -# scraping effectively free for self-hosted/OSS installs; hosted set TRUE. +# (Reddit, Google Search, Google Maps, Amazon, Walmart, YouTube). Default FALSE +# keeps scraping effectively free for self-hosted/OSS installs; hosted set TRUE. # Each rate is micro-USD per item, fully config-driven (no hardcoded rate): # = round(USD_per_1000_items * 1_000) # 3500 == $3.50/1000 | 5000 == $5/1000 | 2000 == $2/1000 @@ -301,6 +301,11 @@ MICROS_PER_PAGE=1000 # Browser-listing retries when a feed is empty (profile feed is withheld from # flagged IPs; each retry draws a fresh rotating exit IP). Set to 1 for a static IP. # TIKTOK_LISTING_MAX_ATTEMPTS=3 +# Walmart products (server-rendered JSON behind residential proxies) priced like +# Amazon; reviews are 10/page (many light requests) so priced on the per-review +# market like Google Maps reviews. +# WALMART_MICROS_PER_PRODUCT=3500 +# WALMART_MICROS_PER_REVIEW=1500 # Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. CREDIT_LOW_BALANCE_WARNING_MICROS=500000 From 07721bda40b9086f5cd05ba90f6a8a7d33cb9105 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 08:49:10 +0200 Subject: [PATCH 106/217] test(walmart): cover capability registry, schemas, and executors for both verbs --- .../unit/capabilities/walmart/__init__.py | 0 .../capabilities/walmart/reviews/__init__.py | 0 .../walmart/reviews/test_executor.py | 38 ++++++++++++ .../walmart/reviews/test_schemas.py | 37 ++++++++++++ .../capabilities/walmart/scrape/__init__.py | 0 .../walmart/scrape/test_executor.py | 42 ++++++++++++++ .../walmart/scrape/test_schemas.py | 58 +++++++++++++++++++ .../capabilities/walmart/test_registry.py | 22 +++++++ 8 files changed, 197 insertions(+) create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/reviews/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/reviews/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/reviews/test_schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/scrape/__init__.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/scrape/test_executor.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/scrape/test_schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/walmart/test_registry.py diff --git a/surfsense_backend/tests/unit/capabilities/walmart/__init__.py b/surfsense_backend/tests/unit/capabilities/walmart/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/walmart/reviews/__init__.py b/surfsense_backend/tests/unit/capabilities/walmart/reviews/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_executor.py b/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_executor.py new file mode 100644 index 000000000..d9d325780 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_executor.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from app.capabilities.walmart.reviews.executor import build_reviews_executor +from app.capabilities.walmart.reviews.schemas import ReviewsInput +from app.proprietary.platforms.walmart import WalmartReviewsInput + + +class _FakeScraper: + def __init__(self) -> None: + self.calls: list[tuple[WalmartReviewsInput, int | None]] = [] + + async def __call__( + self, input_model: WalmartReviewsInput, *, limit: int | None = None + ) -> list[dict]: + self.calls.append((input_model, limit)) + return [{"reviewId": "r1", "rating": 5}] + + +async def test_executor_maps_agent_input_to_scraper_input(): + scraper = _FakeScraper() + execute = build_reviews_executor(scraper) + + output = await execute( + ReviewsInput( + urls=["https://www.walmart.com/ip/123456"], + item_ids=["222"], + max_reviews=50, + sort_by="most-helpful", + ) + ) + + assert output.items[0].reviewId == "r1" + input_model, limit = scraper.calls[0] + assert input_model.itemIds == ["https://www.walmart.com/ip/123456", "222"] + assert input_model.maxReviews == 50 + assert input_model.sort == "most-helpful" + # limit is the pre-flight worst case: 2 sources * 50 reviews + assert limit == 100 diff --git a/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_schemas.py b/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_schemas.py new file mode 100644 index 000000000..2d35ab191 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/walmart/reviews/test_schemas.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput + + +def test_estimated_units_scale_with_sources_and_max_reviews(): + payload = ReviewsInput( + urls=["https://www.walmart.com/ip/1"], item_ids=["222"], max_reviews=150 + ) + + # 2 sources * 150 reviews each + assert payload.estimated_units == 300 + + +def test_at_least_one_source_is_required(): + with pytest.raises(ValidationError): + ReviewsInput() + + +def test_sources_merge_urls_then_item_ids(): + payload = ReviewsInput(urls=["https://www.walmart.com/ip/1"], item_ids=["222"]) + + assert payload.sources() == ["https://www.walmart.com/ip/1", "222"] + + +def test_error_items_are_not_billable(): + output = ReviewsOutput( + items=[ + {"reviewId": "r1", "rating": 5}, + {"error": "reviews_not_found"}, + ] + ) + + assert output.billable_units == 1 diff --git a/surfsense_backend/tests/unit/capabilities/walmart/scrape/__init__.py b/surfsense_backend/tests/unit/capabilities/walmart/scrape/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_executor.py b/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_executor.py new file mode 100644 index 000000000..e31e4b638 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_executor.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from app.capabilities.walmart.scrape.executor import build_scrape_executor +from app.capabilities.walmart.scrape.schemas import MAX_WALMART_RESULTS, ScrapeInput +from app.proprietary.platforms.walmart import WalmartScrapeInput + + +class _FakeScraper: + def __init__(self) -> None: + self.calls: list[tuple[WalmartScrapeInput, int | None]] = [] + + async def __call__( + self, input_model: WalmartScrapeInput, *, limit: int | None = None + ) -> list[dict]: + self.calls.append((input_model, limit)) + return [{"usItemId": "123", "name": "Product"}] + + +async def test_executor_maps_agent_input_to_scraper_input(): + scraper = _FakeScraper() + execute = build_scrape_executor(scraper) + + output = await execute( + ScrapeInput( + search_terms=["air fryer"], + urls=["https://www.walmart.com/ip/123456"], + max_items=5, + include_details=False, + include_reviews_sample=False, + ) + ) + + assert output.items[0].usItemId == "123" + input_model, limit = scraper.calls[0] + assert input_model.startUrls == [ + "https://www.walmart.com/ip/123456", + "https://www.walmart.com/search?q=air+fryer", + ] + assert input_model.maxItemsPerStartUrl == 5 + assert input_model.includeDetails is False + assert input_model.includeReviewsSample is False + assert limit == MAX_WALMART_RESULTS diff --git a/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_schemas.py b/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_schemas.py new file mode 100644 index 000000000..c7ce4ff9e --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/walmart/scrape/test_schemas.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.capabilities.walmart.scrape.schemas import ( + MAX_WALMART_RESULTS, + ScrapeInput, + ScrapeOutput, +) + + +def test_estimated_units_cover_search_and_direct_sources(): + payload = ScrapeInput( + search_terms=["air fryer", "blender"], + urls=["https://www.walmart.com/ip/123456"], + max_items=20, + ) + + # (2 search + 1 direct source) * 20 items each + assert payload.estimated_units == 60 + + +def test_estimated_units_respect_hard_run_ceiling(): + payload = ScrapeInput(search_terms=["x"] * 20, max_items=100) + assert payload.estimated_units == MAX_WALMART_RESULTS + + +def test_at_least_one_source_is_required(): + with pytest.raises(ValidationError): + ScrapeInput() + + +def test_combined_sources_are_capped(): + with pytest.raises(ValidationError): + ScrapeInput(urls=["u"] * 11, search_terms=["t"] * 10) + + +def test_start_urls_synthesize_a_search_url_per_term(): + payload = ScrapeInput( + urls=["https://www.walmart.com/ip/1"], search_terms=["air fryer"] + ) + + assert payload.start_urls() == [ + "https://www.walmart.com/ip/1", + "https://www.walmart.com/search?q=air+fryer", + ] + + +def test_error_items_are_not_billable(): + output = ScrapeOutput( + items=[ + {"usItemId": "123", "name": "Product"}, + {"error": "product_not_found", "errorDescription": "Missing"}, + ] + ) + + assert output.billable_units == 1 diff --git a/surfsense_backend/tests/unit/capabilities/walmart/test_registry.py b/surfsense_backend/tests/unit/capabilities/walmart/test_registry.py new file mode 100644 index 000000000..647ad7976 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/walmart/test_registry.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from app.capabilities.core import BillingUnit +from app.capabilities.core.store import get_capability +from app.capabilities.walmart.reviews.schemas import ReviewsInput, ReviewsOutput +from app.capabilities.walmart.scrape.schemas import ScrapeInput, ScrapeOutput + + +def test_walmart_scrape_is_registered_and_metered(): + capability = get_capability("walmart.scrape") + + assert capability.input_schema is ScrapeInput + assert capability.output_schema is ScrapeOutput + assert capability.billing_unit is BillingUnit.WALMART_PRODUCT + + +def test_walmart_reviews_is_registered_and_metered(): + capability = get_capability("walmart.reviews") + + assert capability.input_schema is ReviewsInput + assert capability.output_schema is ReviewsOutput + assert capability.billing_unit is BillingUnit.WALMART_REVIEW From 1c50a33647ec08549ced4ca43d375d74e2b4b464 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 09:01:04 +0200 Subject: [PATCH 107/217] feat(walmart): add playground catalog entry and icon --- surfsense_web/lib/playground/catalog.ts | 10 ++++++++++ surfsense_web/lib/playground/platform-icons.tsx | 1 + surfsense_web/public/connectors/walmart.svg | 11 +++++++++++ 3 files changed, 22 insertions(+) create mode 100644 surfsense_web/public/connectors/walmart.svg diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index d1c1af5cb..c32a24a20 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -6,6 +6,7 @@ import { InstagramIcon, RedditIcon, TikTokIcon, + WalmartIcon, WebIcon, YouTubeIcon, } from "./platform-icons"; @@ -96,6 +97,15 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ icon: AmazonIcon, verbs: [{ name: "amazon.scrape", verb: "scrape", label: "Scrape" }], }, + { + id: "walmart", + label: "Walmart", + icon: WalmartIcon, + verbs: [ + { name: "walmart.scrape", verb: "scrape", label: "Scrape" }, + { name: "walmart.reviews", verb: "reviews", label: "Reviews" }, + ], + }, { id: "web", label: "Web", diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx index dfa1a828d..c0afc91ef 100644 --- a/surfsense_web/lib/playground/platform-icons.tsx +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -23,6 +23,7 @@ function brandIcon(src: string, alt: string) { } export const AmazonIcon = brandIcon("/connectors/amazon.svg", "Amazon"); +export const WalmartIcon = brandIcon("/connectors/walmart.svg", "Walmart"); export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit"); export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube"); export const InstagramIcon = brandIcon("/connectors/instagram.svg", "Instagram"); diff --git a/surfsense_web/public/connectors/walmart.svg b/surfsense_web/public/connectors/walmart.svg new file mode 100644 index 000000000..dc334ef22 --- /dev/null +++ b/surfsense_web/public/connectors/walmart.svg @@ -0,0 +1,11 @@ + + Walmart + + + + + + + + + From 2a512e0015ca8ad28f18111fcd4b361e78b75fa0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 09:01:11 +0200 Subject: [PATCH 108/217] feat(walmart): add connector marketing page --- .../lib/connectors-marketing/index.ts | 2 + .../lib/connectors-marketing/walmart.tsx | 273 ++++++++++++++++++ 2 files changed, 275 insertions(+) create mode 100644 surfsense_web/lib/connectors-marketing/walmart.tsx diff --git a/surfsense_web/lib/connectors-marketing/index.ts b/surfsense_web/lib/connectors-marketing/index.ts index 7d79efd9b..888b9852a 100644 --- a/surfsense_web/lib/connectors-marketing/index.ts +++ b/surfsense_web/lib/connectors-marketing/index.ts @@ -5,6 +5,7 @@ import { instagram } from "./instagram"; import { reddit } from "./reddit"; import { tiktok } from "./tiktok"; import type { ConnectorPageContent } from "./types"; +import { walmart } from "./walmart"; import { webCrawl } from "./web-crawl"; import { youtube } from "./youtube"; @@ -19,6 +20,7 @@ const CONNECTOR_LIST: ConnectorPageContent[] = [ googleMaps, googleSearch, amazon, + walmart, webCrawl, ]; diff --git a/surfsense_web/lib/connectors-marketing/walmart.tsx b/surfsense_web/lib/connectors-marketing/walmart.tsx new file mode 100644 index 000000000..e959e2638 --- /dev/null +++ b/surfsense_web/lib/connectors-marketing/walmart.tsx @@ -0,0 +1,273 @@ +import { IconBrandWalmart } from "@tabler/icons-react"; +import type { ConnectorPageContent } from "./types"; + +export const walmart: ConnectorPageContent = { + slug: "walmart", + name: "Walmart", + cardTitle: "Walmart Product & Review API", + icon: IconBrandWalmart, + + metaTitle: "Walmart Product & Review API | SurfSense", + metaDescription: + "Scrape public Walmart product data and deep customer reviews as structured JSON: prices, ratings, sellers, variants, availability, and full review history. Start free.", + keywords: [ + "walmart product api", + "walmart scraper api", + "walmart review scraper", + "walmart reviews api", + "scrape walmart product data", + "walmart price scraper", + "walmart product data api", + "walmart price tracking api", + "walmart competitor monitoring", + "walmart seller data", + "walmart marketplace api", + "ecommerce product api", + ], + + h1: "Walmart Product and Review API", + heroLede: + "The SurfSense Walmart API scrapes public Walmart.com listings as structured JSON: price and list price, rating and review count, availability, 1P and marketplace sellers, variants, and specifications. A second verb pages the full public review history — ratings, text, authors, verified-purchase flags, images, and seller responses. Point your AI agents at a search term or product URL — no login, only public data.", + + transcript: { + prompt: "Pull the ratings and top complaints for this Walmart air fryer", + toolCall: + 'walmart.reviews({ urls: ["walmart.com/ip/..."],\n max_reviews: 500, sort_by: "most-recent" })', + rows: [ + { + primary: "Ninja 4-Qt Air Fryer — $79.00 (was $99.00)", + secondary: "4.6 stars · 8,204 reviews · In Stock", + tag: "-20%", + }, + { + primary: "512 reviews paged, 71% verified purchase", + secondary: "sorted most-recent · 132 with photos", + tag: "reviews", + }, + { + primary: 'Top complaint: "basket coating flaking after 3 months"', + secondary: "surfaced across 24 recent 1-2 star reviews", + tag: "theme", + }, + ], + resultSummary: "1 product · 512 reviews · surfaced in 6.4s", + }, + + extractIntro: + "Give the scraper Walmart product, search, category, or browse URLs, or plain search terms. The scrape verb returns product cards and full detail with a free on-page review sample; the reviews verb pages the complete public review history for any product. Walmart data is US marketplace (walmart.com), server-rendered as JSON for stability.", + extractFields: [ + { + label: "Product core", + description: + "Name, item id (usItemId), brand, price, list price, availability, in-stock flag, and canonical URL for every product.", + }, + { + label: "Ratings & review sample", + description: + "Average stars and total review count on every product, plus a free sample of on-page reviews when you fetch full detail.", + }, + { + label: "Deep reviews", + description: + "The reviews verb pages the full public review history: rating, title, text, author, date, verified-purchase flag, helpful votes, images, and seller responses.", + }, + { + label: "Sellers", + description: + "The seller behind each product, typed as Walmart first-party (1P) or third-party marketplace (3P), with id and name.", + }, + { + label: "Variants & media", + description: + "Product variants (color, size, count) with their own item ids and prices, plus the gallery and thumbnail images.", + }, + { + label: "Content & taxonomy", + description: + "Short and long descriptions, specifications, category, and breadcrumb trail for catalog enrichment and classification.", + }, + ], + + useCasesHeading: "What teams do with the Walmart API", + useCases: [ + { + title: "Review mining and product research", + description: + "Page the full review history for a product and brief an agent to cluster complaints, track sentiment over time, and separate verified-purchase signal from noise. Walmart exposes far more review depth than most marketplaces — use it to inform your roadmap or listing copy.", + }, + { + title: "Price and buy-box tracking", + description: + "Watch your own and competitors' prices, list prices, and which seller wins the offer (Walmart 1P vs a third-party marketplace seller). Run the same URLs on a schedule and diff the results to alert on undercuts and stockouts.", + }, + { + title: "Seller and marketplace intelligence", + description: + "See which third-party sellers are winning offers on the products that matter to you, and how first-party Walmart pricing compares, to size the marketplace opportunity in your category.", + }, + { + title: "Catalog and assortment enrichment", + description: + "Enrich a catalog by item id with structured attributes, variants, specifications, and images, and discover products from search or category URLs to map an entire assortment.", + }, + ], + + comparison: { + heading: "A Walmart scraper API built for agents", + intro: + "Most Walmart data APIs bill per request, split product and review data across endpoints, and leave the discovery-to-detail logic to you. Here is how SurfSense compares.", + columnLabel: "Typical Walmart data API", + rows: [ + { + feature: "Pricing", + official: "Per-request pricing that climbs fast at scale", + surfsense: "Pay per product or per review returned, with a free tier to start", + }, + { + feature: "Discovery", + official: "Separate search and detail endpoints you stitch together", + surfsense: "One verb takes search terms or product, search, category, and browse URLs", + }, + { + feature: "Reviews", + official: "A separate paid endpoint, often capped shallow", + surfsense: "A dedicated verb pages the full public review history with photos and seller replies", + }, + { + feature: "Data access", + official: "Requires accounts, keys, or approval for many providers", + surfsense: "Public, anonymous data only — no login or seller account", + }, + { + feature: "Agent-ready", + official: "No; you wire the harness yourself", + surfsense: "MCP server exposes walmart.scrape and walmart.reviews as native tools", + }, + ], + }, + + api: { + platform: "walmart", + verb: "scrape", + mcpTool: "walmart.scrape", + requestBody: { + search_terms: ["air fryer"], + max_items: 5, + include_reviews_sample: true, + }, + }, + + schema: { + requestNote: + "walmart.scrape: provide urls or search_terms (at least one), up to 20 combined sources per call. For the full review history, use the walmart.reviews verb with product urls or item_ids.", + request: [ + { + name: "urls", + type: "string[]", + description: + "Walmart product (/ip/), search (/search), category (/cp/), or browse (/browse/) URLs. Provide urls or search_terms.", + }, + { + name: "search_terms", + type: "string[]", + description: + "Search phrases run on walmart.com, e.g. 'air fryer'. Provide search_terms or urls.", + }, + { + name: "max_items", + type: "integer", + defaultValue: "10", + description: "Max products per search term or category/browse URL, 1 to 100.", + }, + { + name: "include_details", + type: "boolean", + defaultValue: "true", + description: "Fetch full product detail pages; false returns faster card-only results.", + }, + { + name: "include_reviews_sample", + type: "boolean", + defaultValue: "true", + description: + "Attach the free on-page review sample from each detail page. For full history use walmart.reviews.", + }, + ], + responseNote: + "The response is { items: [...] } with one item per product. One returned product is one billable unit; error items are never billed.", + response: [ + { + name: "name / usItemId / brand", + type: "string", + description: "Product identity: display name, Walmart item id, and brand.", + }, + { + name: "price / listPrice", + type: "object", + description: "Current price and strike-through list price, each with value and currency.", + }, + { + name: "stars / reviewsCount", + type: "number", + description: "Average star rating and total public review count.", + }, + { + name: "seller", + type: "object", + description: "Offer seller with id, name, and type (WALMART 1P or MARKETPLACE 3P).", + }, + { + name: "availabilityStatus / inStock", + type: "string", + description: "Availability label and a boolean in-stock flag.", + }, + { + name: "variants", + type: "object[]", + description: "Product variants (color, size, count) with their own item ids and prices.", + }, + { + name: "reviewsSample", + type: "object", + description: "Free sample of on-page reviews returned with full detail pages.", + }, + ], + }, + + faq: [ + { + question: "What is the Walmart product API?", + answer: + "It returns a Walmart listing's data as structured JSON instead of raw HTML. The SurfSense Walmart API scrapes public product pages and gives you price, rating, seller, variants, availability, and specifications as clean JSON your agents can read.", + }, + { + question: "Can I get the full Walmart review history?", + answer: + "Yes. The walmart.reviews verb pages the complete public review history for a product — rating, title, text, author, date, verified-purchase flag, helpful votes, images, and seller responses. Pass product URLs or item ids and set max_reviews and sort_by.", + }, + { + question: "Does it only use public data?", + answer: + "It does. The scraper collects public, anonymous Walmart data only — no login or seller account. That covers product details, prices, ratings, public reviews, seller identity, variants, and availability.", + }, + { + question: "How do I look up a product by item id or search term?", + answer: + "Pass a product URL (which contains the item id) or a phrase in search_terms to discover products. You can also pass search, category, and browse URLs. walmart.scrape handles discovery; walmart.reviews takes urls or item_ids directly.", + }, + { + question: "Which Walmart marketplace is supported?", + answer: + "The scraper targets the US marketplace at walmart.com. Data is read from the server-rendered JSON Walmart ships in the page, which keeps extraction stable against the frequent front-end A/B testing on the site.", + }, + ], + + related: [ + { label: "Amazon Product API", href: "/amazon" }, + { label: "Google Maps API", href: "/google-maps" }, + { label: "Google Search API", href: "/google-search" }, + { label: "Reddit API", href: "/reddit" }, + { label: "SurfSense MCP Server", href: "/mcp-server" }, + { label: "Read the docs", href: "/docs" }, + ], +}; From 6c98bd33a83e4656818c245427a1d9c4801b93b0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 19 Jul 2026 09:01:11 +0200 Subject: [PATCH 109/217] docs(walmart): add connector docs and list in indexes --- .../content/docs/connectors/index.mdx | 2 +- .../content/docs/connectors/native/index.mdx | 7 +- .../content/docs/connectors/native/meta.json | 1 + .../docs/connectors/native/walmart.mdx | 71 +++++++++++++++++++ .../content/docs/how-to/mcp-server.mdx | 2 +- 5 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 surfsense_web/content/docs/connectors/native/walmart.mdx diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx index 04241f79f..eb3f28052 100644 --- a/surfsense_web/content/docs/connectors/index.mdx +++ b/surfsense_web/content/docs/connectors/index.mdx @@ -12,7 +12,7 @@ Connectors bring data into SurfSense — either as searchable knowledge or as li } title="Native Connectors" - description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" + description="SurfSense's built-in scraper APIs: Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" href="/docs/connectors/native" /> + Date: Sun, 19 Jul 2026 09:01:18 +0200 Subject: [PATCH 110/217] feat(walmart): list walmart across marketing surfaces and example prompts --- surfsense_web/app/(home)/mcp-server/page.tsx | 12 +++++++----- surfsense_web/components/homepage/hero-section.tsx | 4 ++-- surfsense_web/components/homepage/persona-paths.tsx | 2 +- surfsense_web/components/seo/json-ld.tsx | 4 ++-- surfsense_web/lib/chat/example-prompts.ts | 1 + 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/surfsense_web/app/(home)/mcp-server/page.tsx b/surfsense_web/app/(home)/mcp-server/page.tsx index ab7769ee9..439e1ae6f 100644 --- a/surfsense_web/app/(home)/mcp-server/page.tsx +++ b/surfsense_web/app/(home)/mcp-server/page.tsx @@ -16,7 +16,7 @@ import type { FaqItem } from "@/lib/connectors-marketing/types"; const canonicalUrl = "https://www.surfsense.com/mcp-server"; const metaDescription = - "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; + "The SurfSense MCP server gives Claude, Cursor, and any MCP client native tools for your workspace: scrape Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, and the web, plus full knowledge base access. One API key."; export const metadata: Metadata = { title: "SurfSense MCP Server: Scraper APIs and Knowledge Base as Agent Tools", @@ -103,6 +103,8 @@ const TOOL_GROUPS = [ "surfsense_google_maps_reviews", "surfsense_google_search", "surfsense_amazon_scrape", + "surfsense_walmart_scrape", + "surfsense_walmart_reviews", "surfsense_web_crawl", "surfsense_list_scraper_runs", "surfsense_get_scraper_run", @@ -134,7 +136,7 @@ const FAQ: FaqItem[] = [ { question: "What is the SurfSense MCP server?", answer: - "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", + "It is a Model Context Protocol server that exposes your SurfSense workspace to MCP clients like Claude Code, Cursor, and Claude Desktop. Your agents get native tools for every scraper API (Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google Search, web crawl) and for searching, reading, and writing your knowledge base.", }, { question: "Which MCP clients does it work with?", @@ -222,9 +224,9 @@ export default function McpServerPage() {

The SurfSense MCP server hands Claude, Cursor, or any MCP client the whole platform: - scrape Reddit, YouTube, Instagram, TikTok, Amazon, Google Maps, Google Search, and - the open web, and search, read, and write your knowledge base. One API key, typed - tools, pay as you go. + scrape Reddit, YouTube, Instagram, TikTok, Amazon, Walmart, Google Maps, Google + Search, and the open web, and search, read, and write your knowledge base. One API + key, typed tools, pay as you go.

- - - ); - return ( <> {showDesktopHeader ? ( @@ -807,7 +749,6 @@ export function EditorPanelContent({ ) : viewerMode === "monaco" && !isLocalFileMode ? ( // Large doc — raw markdown in Monaco. Rich renderers are intentionally skipped.
- {largeDocAlert}
Date: Wed, 22 Jul 2026 12:55:36 +0530 Subject: [PATCH 132/217] refactor: remove unused thread metadata migration and clean up chat thread queries --- .../175_publish_thread_metadata_to_zero.py | 23 ---- surfsense_backend/app/zero_publication.py | 6 +- surfsense_web/hooks/use-resolved-tabs.test.ts | 20 ++-- surfsense_web/hooks/use-resolved-tabs.ts | 100 ++++++++++++------ surfsense_web/zero/queries/chat.ts | 10 -- surfsense_web/zero/queries/documents.ts | 3 - surfsense_web/zero/queries/index.ts | 3 +- surfsense_web/zero/schema/chat.ts | 6 +- 8 files changed, 82 insertions(+), 89 deletions(-) delete mode 100644 surfsense_backend/alembic/versions/175_publish_thread_metadata_to_zero.py diff --git a/surfsense_backend/alembic/versions/175_publish_thread_metadata_to_zero.py b/surfsense_backend/alembic/versions/175_publish_thread_metadata_to_zero.py deleted file mode 100644 index d4d298a4a..000000000 --- a/surfsense_backend/alembic/versions/175_publish_thread_metadata_to_zero.py +++ /dev/null @@ -1,23 +0,0 @@ -"""publish thread metadata to zero_publication - -Revision ID: 175 -Revises: 174 -""" - -from collections.abc import Sequence - -from alembic import op -from app.zero_publication import apply_publication - -revision: str = "175" -down_revision: str | None = "174" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - apply_publication(op.get_bind()) - - -def downgrade() -> None: - """No-op. Historical publication shapes are immutable.""" diff --git a/surfsense_backend/app/zero_publication.py b/surfsense_backend/app/zero_publication.py index f6dff6cdf..ea4d1c90c 100644 --- a/surfsense_backend/app/zero_publication.py +++ b/surfsense_backend/app/zero_publication.py @@ -57,12 +57,12 @@ AUTOMATION_COLS = [ "workspace_id", ] +# Only the columns Zero needs as the authz *parent* of chat messages/comments/ +# session-state (``whereExists("thread")`` + space constraint). Tab titles and +# visibility are resolved over REST (react-query), not pushed via Zero. NEW_CHAT_THREAD_COLS = [ "id", "workspace_id", - "title", - "visibility", - "created_by_id", ] # Enough to drive the lifecycle UI by push: status, the reviewable brief, and diff --git a/surfsense_web/hooks/use-resolved-tabs.test.ts b/surfsense_web/hooks/use-resolved-tabs.test.ts index 34a51df86..dfad6a5a6 100644 --- a/surfsense_web/hooks/use-resolved-tabs.test.ts +++ b/surfsense_web/hooks/use-resolved-tabs.test.ts @@ -1,30 +1,24 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { - getMissingCompleteChatIds, - resolveTabPointers, - type ResolvedTab, -} from "./use-resolved-tabs"; +import { getMissingChatIds, resolveTabPointers, type ResolvedTab } from "./use-resolved-tabs"; // Run with: pnpm exec tsx --test hooks/use-resolved-tabs.test.ts -test("does not prune unresolved chat tabs before Zero completes", () => { - const missing = getMissingCompleteChatIds({ +test("does not prune chat tabs that have not resolved as missing", () => { + const missing = getMissingChatIds({ tabs: [{ id: "chat-42", type: "chat", entityId: 42, workspaceId: 7 }], - threadRows: [], - resultType: "unknown", + notFoundIds: new Set(), }); assert.equal(missing.size, 0); }); -test("prunes unresolved chat tabs only after Zero completes", () => { - const missing = getMissingCompleteChatIds({ +test("prunes only chat tabs whose thread resolved as not found", () => { + const missing = getMissingChatIds({ tabs: [ { id: "chat-42", type: "chat", entityId: 42, workspaceId: 7 }, { id: "chat-43", type: "chat", entityId: 43, workspaceId: 7 }, ], - threadRows: [{ id: 43, title: "Present", visibility: "PRIVATE" }], - resultType: "complete", + notFoundIds: new Set([42]), }); assert.deepEqual([...missing], [42]); diff --git a/surfsense_web/hooks/use-resolved-tabs.ts b/surfsense_web/hooks/use-resolved-tabs.ts index 1b072857a..112d82983 100644 --- a/surfsense_web/hooks/use-resolved-tabs.ts +++ b/surfsense_web/hooks/use-resolved-tabs.ts @@ -1,11 +1,19 @@ "use client"; -import { useQuery } from "@rocicorp/zero/react"; +import { useQueries } from "@tanstack/react-query"; import { useAtomValue, useSetAtom } from "jotai"; import { useEffect, useMemo } from "react"; import { pruneMissingChatTabsAtom, type Tab, tabsAtom } from "@/atoms/tabs/tabs.atom"; -import type { ChatVisibility } from "@/lib/chat/thread-persistence"; -import { queries } from "@/zero/queries"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; +import { type ChatVisibility, getThreadFull } from "@/lib/chat/thread-persistence"; +import { NotFoundError } from "@/lib/error"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; + +// Thread/document metadata is read-only for tabs: the DB is the single source +// of truth and react-query is the cache. Titles/visibility stay fresh because +// the rename/visibility/delete mutations patch these same query caches (see +// lib/chat/thread-cache.ts), so a rename reflects in the tab bar immediately. +const METADATA_STALE_TIME_MS = 60 * 1000; interface ThreadRow { id: number; @@ -38,6 +46,13 @@ function rowById(rows: readonly T[] | undefined): Map< return new Map((rows ?? []).map((row) => [row.id, row])); } +// Retry transient failures (network, 5xx) but never a definitive 404 — a +// missing thread/document is authoritative and should settle immediately so +// the tab can be pruned rather than spun on. +function retryUnlessNotFound(failureCount: number, error: Error): boolean { + return !(error instanceof NotFoundError) && failureCount < 2; +} + export function getChatUrl(workspaceId: number, threadId: number | null): string { return threadId ? `/dashboard/${workspaceId}/new-chat/${threadId}` @@ -59,7 +74,9 @@ export function resolveTabPointers({ return tabs.map((tab) => { if (tab.type === "document") { const title = - tab.entityId === null ? "Document" : (documents.get(tab.entityId)?.title ?? `Document ${tab.entityId}`); + tab.entityId === null + ? "Document" + : (documents.get(tab.entityId)?.title ?? `Document ${tab.entityId}`); return { ...tab, title }; } @@ -73,23 +90,23 @@ export function resolveTabPointers({ }); } -export function getMissingCompleteChatIds({ +/** + * A chat tab is prunable only once its thread metadata fetch has settled as a + * definitive 404 (thread deleted). Transient network/5xx errors are excluded so + * an outage never wrongly closes open tabs. + */ +export function getMissingChatIds({ tabs, - threadRows, - resultType, + notFoundIds, }: { tabs: Tab[]; - threadRows?: readonly ThreadRow[]; - resultType: string; + notFoundIds: Set; }): Set { - if (resultType !== "complete") return new Set(); - - const threadIds = new Set((threadRows ?? []).map((row) => row.id)); return new Set( tabs .filter( (tab): tab is Tab & { type: "chat"; entityId: number } => - tab.type === "chat" && tab.entityId !== null && !threadIds.has(tab.entityId) + tab.type === "chat" && tab.entityId !== null && notFoundIds.has(tab.entityId) ) .map((tab) => tab.entityId) ); @@ -101,29 +118,48 @@ export function useResolvedTabs(): ResolvedTab[] { const chatIds = useMemo(() => uniqueEntityIds(tabs, "chat"), [tabs]); const documentIds = useMemo(() => uniqueEntityIds(tabs, "document"), [tabs]); - const [threadRows, threadResult] = useQuery( - queries.threads.byIds({ ids: chatIds.length > 0 ? chatIds : [-1] }) + + const threadResults = useQueries({ + queries: chatIds.map((id) => ({ + queryKey: cacheKeys.threads.detail(id), + queryFn: () => getThreadFull(id), + staleTime: METADATA_STALE_TIME_MS, + retry: retryUnlessNotFound, + })), + }); + + const documentResults = useQueries({ + queries: documentIds.map((id) => ({ + queryKey: cacheKeys.documents.document(String(id)), + queryFn: () => documentsApiService.getDocument({ id }), + staleTime: METADATA_STALE_TIME_MS, + retry: retryUnlessNotFound, + })), + }); + + const threadRows: ThreadRow[] = threadResults.flatMap((result, index) => + result.data + ? [{ id: chatIds[index], title: result.data.title, visibility: result.data.visibility }] + : [] ); - const [documentRows] = useQuery( - queries.documents.byIds({ ids: documentIds.length > 0 ? documentIds : [-1] }) + const documentRows: DocumentRow[] = documentResults.flatMap((result, index) => + result.data ? [{ id: documentIds[index], title: result.data.title }] : [] ); - const missingChatIds = useMemo( - () => - getMissingCompleteChatIds({ - tabs, - threadRows, - resultType: threadResult.type, - }), - [tabs, threadRows, threadResult.type] - ); + // Stable primitive key of the threads that settled as 404, so the prune + // effect fires only when that set changes — not on every react-query render. + const notFoundChatIdsKey = threadResults + .flatMap((result, index) => (result.error instanceof NotFoundError ? [chatIds[index]] : [])) + .sort((a, b) => a - b) + .join(","); useEffect(() => { - pruneMissingChatTabs(missingChatIds); - }, [missingChatIds, pruneMissingChatTabs]); + const notFoundIds = new Set( + notFoundChatIdsKey ? notFoundChatIdsKey.split(",").map(Number) : [] + ); + const missing = getMissingChatIds({ tabs, notFoundIds }); + if (missing.size > 0) pruneMissingChatTabs(missing); + }, [notFoundChatIdsKey, tabs, pruneMissingChatTabs]); - return useMemo( - () => resolveTabPointers({ tabs, threadRows, documentRows }), - [tabs, threadRows, documentRows] - ); + return resolveTabPointers({ tabs, threadRows, documentRows }); } diff --git a/surfsense_web/zero/queries/chat.ts b/surfsense_web/zero/queries/chat.ts index 06969a4e9..40e09a6ee 100644 --- a/surfsense_web/zero/queries/chat.ts +++ b/surfsense_web/zero/queries/chat.ts @@ -29,13 +29,3 @@ export const chatSessionQueries = { .one() ), }; - -export const threadQueries = { - byIds: defineQuery(z.object({ ids: z.array(z.number()) }), ({ args: { ids }, ctx }) => - constrainToAllowedSpaces(zql.new_chat_threads, ctx) - .where("id", "IN", ids) - .where(({ or, cmp }) => - or(cmp("createdById", ctx?.userId ?? ""), cmp("visibility", "SEARCH_SPACE")) - ) - ), -}; diff --git a/surfsense_web/zero/queries/documents.ts b/surfsense_web/zero/queries/documents.ts index 6c110aee2..d2f483989 100644 --- a/surfsense_web/zero/queries/documents.ts +++ b/surfsense_web/zero/queries/documents.ts @@ -9,9 +9,6 @@ export const documentQueries = { if (!canReadSpace(ctx, workspaceId)) return denySpace(query).orderBy("createdAt", "desc"); return constrainToAllowedSpaces(query, ctx).orderBy("createdAt", "desc"); }), - byIds: defineQuery(z.object({ ids: z.array(z.number()) }), ({ args: { ids }, ctx }) => - constrainToAllowedSpaces(zql.documents, ctx).where("id", "IN", ids) - ), }; export const connectorQueries = { diff --git a/surfsense_web/zero/queries/index.ts b/surfsense_web/zero/queries/index.ts index 2e51bf9a8..45df8fa98 100644 --- a/surfsense_web/zero/queries/index.ts +++ b/surfsense_web/zero/queries/index.ts @@ -1,6 +1,6 @@ import { defineQueries } from "@rocicorp/zero"; import { automationRunQueries } from "./automations"; -import { chatSessionQueries, commentQueries, messageQueries, threadQueries } from "./chat"; +import { chatSessionQueries, commentQueries, messageQueries } from "./chat"; import { connectorQueries, documentQueries } from "./documents"; import { folderQueries } from "./folders"; import { notificationQueries } from "./inbox"; @@ -15,7 +15,6 @@ export const queries = defineQueries({ messages: messageQueries, comments: commentQueries, chatSession: chatSessionQueries, - threads: threadQueries, user: userQueries, automationRuns: automationRunQueries, podcasts: podcastQueries, diff --git a/surfsense_web/zero/schema/chat.ts b/surfsense_web/zero/schema/chat.ts index 81673feaa..c2790d5bc 100644 --- a/surfsense_web/zero/schema/chat.ts +++ b/surfsense_web/zero/schema/chat.ts @@ -20,13 +20,13 @@ export const newChatMessageTable = table("new_chat_messages") }) .primaryKey("id"); +// Published only as the authz parent of chat messages/comments/session-state +// (whereExists("thread") + space constraint). Title/visibility are resolved +// over REST (react-query) for the tab bar, not synced through Zero. export const newChatThreadTable = table("new_chat_threads") .columns({ id: number(), workspaceId: number().from("workspace_id"), - title: string(), - visibility: string(), - createdById: string().optional().from("created_by_id"), }) .primaryKey("id"); From fd6e54f12274623fe551a9b51bb7b863cbced289 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:58:13 +0530 Subject: [PATCH 133/217] refactor: remove test file for use-resolved-tabs to streamline codebase --- surfsense_web/hooks/use-resolved-tabs.test.ts | 47 ------------------- 1 file changed, 47 deletions(-) delete mode 100644 surfsense_web/hooks/use-resolved-tabs.test.ts diff --git a/surfsense_web/hooks/use-resolved-tabs.test.ts b/surfsense_web/hooks/use-resolved-tabs.test.ts deleted file mode 100644 index dfad6a5a6..000000000 --- a/surfsense_web/hooks/use-resolved-tabs.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { getMissingChatIds, resolveTabPointers, type ResolvedTab } from "./use-resolved-tabs"; - -// Run with: pnpm exec tsx --test hooks/use-resolved-tabs.test.ts -test("does not prune chat tabs that have not resolved as missing", () => { - const missing = getMissingChatIds({ - tabs: [{ id: "chat-42", type: "chat", entityId: 42, workspaceId: 7 }], - notFoundIds: new Set(), - }); - - assert.equal(missing.size, 0); -}); - -test("prunes only chat tabs whose thread resolved as not found", () => { - const missing = getMissingChatIds({ - tabs: [ - { id: "chat-42", type: "chat", entityId: 42, workspaceId: 7 }, - { id: "chat-43", type: "chat", entityId: 43, workspaceId: 7 }, - ], - notFoundIds: new Set([42]), - }); - - assert.deepEqual([...missing], [42]); -}); - -test("merges pointer tabs with synced row titles", () => { - const resolved = resolveTabPointers({ - tabs: [ - { id: "chat-42", type: "chat", entityId: 42, workspaceId: 7 }, - { id: "doc-9", type: "document", entityId: 9, workspaceId: 7 }, - ], - threadRows: [{ id: 42, title: "Live chat title", visibility: "SEARCH_SPACE" }], - documentRows: [{ id: 9, title: "Live document title" }], - }); - - assert.deepEqual( - resolved.map((tab): Pick => ({ - id: tab.id, - title: tab.title, - })), - [ - { id: "chat-42", title: "Live chat title" }, - { id: "doc-9", title: "Live document title" }, - ] - ); -}); From ddfddb2d834d11980894baea6f2372b47b7d618d Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:58:31 +0530 Subject: [PATCH 134/217] refactor: update comments and remove staleTime from useResolvedTabs for clarity and consistency --- surfsense_web/hooks/use-resolved-tabs.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/surfsense_web/hooks/use-resolved-tabs.ts b/surfsense_web/hooks/use-resolved-tabs.ts index 112d82983..99bc90100 100644 --- a/surfsense_web/hooks/use-resolved-tabs.ts +++ b/surfsense_web/hooks/use-resolved-tabs.ts @@ -10,11 +10,10 @@ import { NotFoundError } from "@/lib/error"; import { cacheKeys } from "@/lib/query-client/cache-keys"; // Thread/document metadata is read-only for tabs: the DB is the single source -// of truth and react-query is the cache. Titles/visibility stay fresh because -// the rename/visibility/delete mutations patch these same query caches (see -// lib/chat/thread-cache.ts), so a rename reflects in the tab bar immediately. -const METADATA_STALE_TIME_MS = 60 * 1000; - +// of truth and react-query is the cache (default staleTime from the shared +// QueryClient). Titles/visibility stay fresh because the rename/visibility/ +// delete mutations patch these same query caches (see lib/chat/thread-cache.ts), +// so a rename reflects in the tab bar immediately regardless of staleness. interface ThreadRow { id: number; title: string; @@ -123,7 +122,6 @@ export function useResolvedTabs(): ResolvedTab[] { queries: chatIds.map((id) => ({ queryKey: cacheKeys.threads.detail(id), queryFn: () => getThreadFull(id), - staleTime: METADATA_STALE_TIME_MS, retry: retryUnlessNotFound, })), }); @@ -132,7 +130,6 @@ export function useResolvedTabs(): ResolvedTab[] { queries: documentIds.map((id) => ({ queryKey: cacheKeys.documents.document(String(id)), queryFn: () => documentsApiService.getDocument({ id }), - staleTime: METADATA_STALE_TIME_MS, retry: retryUnlessNotFound, })), }); From 609adea006a6080e7279b1c57b1abee072844a1e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:34:56 +0200 Subject: [PATCH 135/217] feat: add RUN citation source type --- .../app/agents/chat/multi_agent_chat/shared/citations/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/citations/models.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/citations/models.py index 1273271af..31a0e3372 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/citations/models.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/citations/models.py @@ -17,6 +17,7 @@ class CitationSourceType(StrEnum): WEB_RESULT = "web_result" CHAT_TURN = "chat_turn" ANON_CHUNK = "anon_chunk" + RUN = "run" class CitationEntry(BaseModel): From 81f2d694bd6e2ec37f46197dbbf2ac69ccd24379 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:34:56 +0200 Subject: [PATCH 136/217] feat: map run citation to run_ payload --- .../multi_agent_chat/shared/citations/markers.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/citations/markers.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/citations/markers.py index 025d364f6..61d1cbd8b 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/citations/markers.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/citations/markers.py @@ -1,10 +1,11 @@ """Map a registered citation to the frontend ``[citation:]`` payload. The citation renderer understands a chunk id (``42``), a negative chunk id for -anonymous uploads (``-3``), and a URL. This is the seam that turns a server-side -source into one the renderer can resolve; it grows as more source kinds become -renderable. Kinds with no renderable form yet return ``None`` so the marker is -dropped rather than emitted broken. +anonymous uploads (``-3``), a URL, and a scraper-run handle (``run_``). +This is the seam that turns a server-side source into one the renderer can +resolve; it grows as more source kinds become renderable. Kinds with no +renderable form yet return ``None`` so the marker is dropped rather than +emitted broken. """ from __future__ import annotations @@ -22,6 +23,9 @@ def to_frontend_payload(entry: CitationEntry) -> str | None: case CitationSourceType.WEB_RESULT: url = locator.get("url") return url or None + case CitationSourceType.RUN: + run_id = locator.get("run_id") + return str(run_id) if run_id else None case _: # Connector items and chat turns have no client-side renderer yet # (the frontend resolves only chunk ids and URLs), so they stay From 957e57bb82b8c71e627640a6fe32912a0532c761 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:34:56 +0200 Subject: [PATCH 137/217] feat: register a scraper run as a citation --- .../capabilities/core/access/run_citation.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 surfsense_backend/app/capabilities/core/access/run_citation.py diff --git a/surfsense_backend/app/capabilities/core/access/run_citation.py b/surfsense_backend/app/capabilities/core/access/run_citation.py new file mode 100644 index 000000000..9b7c3f3db --- /dev/null +++ b/surfsense_backend/app/capabilities/core/access/run_citation.py @@ -0,0 +1,23 @@ +"""Register a recorded scraper run as a citable ``[n]``.""" + +from __future__ import annotations + +from app.agents.chat.multi_agent_chat.shared.citations import ( + CitationRegistry, + CitationSourceType, +) + + +def attach_run_citation( + registry: CitationRegistry, + *, + run_external_id: str, + capability: str, +) -> tuple[int, str]: + """Register the ``run_`` handle; return its ``[n]`` and the label line.""" + n = registry.register( + CitationSourceType.RUN, + {"run_id": run_external_id}, + {"capability": capability}, + ) + return n, f"\n\nCite this scraper run as [{n}] after any claim drawn from its data." From c1ba4215fd3af4a102fd8efb19acd6d369dc859a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:34:56 +0200 Subject: [PATCH 138/217] feat: mint run citation from capability tool --- .../app/capabilities/core/access/agent.py | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/capabilities/core/access/agent.py b/surfsense_backend/app/capabilities/core/access/agent.py index 4aa569e24..35c35774f 100644 --- a/surfsense_backend/app/capabilities/core/access/agent.py +++ b/surfsense_backend/app/capabilities/core/access/agent.py @@ -11,9 +11,13 @@ subagent can follow a truncation reference without extra wiring. from __future__ import annotations +import json import time +from langchain.tools import ToolRuntime +from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool, StructuredTool +from langgraph.types import Command from app.capabilities.core.billing import charge_capability, gate_capability from app.capabilities.core.progress import progress_scope @@ -64,7 +68,7 @@ def _capability_tool(capability: Capability, workspace_id: int) -> BaseTool: executor = capability.executor name = capability.name - async def _run(**kwargs: object) -> dict | str: + async def _run(runtime: ToolRuntime, **kwargs: object) -> dict | str | Command: payload = input_model(**kwargs) input_dump = payload.model_dump(exclude_none=True) thread_id = _current_thread_id() @@ -119,13 +123,39 @@ def _capability_tool(capability: Capability, workspace_id: int) -> BaseTool: progress=reporter.coarse, ) + # No stored run to cite: keep the legacy return shape, no citation. + if run_id is None: + if serialized.char_count <= RUN_OUTPUT_CHAR_CAP: + return output.model_dump(exclude_none=True) + return _build_preview(serialized, run_id) + + run_external_id = f"run_{run_id}" if serialized.char_count <= RUN_OUTPUT_CHAR_CAP: dump = output.model_dump(exclude_none=True) - if run_id is not None: - dump["run_id"] = f"run_{run_id}" - return dump + dump["run_id"] = run_external_id + content = json.dumps(dump, ensure_ascii=False, default=str) + else: + content = _build_preview(serialized, run_id) - return _build_preview(serialized, run_id) + # Deferred import: the citation spine imports from here; lazy avoids a cycle. + from app.agents.chat.multi_agent_chat.shared.citations import load_registry + from app.capabilities.core.access.run_citation import attach_run_citation + + registry = load_registry(getattr(runtime, "state", None)) + _, label = attach_run_citation( + registry, run_external_id=run_external_id, capability=name + ) + return Command( + update={ + "messages": [ + ToolMessage( + content=content + label, + tool_call_id=runtime.tool_call_id, + ) + ], + "citation_registry": registry, + } + ) return StructuredTool.from_function( coroutine=_run, From 95b4afddebcb72d6bdfd80212171c70504496032 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:34:56 +0200 Subject: [PATCH 139/217] feat: declare citation channel for subagents --- .../chat/multi_agent_chat/subagents/middleware_stack.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/middleware_stack.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/middleware_stack.py index 124ccf704..4181180d1 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/middleware_stack.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/middleware_stack.py @@ -15,6 +15,9 @@ from __future__ import annotations from typing import Any from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.middleware.citation_state import ( + build_citation_state_mw, +) from app.agents.chat.multi_agent_chat.shared.middleware.resilience import ( ResilienceMiddlewares, ) @@ -45,6 +48,8 @@ def build_subagent_middleware_stack( return { "todos": build_todos_mw(), "permission": permission, + # Declares the citation_registry channel so run [n]s merge up from tools. + "citation": build_citation_state_mw(), "retry": resilience.retry, "fallback": resilience.fallback, "model_call_limit": resilience.model_call_limit, From dd5d267056ef623a2a78381e4c7edc49e3a30735 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:34:56 +0200 Subject: [PATCH 140/217] docs: instruct agents to cite scraper runs --- .../main_agent/system_prompt/prompts/citations/on.md | 5 +++-- .../subagents/shared/snippets/output_contract_base.md | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md index 371cd1e57..effe5078c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md @@ -1,8 +1,9 @@ Cite with one token: the bracket label `[n]`. Every citable result — prose from a `task` knowledge_base/research specialist (including the -knowledge_base specialist's `[n]`-labelled workspace findings) — already -carries `[n]` labels on a single shared count. +knowledge_base specialist's `[n]`-labelled workspace findings) and +scraper specialists' run-backed findings — already carries `[n]` labels +on a single shared count. Those labels are the only citation you write; the server resolves each one back to its source after the turn. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md index 35fd48814..cd3b94400 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md @@ -5,3 +5,4 @@ Rules (universal): - `status=blocked` due to missing required inputs -> `missing_fields` must be non-null. - `assumptions`: any inferences you made about the user's intent; `null` when no inferences were needed. - The `evidence` object's fields are documented in your route-specific `` above; never invent fields the tool did not return. +- When a finding is drawn from a scraper run, append that run's `[n]` (the tool result states `Cite this scraper run as [n]`) to the finding text so the citation survives into the final answer. Copy the label exactly; never invent one. From 57d6182ca3f20be86d1c62d092e03abfdd82e44f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:35:04 +0200 Subject: [PATCH 141/217] feat: parse run citation tokens --- surfsense_web/lib/citations/citation-parser.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/surfsense_web/lib/citations/citation-parser.ts b/surfsense_web/lib/citations/citation-parser.ts index 533c644c2..3c2f0196c 100644 --- a/surfsense_web/lib/citations/citation-parser.ts +++ b/surfsense_web/lib/citations/citation-parser.ts @@ -11,18 +11,20 @@ import { FENCED_OR_INLINE_CODE } from "@/lib/markdown/code-regions"; /** * Matches `[citation:...]` with numeric IDs (incl. negative, doc- prefix, - * comma-separated), URL-based IDs from live web search, or `urlciteN` - * placeholders produced by `preprocessCitationMarkdown`. + * comma-separated), URL-based IDs from live web search, `urlciteN` + * placeholders produced by `preprocessCitationMarkdown`, or a scraper-run + * handle (`run_`). * * Also matches Chinese brackets 【】 and zero-width spaces that LLMs * sometimes emit. */ export const CITATION_REGEX = - /[[【]\u200B?citation:\s*(https?:\/\/[^\]】\u200B]+|urlcite\d+|(?:doc-)?-?\d+(?:\s*,\s*(?:doc-)?-?\d+)*)\s*\u200B?[\]】]/g; + /[[【]\u200B?citation:\s*(https?:\/\/[^\]】\u200B]+|urlcite\d+|run_[0-9a-fA-F-]+|(?:doc-)?-?\d+(?:\s*,\s*(?:doc-)?-?\d+)*)\s*\u200B?[\]】]/g; /** A single parsed citation reference. */ export type CitationToken = | { kind: "url"; url: string } + | { kind: "run"; runId: string } | { kind: "chunk"; chunkId: number; isDocsChunk: boolean }; /** Output of `parseTextWithCitations` — interleaved text + citation tokens. */ @@ -102,6 +104,8 @@ export function parseTextWithCitations(text: string, urlMap: CitationUrlMap): Pa if (url) { segments.push({ kind: "url", url }); } + } else if (captured.startsWith("run_")) { + segments.push({ kind: "run", runId: captured.trim() }); } else { const rawIds = captured.split(",").map((s) => s.trim()); for (const rawId of rawIds) { From 979ad02ef3006791b41230516f6ae71fb7091e76 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:35:04 +0200 Subject: [PATCH 142/217] feat: render run citation chip --- .../citations/citation-renderer.tsx | 4 +++ .../components/citations/run-citation.tsx | 31 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 surfsense_web/components/citations/run-citation.tsx diff --git a/surfsense_web/components/citations/citation-renderer.tsx b/surfsense_web/components/citations/citation-renderer.tsx index f2de4b27d..d3d78aa39 100644 --- a/surfsense_web/components/citations/citation-renderer.tsx +++ b/surfsense_web/components/citations/citation-renderer.tsx @@ -2,6 +2,7 @@ import type { ReactNode } from "react"; import { InlineCitation, UrlCitation } from "@/components/assistant-ui/inline-citation"; +import { RunCitation } from "@/components/citations/run-citation"; import { type CitationToken, type CitationUrlMap, @@ -21,6 +22,9 @@ export function renderCitationToken(token: CitationToken, ordinalKey: number): R if (token.kind === "url") { return ; } + if (token.kind === "run") { + return ; + } return ( = ({ runId }) => { + const openRunPanel = useSetAtom(openRunCitationPanelAtom); + + return ( + + + + + View scraper run + + ); +}; From b86eadb8474069d7756ea383e738a1e592c7313f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:35:04 +0200 Subject: [PATCH 143/217] feat: open runs in the citation panel --- .../atoms/citation/citation-panel.atom.ts | 26 ++++++---- .../citations/run-citation-panel.tsx | 47 +++++++++++++++++++ .../layout/ui/right-panel/RightPanel.tsx | 28 +++++++++-- 3 files changed, 88 insertions(+), 13 deletions(-) create mode 100644 surfsense_web/components/citations/run-citation-panel.tsx diff --git a/surfsense_web/atoms/citation/citation-panel.atom.ts b/surfsense_web/atoms/citation/citation-panel.atom.ts index ca7312857..f92fd6f90 100644 --- a/surfsense_web/atoms/citation/citation-panel.atom.ts +++ b/surfsense_web/atoms/citation/citation-panel.atom.ts @@ -1,14 +1,19 @@ -import { atom } from "jotai"; +import { atom, type Getter, type Setter } from "jotai"; import { rightPanelCollapsedAtom, rightPanelTabAtom } from "@/atoms/layout/right-panel.atom"; +/** The source the citation panel is showing: a KB chunk or a scraper run. */ +export type CitationTarget = + | { kind: "chunk"; chunkId: number } + | { kind: "run"; runId: string }; + interface CitationPanelState { isOpen: boolean; - chunkId: number | null; + target: CitationTarget | null; } const initialState: CitationPanelState = { isOpen: false, - chunkId: null, + target: null, }; export const citationPanelAtom = atom(initialState); @@ -17,16 +22,21 @@ export const citationPanelOpenAtom = atom((get) => get(citationPanelAtom).isOpen const preCitationCollapsedAtom = atom(null); -export const openCitationPanelAtom = atom(null, (get, set, payload: { chunkId: number }) => { +function openWithTarget(get: Getter, set: Setter, target: CitationTarget) { if (!get(citationPanelAtom).isOpen) { set(preCitationCollapsedAtom, get(rightPanelCollapsedAtom)); } - set(citationPanelAtom, { - isOpen: true, - chunkId: payload.chunkId, - }); + set(citationPanelAtom, { isOpen: true, target }); set(rightPanelTabAtom, "citation"); set(rightPanelCollapsedAtom, false); +} + +export const openCitationPanelAtom = atom(null, (get, set, payload: { chunkId: number }) => { + openWithTarget(get, set, { kind: "chunk", chunkId: payload.chunkId }); +}); + +export const openRunCitationPanelAtom = atom(null, (get, set, payload: { runId: string }) => { + openWithTarget(get, set, { kind: "run", runId: payload.runId }); }); export const closeCitationPanelAtom = atom(null, (get, set) => { diff --git a/surfsense_web/components/citations/run-citation-panel.tsx b/surfsense_web/components/citations/run-citation-panel.tsx new file mode 100644 index 000000000..3b7af7ca1 --- /dev/null +++ b/surfsense_web/components/citations/run-citation-panel.tsx @@ -0,0 +1,47 @@ +"use client"; + +import { XIcon } from "lucide-react"; +import { useParams } from "next/navigation"; +import type { FC } from "react"; +import { RunDetail } from "@/app/dashboard/[workspace_id]/playground/components/run-detail"; +import { Button } from "@/components/ui/button"; + +/** Right-panel viewer for a cited scraper run. `runId` is the `run_` handle. */ +export const RunCitationPanelContent: FC<{ runId: string; onClose?: () => void }> = ({ + runId, + onClose, +}) => { + const params = useParams(); + const rawWorkspaceId = Array.isArray(params?.workspace_id) + ? params.workspace_id[0] + : params?.workspace_id; + const workspaceId = Number(rawWorkspaceId); + const scraperRunId = runId.replace(/^run_/, ""); + + return ( + <> +
+

Scraper run

+ {onClose && ( + + )} +
+ +
+ {Number.isFinite(workspaceId) ? ( + + ) : ( +

Open a workspace to view this run.

+ )} +
+ + ); +}; diff --git a/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx b/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx index 15025f4eb..362e676d8 100644 --- a/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx +++ b/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx @@ -35,6 +35,14 @@ const CitationPanelContent = dynamic( { ssr: false, loading: () => null } ); +const RunCitationPanelContent = dynamic( + () => + import("@/components/citations/run-citation-panel").then((m) => ({ + default: m.RunCitationPanelContent, + })), + { ssr: false, loading: () => null } +); + const HitlEditPanelContent = dynamic( () => import("@/features/chat-messages/hitl").then((m) => ({ @@ -89,7 +97,7 @@ export function RightPanelToggleButton({ ? !!editorState.memoryScope : !!editorState.localFilePath); const hitlEditOpen = hitlEditState.isOpen && !!hitlEditState.onSave; - const citationOpen = citationState.isOpen && citationState.chunkId != null; + const citationOpen = citationState.isOpen && citationState.target != null; const hasContent = reportOpen || editorOpen || hitlEditOpen || citationOpen || artifactsOpen; const label = collapsed ? "Expand panel" : "Collapse panel"; @@ -141,7 +149,7 @@ export function RightPanelExpandButton() { ? !!editorState.memoryScope : !!editorState.localFilePath); const hitlEditOpen = hitlEditState.isOpen && !!hitlEditState.onSave; - const citationOpen = citationState.isOpen && citationState.chunkId != null; + const citationOpen = citationState.isOpen && citationState.target != null; const hasContent = reportOpen || editorOpen || hitlEditOpen || citationOpen || artifactsOpen; if (!collapsed || !hasContent) return null; @@ -212,7 +220,7 @@ export function RightPanel({ showTopBorder = false }: RightPanelProps) { ? !!editorState.memoryScope : !!editorState.localFilePath); const hitlEditOpen = hitlEditState.isOpen && !!hitlEditState.onSave; - const citationOpen = citationState.isOpen && citationState.chunkId != null; + const citationOpen = citationState.isOpen && citationState.target != null; useEffect(() => { if (!reportOpen && !editorOpen && !hitlEditOpen && !citationOpen && !artifactsOpen) return; @@ -306,9 +314,19 @@ export function RightPanel({ showTopBorder = false }: RightPanelProps) { />
)} - {effectiveTab === "citation" && citationOpen && citationState.chunkId != null && ( + {effectiveTab === "citation" && citationOpen && citationState.target && (
- + {citationState.target.kind === "run" ? ( + + ) : ( + + )}
)} {effectiveTab === "artifacts" && artifactsOpen && ( From 120eb1f31b155e9cd2d27bec71b503ab8c8644a6 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:35:04 +0200 Subject: [PATCH 144/217] test: cover run citation registration --- .../shared/citations/test_markers.py | 12 ++++++ .../capabilities/access/test_run_citation.py | 43 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 surfsense_backend/tests/unit/capabilities/access/test_run_citation.py diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/shared/citations/test_markers.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/shared/citations/test_markers.py index 53cf058a8..5710bc7a6 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/shared/citations/test_markers.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/shared/citations/test_markers.py @@ -37,6 +37,18 @@ def test_web_result_maps_to_url() -> None: assert to_frontend_payload(entry) == "https://example.com/a" +def test_run_maps_to_run_handle() -> None: + entry = _entry(CitationSourceType.RUN, {"run_id": "run_abc-123"}) + + assert to_frontend_payload(entry) == "run_abc-123" + + +def test_run_without_handle_is_dropped() -> None: + entry = _entry(CitationSourceType.RUN, {}) + + assert to_frontend_payload(entry) is None + + def test_not_yet_renderable_kind_is_dropped() -> None: entry = _entry(CitationSourceType.CHAT_TURN, {"thread_id": 1, "turn": 2}) diff --git a/surfsense_backend/tests/unit/capabilities/access/test_run_citation.py b/surfsense_backend/tests/unit/capabilities/access/test_run_citation.py new file mode 100644 index 000000000..6da9e5f29 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/access/test_run_citation.py @@ -0,0 +1,43 @@ +"""Unit tests for registering a scraper run as a citation.""" + +from __future__ import annotations + +import pytest + +from app.agents.chat.multi_agent_chat.shared.citations import ( + CitationRegistry, + CitationSourceType, + to_frontend_payload, +) +from app.capabilities.core.access.run_citation import attach_run_citation + +pytestmark = pytest.mark.unit + + +def test_attaches_run_and_returns_label_with_ordinal() -> None: + registry = CitationRegistry() + + n, label = attach_run_citation( + registry, run_external_id="run_abc-123", capability="walmart.scrape" + ) + + assert n == 1 + assert f"[{n}]" in label + entry = registry.resolve(n) + assert entry is not None + assert entry.source_type is CitationSourceType.RUN + assert to_frontend_payload(entry) == "run_abc-123" + + +def test_same_run_dedups_to_one_label() -> None: + registry = CitationRegistry() + + first, _ = attach_run_citation( + registry, run_external_id="run_x", capability="walmart.scrape" + ) + again, _ = attach_run_citation( + registry, run_external_id="run_x", capability="walmart.reviews" + ) + + assert first == again + assert len(registry.by_n) == 1 From d356605a0dccc96dca1ca84f55248e6f58a39b63 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:35:04 +0200 Subject: [PATCH 145/217] test: capability tool mints run citation --- .../capabilities/access/test_agent_tools.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py b/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py index cdab68bc0..fb8f0c540 100644 --- a/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py +++ b/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py @@ -69,6 +69,12 @@ def _verb_tool(tools, name: str): return next(t for t in tools if t.name == name) +def _invoke(tool, text: str, *, state=None): + """Call the coroutine with a stand-in runtime (ToolNode injects it in prod).""" + runtime = SimpleNamespace(state=state or {}, tool_call_id="tc_1", context=None) + return tool.coroutine(runtime, text=text) + + async def test_registry_becomes_one_tool_per_verb_plus_readers(isolate): caps = [ _capability(name="web.scrape", output=_EchoOutput(echoed="a")), @@ -104,20 +110,40 @@ async def test_tool_runs_executor_and_returns_serialized_output(isolate): tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap]) tool = _verb_tool(tools, "web_scrape") - result = await tool.ainvoke({"text": "ping"}) + result = await _invoke(tool, "ping") # Fake session makes record_run fail -> no run_id key, plain serialized output. assert result == {"echoed": "hi there"} assert cap.executor.seen.text == "ping" +async def test_tool_registers_run_citation_when_stored(isolate, monkeypatch): + from langgraph.types import Command + + cap = _capability(name="web.scrape", output=_EchoOutput(echoed="hi")) + monkeypatch.setattr(isolate.module, "record_run", AsyncMock(return_value="abc-123")) + tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap]) + tool = _verb_tool(tools, "web_scrape") + + result = await _invoke(tool, "ping") + + assert isinstance(result, Command) + registry = result.update["citation_registry"] + entry = registry.resolve(1) + assert entry is not None + assert entry.locator["run_id"] == "run_abc-123" + message = result.update["messages"][0] + assert "[1]" in message.content + assert "run_abc-123" in message.content + + async def test_tool_charges_owner(isolate): output = _EchoOutput(echoed="hi") cap = _capability(name="web.scrape", output=output) tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap]) tool = _verb_tool(tools, "web_scrape") - await tool.ainvoke({"text": "ping"}) + await _invoke(tool, "ping") isolate.charge.assert_awaited_once() (charged_output, unit, ctx), _ = isolate.charge.call_args @@ -136,7 +162,7 @@ async def test_over_budget_returns_friendly_message(isolate): tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap]) tool = _verb_tool(tools, "web_scrape") - result = await tool.ainvoke({"text": "ping"}) + result = await _invoke(tool, "ping") assert isinstance(result, str) assert "credit" in result.lower() From 6c8d109d761efe91728bc58ade166bc0ef84423d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 19:35:04 +0200 Subject: [PATCH 146/217] test: parse run citation tokens --- .../lib/citations/citation-parser.test.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 surfsense_web/lib/citations/citation-parser.test.ts diff --git a/surfsense_web/lib/citations/citation-parser.test.ts b/surfsense_web/lib/citations/citation-parser.test.ts new file mode 100644 index 000000000..1f5749f2f --- /dev/null +++ b/surfsense_web/lib/citations/citation-parser.test.ts @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { parseTextWithCitations } from "./citation-parser"; + +// Run with: pnpm exec tsx --test lib/citations/citation-parser.test.ts + +const NO_URLS = new Map(); + +test("parses a scraper-run handle into a run token", () => { + const segments = parseTextWithCitations( + "Price is $19 [citation:run_1b2c3d4e-5f60-7081-9abc-def012345678].", + NO_URLS + ); + + assert.deepEqual(segments, [ + "Price is $19 ", + { kind: "run", runId: "run_1b2c3d4e-5f60-7081-9abc-def012345678" }, + ".", + ]); +}); + +test("run handles do not collide with numeric chunk citations", () => { + const segments = parseTextWithCitations( + "chunk [citation:42] vs run [citation:run_ab-12].", + NO_URLS + ); + + assert.deepEqual(segments, [ + "chunk ", + { kind: "chunk", chunkId: 42, isDocsChunk: false }, + " vs run ", + { kind: "run", runId: "run_ab-12" }, + ".", + ]); +}); From 950de5e670d919b7ee7076821de09772b7d7fb11 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 20:06:18 +0200 Subject: [PATCH 147/217] fix: forward ToolRuntime to capability tools --- surfsense_backend/app/capabilities/core/access/agent.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/surfsense_backend/app/capabilities/core/access/agent.py b/surfsense_backend/app/capabilities/core/access/agent.py index 35c35774f..3d3864a53 100644 --- a/surfsense_backend/app/capabilities/core/access/agent.py +++ b/surfsense_backend/app/capabilities/core/access/agent.py @@ -157,6 +157,9 @@ def _capability_tool(capability: Capability, workspace_id: int) -> BaseTool: } ) + # Un-stringify for StructuredTool's signature-based runtime injection. + _run.__annotations__["runtime"] = ToolRuntime + return StructuredTool.from_function( coroutine=_run, name=name.replace(".", "_"), From 47d35faeedf91dca7dbb5416f1af6ea7f8873548 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 20:06:18 +0200 Subject: [PATCH 148/217] test: guard capability tool runtime injection --- .../unit/capabilities/access/test_agent_tools.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py b/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py index fb8f0c540..3d4bed199 100644 --- a/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py +++ b/surfsense_backend/tests/unit/capabilities/access/test_agent_tools.py @@ -137,6 +137,17 @@ async def test_tool_registers_run_citation_when_stored(isolate, monkeypatch): assert "run_abc-123" in message.content +async def test_runtime_survives_langchain_arg_parsing(isolate): + """runtime must survive langchain arg parsing (else ToolNode drops it).""" + cap = _capability(name="web.scrape", output=_EchoOutput(echoed="hi")) + tools = isolate.module.build_capability_tools(workspace_id=7, capabilities=[cap]) + tool = _verb_tool(tools, "web_scrape") + + parsed = tool._parse_input({"text": "x", "runtime": "RT"}, "tc_1") + + assert parsed["runtime"] == "RT" + + async def test_tool_charges_owner(isolate): output = _EchoOutput(echoed="hi") cap = _capability(name="web.scrape", output=output) From 3f23f436fb57180a05b1d93876f37790649dc853 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 22 Jul 2026 21:01:05 +0200 Subject: [PATCH 149/217] refactor(citations): rename run chip to Source for non-technical users --- surfsense_web/components/citations/run-citation.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/surfsense_web/components/citations/run-citation.tsx b/surfsense_web/components/citations/run-citation.tsx index ef388373b..d8799a24c 100644 --- a/surfsense_web/components/citations/run-citation.tsx +++ b/surfsense_web/components/citations/run-citation.tsx @@ -19,13 +19,13 @@ export const RunCitation: FC<{ runId: string }> = ({ runId }) => { variant="ghost" onClick={() => openRunPanel({ runId })} className="ml-0.5 inline-flex h-5 min-w-5 items-center justify-center gap-0.5 rounded-md bg-popover px-1.5 text-[11px] font-medium text-popover-foreground/80 align-baseline" - aria-label="View scraper run" + aria-label="See where this came from" > - run + Source - View scraper run + See where this came from ); }; From dbedf0cfa53e604e4b9bc3f26f29691b691ddeb4 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 22 Jul 2026 22:16:28 -0700 Subject: [PATCH 150/217] feat(backend): integrate PostHog analytics for enhanced observability - Added PostHog configuration options to .env.example files for both Docker and Surfsense backend. - Introduced PostHog dependency in pyproject.toml. - Implemented analytics middleware to capture various events across the application, including user authentication, automation runs, and API requests. - Enhanced existing routes and services to emit analytics events, providing insights into user interactions and system performance. - Ensured graceful shutdown of analytics clients in worker processes and application lifecycles. --- docker/.env.example | 9 + surfsense_backend/.env.example | 8 + surfsense_backend/app/app.py | 52 +++ .../app/automations/runtime/executor.py | 30 ++ .../app/automations/services/automation.py | 47 +++ .../app/automations/services/trigger.py | 40 ++ .../app/capabilities/core/access/rest.py | 92 ++++- surfsense_backend/app/celery_app.py | 13 + surfsense_backend/app/config/__init__.py | 13 + .../app/observability/__init__.py | 2 +- .../app/observability/analytics.py | 202 ++++++++++ .../app/podcasts/tasks/render.py | 27 ++ .../app/routes/anonymous_chat_routes.py | 42 +++ .../app/routes/image_generation_routes.py | 15 + .../app/routes/incentive_tasks_routes.py | 15 + .../app/routes/new_chat_routes.py | 10 + .../routes/personal_access_tokens_routes.py | 10 + .../app/routes/public_chat_routes.py | 16 +- surfsense_backend/app/routes/rbac_routes.py | 24 ++ .../routes/search_source_connectors_routes.py | 58 +++ surfsense_backend/app/routes/stripe_routes.py | 24 ++ .../app/routes/workspaces_routes.py | 11 + .../app/tasks/celery_tasks/connector_tasks.py | 83 ++++- .../app/tasks/celery_tasks/document_tasks.py | 98 ++++- .../celery_tasks/video_presentation_tasks.py | 15 + .../streaming/flows/new_chat/orchestrator.py | 42 +++ .../flows/resume_chat/orchestrator.py | 39 ++ .../chat/streaming/flows/shared/analytics.py | 117 ++++++ surfsense_backend/app/users.py | 28 +- surfsense_backend/pyproject.toml | 1 + .../unit/observability/test_analytics.py | 149 ++++++++ surfsense_backend/uv.lock | 158 ++++++++ surfsense_mcp/mcp_server/core/client.py | 6 +- .../app/(home)/login/LocalLoginForm.tsx | 6 +- surfsense_web/app/(home)/register/page.tsx | 10 +- .../[workspace_id]/team/team-content.tsx | 23 +- .../app/invite/[invite_code]/page.tsx | 14 +- .../automations/automations-mutation.atoms.ts | 73 +--- .../hooks/use-connector-dialog.ts | 28 +- .../ui/dialogs/CreateWorkspaceDialog.tsx | 4 +- .../settings/earn-credits-content.tsx | 19 +- .../components/sources/DocumentUploadTab.tsx | 14 +- surfsense_web/content/docs/observability.mdx | 34 ++ surfsense_web/hooks/use-run-stream.ts | 4 +- surfsense_web/lib/apis/base-api.service.ts | 51 ++- .../lib/chat/stream-engine/engine.ts | 4 +- surfsense_web/lib/posthog/events.ts | 351 ++---------------- 47 files changed, 1618 insertions(+), 513 deletions(-) create mode 100644 surfsense_backend/app/observability/analytics.py create mode 100644 surfsense_backend/app/tasks/chat/streaming/flows/shared/analytics.py create mode 100644 surfsense_backend/tests/unit/observability/test_analytics.py diff --git a/docker/.env.example b/docker/.env.example index 78c0bd93a..fb7f5ff42 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -392,6 +392,15 @@ STT_SERVICE=local/base # OTEL_HTTP_PORT=4318 # OTEL_HEALTH_PORT=13133 +# PostHog product analytics (server-side). Opt-in like OTel: leave +# POSTHOG_API_KEY unset for zero telemetry. Passed to backend/worker/beat via +# env_file, so no compose changes are needed. Use the SAME project key as the +# frontend's NEXT_PUBLIC_POSTHOG_KEY so server events merge onto the persons the +# web app already identifies by user id. +# POSTHOG_API_KEY=phc_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +# POSTHOG_HOST=https://us.i.posthog.com +# POSTHOG_AI_PRIVACY_MODE=true # false ships LLM prompt/completion bodies to PostHog + # ------------------------------------------------------------------------------ # Advanced (optional) # ------------------------------------------------------------------------------ diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 266985e6c..1d60c98c3 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -555,6 +555,14 @@ LANGSMITH_PROJECT=surfsense # OTEL_METRIC_EXPORT_INTERVAL=300000 # ms; 5 minutes # OTEL_SDK_DISABLED=true # emergency kill-switch +# Observability - PostHog product analytics (server-side) +# Opt-in like OTel: leave POSTHOG_API_KEY unset for zero telemetry. Use the +# SAME project key as the frontend's NEXT_PUBLIC_POSTHOG_KEY so server events +# merge onto the persons the web app already identifies by user id. +# POSTHOG_API_KEY=phc_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +# POSTHOG_HOST=https://us.i.posthog.com +# POSTHOG_AI_PRIVACY_MODE=true # false ships LLM prompt/completion bodies to PostHog + # Skills + subagents # SURFSENSE_ENABLE_SKILLS=false # SURFSENSE_ENABLE_SPECIALIZED_SUBAGENTS=false diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index dde2ce7fa..42efe38a7 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -50,6 +50,7 @@ from app.gateway.inbox_worker import ( start_gateway_inbox_worker, stop_gateway_inbox_worker, ) +from app.observability import analytics as ph_analytics from app.observability import metrics as ot_metrics from app.observability.bootstrap import init_otel, shutdown_otel from app.rate_limiter import get_real_client_ip, limiter @@ -690,6 +691,7 @@ async def lifespan(app: FastAPI): await stop_gateway_inbox_worker() _stop_openrouter_background_refresh() await close_checkpointer() + ph_analytics.shutdown() shutdown_otel() @@ -796,6 +798,56 @@ class RequestPerfMiddleware(BaseHTTPMiddleware): app.add_middleware(RequestPerfMiddleware) + +# --------------------------------------------------------------------------- +# PAT / MCP API attribution middleware +# --------------------------------------------------------------------------- +# Emits a PostHog ``pat_api_request`` event for any request authenticated by a +# Personal Access Token, so "documents added via MCP", "searches via MCP" etc. +# are queryable without instrumenting each route. Relies on ``get_auth_context`` +# stashing the resolved principal on ``request.state.auth_context``; requests +# that never resolve a PAT principal are silently skipped. No-op when PostHog +# is unconfigured. + + +class PatApiAnalyticsMiddleware(BaseHTTPMiddleware): + """Capture PAT-authenticated API usage (incl. MCP) after each response.""" + + async def dispatch( + self, request: StarletteRequest, call_next: RequestResponseEndpoint + ) -> StarletteResponse: + response = await call_next(request) + with contextlib.suppress(Exception): + ctx = getattr(request.state, "auth_context", None) + if ( + ctx is not None + and ctx.method == "pat" + and ph_analytics.is_enabled() + ): + # Use the route *template* (e.g. /documents/{id}) to keep the + # ``route`` property low-cardinality; fall back to the raw path. + route = request.scope.get("route") + route_path = getattr(route, "path", None) or request.url.path + client = ( + "mcp" + if request.headers.get("X-SurfSense-Client") == "mcp" + else "pat_script" + ) + ph_analytics.capture_for( + ctx, + "pat_api_request", + { + "route": route_path, + "method": request.method, + "status_code": response.status_code, + "client": client, + }, + ) + return response + + +app.add_middleware(PatApiAnalyticsMiddleware) + # Add SlowAPI middleware for automatic rate limiting # Uses Starlette BaseHTTPMiddleware (not the raw ASGI variant) to avoid # corrupting StreamingResponse — SlowAPIASGIMiddleware re-sends diff --git a/surfsense_backend/app/automations/runtime/executor.py b/surfsense_backend/app/automations/runtime/executor.py index d9544cd0c..cbf682d60 100644 --- a/surfsense_backend/app/automations/runtime/executor.py +++ b/surfsense_backend/app/automations/runtime/executor.py @@ -15,11 +15,38 @@ from app.automations.schemas.definition.envelope import ( ) from app.automations.schemas.definition.plan_step import PlanStep from app.automations.templating import build_run_context +from app.observability import analytics as ph_analytics from . import repository from .step import execute_step +def _capture_run_outcome(run: AutomationRun, status: str) -> None: + """Emit ``automation_run_completed`` — headless runs the frontend never sees. + + No-op when PostHog is unconfigured or the owning user is unknown. + """ + automation = run.automation + creator_id = getattr(automation, "created_by_user_id", None) + if not creator_id: + return + workspace_id = getattr(automation, "workspace_id", None) + ph_analytics.capture( + "automation_run_completed", + distinct_id=str(creator_id), + properties={ + "automation_id": run.automation_id, + "run_id": run.id, + "workspace_id": workspace_id, + "status": status, + "trigger_type": run.trigger.type.value if run.trigger else None, + }, + groups={"workspace": str(workspace_id)} + if workspace_id is not None + else None, + ) + + async def execute_run(session: AsyncSession, run_id: int) -> None: """Load run ``run_id`` and execute its snapshot plan to a terminal state.""" run = await repository.load_run(session, run_id) @@ -41,6 +68,7 @@ async def execute_run(session: AsyncSession, run_id: int) -> None: }, ) await session.commit() + _capture_run_outcome(run, "failed") return await repository.mark_running(session, run) @@ -66,6 +94,7 @@ async def execute_run(session: AsyncSession, run_id: int) -> None: await _run_on_failure(session, run, definition) await repository.mark_failed(session, run, result.get("error")) await session.commit() + _capture_run_outcome(run, "failed") return if result["status"] == "succeeded": @@ -73,6 +102,7 @@ async def execute_run(session: AsyncSession, run_id: int) -> None: await repository.mark_succeeded(session, run) await session.commit() + _capture_run_outcome(run, "succeeded") async def _run_on_failure( diff --git a/surfsense_backend/app/automations/services/automation.py b/surfsense_backend/app/automations/services/automation.py index a419327bf..a7bbd3af3 100644 --- a/surfsense_backend/app/automations/services/automation.py +++ b/surfsense_backend/app/automations/services/automation.py @@ -29,6 +29,7 @@ from app.automations.services.model_policy import ( from app.automations.triggers import get_trigger from app.automations.triggers.builtin.schedule import compute_next_fire_at from app.db import Permission, Workspace, get_async_session +from app.observability import analytics as ph_analytics from app.users import get_auth_context from app.utils.rbac import check_permission @@ -75,6 +76,18 @@ class AutomationService: self.session.add(automation) await self.session.commit() + + # Authoritative creation (migrated from automations-mutation.atoms.ts). + ph_analytics.capture_for( + self.auth, + "automation_created", + { + "automation_id": automation.id, + "workspace_id": automation.workspace_id, + "trigger_count": len(payload.triggers), + }, + groups={"workspace": str(automation.workspace_id)}, + ) return await self._get_with_triggers_or_raise(automation.id) async def list( @@ -150,6 +163,31 @@ class AutomationService: automation.version += 1 await self.session.commit() + + # Migrated from automations-mutation.atoms.ts: a status-only change is a + # distinct event; other field edits are ``automation_updated``. + if "status" in data: + ph_analytics.capture_for( + self.auth, + "automation_status_changed", + { + "automation_id": automation.id, + "workspace_id": automation.workspace_id, + "next_status": str(data["status"]), + }, + groups={"workspace": str(automation.workspace_id)}, + ) + if any(k in data for k in ("name", "description", "definition")): + ph_analytics.capture_for( + self.auth, + "automation_updated", + { + "automation_id": automation.id, + "workspace_id": automation.workspace_id, + "has_definition_change": "definition" in data, + }, + groups={"workspace": str(automation.workspace_id)}, + ) return await self._get_with_triggers_or_raise(automation_id) async def delete(self, automation_id: int) -> None: @@ -158,9 +196,18 @@ class AutomationService: await self._authorize( automation.workspace_id, Permission.AUTOMATIONS_DELETE.value ) + workspace_id = automation.workspace_id await self.session.delete(automation) await self.session.commit() + # Authoritative deletion (migrated from automations-mutation.atoms.ts). + ph_analytics.capture_for( + self.auth, + "automation_deleted", + {"automation_id": automation_id, "workspace_id": workspace_id}, + groups={"workspace": str(workspace_id)}, + ) + async def _get_or_raise(self, automation_id: int) -> Automation: automation = await self.session.get(Automation, automation_id) if automation is None: diff --git a/surfsense_backend/app/automations/services/trigger.py b/surfsense_backend/app/automations/services/trigger.py index 5175eb15e..866e66395 100644 --- a/surfsense_backend/app/automations/services/trigger.py +++ b/surfsense_backend/app/automations/services/trigger.py @@ -16,6 +16,7 @@ from app.automations.schemas.api import TriggerCreate, TriggerUpdate from app.automations.triggers import get_trigger from app.automations.triggers.builtin.schedule import compute_next_fire_at from app.db import Permission, get_async_session +from app.observability import analytics as ph_analytics from app.users import get_auth_context from app.utils.rbac import check_permission @@ -48,6 +49,18 @@ class TriggerService: self.session.add(trigger) await self.session.commit() await self.session.refresh(trigger) + + # Migrated from automations-mutation.atoms.ts. + ph_analytics.capture_for( + self.auth, + "automation_trigger_added", + { + "automation_id": automation_id, + "trigger_id": trigger.id, + "trigger_type": getattr(trigger.type, "value", str(trigger.type)), + "enabled": trigger.enabled, + }, + ) return trigger async def update( @@ -82,6 +95,26 @@ class TriggerService: await self.session.commit() await self.session.refresh(trigger) + + # Migrated from automations-mutation.atoms.ts. ``change`` mirrors the + # frontend's coarse categorisation. + _change = ( + "enabled" + if "enabled" in data and "params" not in data + else "params" + if "params" in data + else "other" + ) + ph_analytics.capture_for( + self.auth, + "automation_trigger_updated", + { + "automation_id": automation_id, + "trigger_id": trigger_id, + "change": _change, + "enabled": trigger.enabled, + }, + ) return trigger async def remove(self, *, automation_id: int, trigger_id: int) -> None: @@ -92,6 +125,13 @@ class TriggerService: await self.session.delete(trigger) await self.session.commit() + # Migrated from automations-mutation.atoms.ts. + ph_analytics.capture_for( + self.auth, + "automation_trigger_removed", + {"automation_id": automation_id, "trigger_id": trigger_id}, + ) + async def _authorize_automation( self, automation_id: int, permission: str ) -> Automation: diff --git a/surfsense_backend/app/capabilities/core/access/rest.py b/surfsense_backend/app/capabilities/core/access/rest.py index 2047b21ae..26ac46318 100644 --- a/surfsense_backend/app/capabilities/core/access/rest.py +++ b/surfsense_backend/app/capabilities/core/access/rest.py @@ -45,6 +45,7 @@ from app.capabilities.core.store import all_capabilities from app.capabilities.core.types import Capability, CapabilityContext from app.db import Run, async_session_maker, get_async_session from app.exceptions import ExternalServiceError, SurfSenseError +from app.observability import analytics as ph_analytics from app.services.web_crawl_credit_service import InsufficientCreditsError from app.users import get_auth_context from app.utils.rbac import check_workspace_access @@ -107,6 +108,41 @@ def _origin_for(auth: AuthContext) -> str: return "ui" if getattr(auth, "method", None) == "session" else "api" +def _capture_scraper_run( + *, + capability: str, + status: str, + user_id, + origin: str, + duration_ms: int | None = None, + item_count: int | None = None, + cost_micros: int | None = None, +) -> None: + """Emit ``scraper_run_completed`` — scrapers are the highest-value MCP surface. + + Covers both sync and async runs; the async background task never flows + through the PAT middleware, so this is the only place its outcome is + captured. No-op when PostHog is unconfigured. + """ + if not ph_analytics.is_enabled() or not user_id: + return + platform, _, verb = capability.partition(".") + ph_analytics.capture( + "scraper_run_completed", + distinct_id=str(user_id), + properties={ + "platform": platform, + "verb": verb, + "capability": capability, + "status": status, + "origin": origin, + "duration_ms": duration_ms, + "item_count": item_count, + "cost_micros": cost_micros, + }, + ) + + def _now_ms() -> int: return int(time.time() * 1000) @@ -185,6 +221,8 @@ async def _execute_async_run( unit, executor, payload, + user_id=None, + origin: str = "api", ) -> None: """Run a scrape in the background: stream progress, charge, finalize the row. @@ -218,6 +256,13 @@ async def _execute_async_run( progress=reporter.coarse, ) _publish_finished(run_id, "error", error=str(exc)) + _capture_scraper_run( + capability=capability, + status="error", + user_id=user_id, + origin=origin, + duration_ms=int((time.perf_counter() - started) * 1000), + ) return except Exception: logger.exception("async run %s failed with an upstream error", run_id) @@ -229,6 +274,13 @@ async def _execute_async_run( progress=reporter.coarse, ) _publish_finished(run_id, "error", error="upstream error") + _capture_scraper_run( + capability=capability, + status="error", + user_id=user_id, + origin=origin, + duration_ms=int((time.perf_counter() - started) * 1000), + ) return duration_ms = int((time.perf_counter() - started) * 1000) @@ -252,6 +304,15 @@ async def _execute_async_run( progress=reporter.coarse, ) _publish_finished(run_id, "success", item_count=serialized.item_count) + _capture_scraper_run( + capability=capability, + status="success", + user_id=user_id, + origin=origin, + duration_ms=duration_ms, + item_count=serialized.item_count, + cost_micros=cost_micros, + ) async def _finalize_async( @@ -358,6 +419,8 @@ def _register_verb(router: APIRouter, capability: Capability) -> None: unit=unit, executor=executor, payload=payload, + user_id=user_id, + origin=origin, ) ) run_event_bus.register_task(run_id, task) @@ -373,6 +436,7 @@ def _register_verb(router: APIRouter, capability: Capability) -> None: try: output = await executor(payload) except (SurfSenseError, HTTPException) as exc: + _sync_err_duration = int((time.perf_counter() - started) * 1000) await _record_rest_run( workspace_id=workspace_id, capability=name, @@ -381,11 +445,19 @@ def _register_verb(router: APIRouter, capability: Capability) -> None: input=input_dump, user_id=user_id, error=str(exc), - duration_ms=int((time.perf_counter() - started) * 1000), + duration_ms=_sync_err_duration, progress=reporter.coarse, ) + _capture_scraper_run( + capability=name, + status="error", + user_id=user_id, + origin=origin, + duration_ms=_sync_err_duration, + ) raise except Exception as exc: + _sync_err_duration = int((time.perf_counter() - started) * 1000) await _record_rest_run( workspace_id=workspace_id, capability=name, @@ -394,9 +466,16 @@ def _register_verb(router: APIRouter, capability: Capability) -> None: input=input_dump, user_id=user_id, error=str(exc), - duration_ms=int((time.perf_counter() - started) * 1000), + duration_ms=_sync_err_duration, progress=reporter.coarse, ) + _capture_scraper_run( + capability=name, + status="error", + user_id=user_id, + origin=origin, + duration_ms=_sync_err_duration, + ) raise ExternalServiceError( f"The '{name}' capability failed due to an upstream error.", code="CAPABILITY_UPSTREAM_ERROR", @@ -418,6 +497,15 @@ def _register_verb(router: APIRouter, capability: Capability) -> None: cost_micros=cost_micros, progress=reporter.coarse, ) + _capture_scraper_run( + capability=name, + status="success", + user_id=user_id, + origin=origin, + duration_ms=duration_ms, + item_count=serialized.item_count, + cost_micros=cost_micros, + ) if run_id is not None: response.headers["X-Run-Id"] = f"run_{run_id}" return output diff --git a/surfsense_backend/app/celery_app.py b/surfsense_backend/app/celery_app.py index 471d80197..2a9cddbd5 100644 --- a/surfsense_backend/app/celery_app.py +++ b/surfsense_backend/app/celery_app.py @@ -10,6 +10,7 @@ from celery.signals import ( task_postrun, task_prerun, worker_process_init, + worker_process_shutdown, ) from dotenv import load_dotenv @@ -123,6 +124,18 @@ def init_worker(**kwargs): initialize_image_gen_router() +@worker_process_shutdown.connect +def shutdown_worker(**kwargs): + """Flush queued PostHog events before a Celery worker process exits. + + The analytics client init is lazy (fork-safe), so there is nothing to + start here — only a flush to avoid dropping events captured by tasks. + """ + from app.observability import analytics as ph_analytics + + ph_analytics.shutdown() + + # Celery configuration, sourced from the central Config singleton CELERY_BROKER_URL = config.CELERY_BROKER_URL CELERY_RESULT_BACKEND = config.CELERY_RESULT_BACKEND diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 73062775c..45ef065df 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -1179,6 +1179,19 @@ class Config: os.getenv("CRAWL_HEADED_XVFB_ENABLED", "FALSE").upper() == "TRUE" ) + # PostHog server-side product analytics (opt-in, mirrors the OTel pattern: + # no key set => the analytics wrapper is a silent no-op). Use the SAME + # project key as the frontend's NEXT_PUBLIC_POSTHOG_KEY so server events + # merge onto the persons the web app already identifies by user id. + POSTHOG_API_KEY = os.getenv("POSTHOG_API_KEY") + POSTHOG_HOST = os.getenv("POSTHOG_HOST", "https://us.i.posthog.com") + # When true (default), the LLM-analytics LangChain handler suppresses + # prompt/completion bodies ($ai_input / $ai_output_choices) and captures + # only metrics — chat content includes users' private documents. + POSTHOG_AI_PRIVACY_MODE = ( + os.getenv("POSTHOG_AI_PRIVACY_MODE", "TRUE").upper() == "TRUE" + ) + # Litellm TTS Configuration TTS_SERVICE = os.getenv("TTS_SERVICE") TTS_SERVICE_API_BASE = os.getenv("TTS_SERVICE_API_BASE") diff --git a/surfsense_backend/app/observability/__init__.py b/surfsense_backend/app/observability/__init__.py index a675b1dae..4ec9f1b03 100644 --- a/surfsense_backend/app/observability/__init__.py +++ b/surfsense_backend/app/observability/__init__.py @@ -6,4 +6,4 @@ wrapper is a no-op when OTEL is not configured, so importing it from performance-critical paths is safe. """ -__all__ = ["bootstrap", "metrics", "otel"] +__all__ = ["analytics", "bootstrap", "metrics", "otel"] diff --git a/surfsense_backend/app/observability/analytics.py b/surfsense_backend/app/observability/analytics.py new file mode 100644 index 000000000..312335d24 --- /dev/null +++ b/surfsense_backend/app/observability/analytics.py @@ -0,0 +1,202 @@ +"""Server-side PostHog product analytics for SurfSense. + +Opt-in, mirroring the OpenTelemetry bootstrap contract: when +``POSTHOG_API_KEY`` is unset every function here is a silent no-op, so it is +safe to call from hot paths (including async request handlers) and from +self-hosted installs that never configure telemetry. + +Design notes: +- The underlying ``posthog`` client enqueues events onto a background + consumer thread, so ``capture()`` is a non-blocking queue append; the only + network I/O happens off-thread. ``shutdown()`` flushes and joins that thread + and MUST run before a process exits or queued events are lost. +- The client is created lazily on first use, never at import time. This keeps + it fork-safe under Celery's prefork pool: a client (and its consumer thread) + created in the parent would not survive ``fork()``, so each worker process + builds its own on first capture. +- ``distinct_id`` is always ``str(user.id)`` so server events merge onto the + same PostHog persons the web frontend identifies (see + ``surfsense_web/components/providers/PostHogIdentify.tsx``). +- Every event passes ``disable_geoip=True``; without it PostHog would resolve + the *server's* IP and overwrite each person's real (client-derived) location. +""" + +from __future__ import annotations + +import logging +import threading +from typing import TYPE_CHECKING, Any + +from app.config import config + +if TYPE_CHECKING: + from app.auth.context import AuthContext + +logger = logging.getLogger(__name__) + +_client: Any | None = None +_init_attempted = False +_lock = threading.Lock() + +# Stamped on every backend event so client-observed (frontend) and +# server-truth events are always distinguishable in PostHog. +_SOURCE = "backend" + + +def _get_client() -> Any | None: + """Return the process-local PostHog client, or ``None`` when disabled. + + Lazy + fork-safe: built on first use inside whichever process (web worker + or Celery worker) calls it, never at import time. + """ + global _client, _init_attempted + + if _init_attempted: + return _client + + with _lock: + if _init_attempted: + return _client + _init_attempted = True + + api_key = config.POSTHOG_API_KEY + if not api_key: + # ponytail: opt-in like OTel — no key means telemetry is off, not + # a misconfiguration. Stay silent so self-hosters see no noise. + return None + + try: + from posthog import Posthog + + _client = Posthog( + project_api_key=api_key, + host=config.POSTHOG_HOST, + ) + except Exception: + logger.warning("PostHog analytics init failed; disabling", exc_info=True) + _client = None + + return _client + + +def is_enabled() -> bool: + """True when a PostHog client is configured and available.""" + return _get_client() is not None + + +def get_client() -> Any | None: + """Raw PostHog client for integrations that need it (e.g. the LLM handler).""" + return _get_client() + + +def _client_label(auth: AuthContext) -> str: + """Best-effort ``client`` property derived from the auth principal. + + ``session`` can't be split into web vs desktop from auth alone, so callers + that know better may override ``client`` in ``properties``. + """ + if auth.method == "system": + return auth.source or "system" + if auth.method == "pat": + return "pat" + return "web" + + +def capture( + event: str, + *, + distinct_id: str, + properties: dict[str, Any] | None = None, + groups: dict[str, str] | None = None, +) -> None: + """Capture a product event. No-op (and never raises) when disabled. + + Wrapped in try/except like the frontend ``safeCapture`` — analytics must + never break a request. ``posthog`` v6 signature is ``capture(event, + distinct_id=..., properties=...)`` (event first, distinct_id a kwarg). + """ + client = _get_client() + if client is None: + return + + try: + props = {"source": _SOURCE, **(properties or {})} + client.capture( + event, + distinct_id=distinct_id, + properties=props, + groups=groups, + disable_geoip=True, + ) + except Exception: + logger.debug("PostHog capture failed for %s", event, exc_info=True) + + +def capture_for( + auth: AuthContext, + event: str, + properties: dict[str, Any] | None = None, + groups: dict[str, str] | None = None, +) -> None: + """Capture an event attributed to an ``AuthContext`` principal. + + Derives ``distinct_id`` from the user id and stamps ``auth_method`` and a + best-effort ``client`` so events are attributable to their surface + (web/desktop/pat/gateway/automation). + """ + if _get_client() is None: + return + + props = { + "auth_method": auth.method, + "client": _client_label(auth), + **(properties or {}), + } + capture( + event, + distinct_id=str(auth.user.id), + properties=props, + groups=groups, + ) + + +def group_identify( + group_type: str, + group_key: str, + properties: dict[str, Any] | None = None, +) -> None: + """Upsert group properties (e.g. per-workspace metadata). No-op when disabled.""" + client = _get_client() + if client is None: + return + + try: + client.group_identify( + group_type=group_type, + group_key=group_key, + properties=properties or {}, + ) + except Exception: + logger.debug("PostHog group_identify failed for %s", group_type, exc_info=True) + + +def shutdown() -> None: + """Flush queued events and stop the consumer thread. Safe to call always.""" + global _client + client = _client + if client is None: + return + try: + client.shutdown() + except Exception: + logger.debug("PostHog shutdown failed", exc_info=True) + + +__all__ = [ + "capture", + "capture_for", + "get_client", + "group_identify", + "is_enabled", + "shutdown", +] diff --git a/surfsense_backend/app/podcasts/tasks/render.py b/surfsense_backend/app/podcasts/tasks/render.py index 7759691a9..0cd6fbe5a 100644 --- a/surfsense_backend/app/podcasts/tasks/render.py +++ b/surfsense_backend/app/podcasts/tasks/render.py @@ -11,7 +11,10 @@ import logging import tempfile from pathlib import Path +from sqlalchemy import select + from app.celery_app import celery_app +from app.observability import analytics as ph_analytics from app.podcasts.persistence import PodcastRepository from app.podcasts.rendering import PodcastRenderer from app.podcasts.service import ( @@ -76,6 +79,30 @@ async def _render_audio(podcast_id: int) -> dict: podcast, storage_backend=backend_name, storage_key=key ) await session.commit() + + # Credit-consuming deliverable; the frontend never confirms the + # render finished. Owner (workspace.user_id) resolved lazily so + # disabled installs pay nothing for the extra query. + if ph_analytics.is_enabled(): + # Local import: app.db <-> app.podcasts.persistence have a + # module-init cycle; deferring keeps this task importable. + from app.db import Workspace + + owner_id = await session.scalar( + select(Workspace.user_id).where( + Workspace.id == podcast.workspace_id + ) + ) + if owner_id: + ph_analytics.capture( + "podcast_generated", + distinct_id=str(owner_id), + properties={ + "workspace_id": podcast.workspace_id, + "podcast_id": podcast_id, + }, + groups={"workspace": str(podcast.workspace_id)}, + ) except InvalidTransitionError: # A user back-out won the race (e.g. the regeneration was # reverted): drop the stale render and leave the row alone. diff --git a/surfsense_backend/app/routes/anonymous_chat_routes.py b/surfsense_backend/app/routes/anonymous_chat_routes.py index 3ff2fd38a..f53217826 100644 --- a/surfsense_backend/app/routes/anonymous_chat_routes.py +++ b/surfsense_backend/app/routes/anonymous_chat_routes.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import logging import secrets import uuid @@ -13,6 +14,10 @@ from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field from app.config import config +from app.observability import analytics as ph_analytics +from app.tasks.chat.streaming.flows.shared.analytics import ( + build_llm_callback_handler, +) from app.etl_pipeline.file_classifier import ( DIRECT_CONVERT_EXTENSIONS, PLAINTEXT_EXTENSIONS, @@ -354,6 +359,7 @@ async def stream_anonymous_chat( accumulator = start_turn() streaming_service = VercelStreamingService() + anon_outcome = "success" try: async with shielded_async_session(): @@ -394,6 +400,21 @@ async def stream_anonymous_chat( "recursion_limit": 40, } + # PostHog LLM analytics for the free tier — model spend per + # model is the highest-value cost insight. distinct_id is the + # anon session id (not joined to any registered person). + _anon_llm_handler = build_llm_callback_handler( + distinct_id=session_id, + trace_id=anon_thread_id, + properties={ + "client": "anonymous", + "model_slug": body.model_slug, + "$ai_session_id": session_id, + }, + ) + if _anon_llm_handler is not None: + langgraph_config["callbacks"] = [_anon_llm_handler] + yield streaming_service.format_message_start() yield streaming_service.format_start_step() @@ -465,6 +486,7 @@ async def stream_anonymous_chat( except Exception as e: logger.exception("Anonymous chat stream error") + anon_outcome = "error" await TokenQuotaService.anon_release(session_key, ip_key, request_id) _, error_code, _, _, user_message, extra = classify_stream_exception( e, @@ -479,6 +501,26 @@ async def stream_anonymous_chat( finally: await TokenQuotaService.anon_release_stream_slot(client_ip) + # Server-truth free-tier volume/model/outcome/token-burn. distinct_id + # is the anon session id — deliberately NOT joined to the frontend's + # PostHog anonymous id (the point is server truth, not funnel merge). + with contextlib.suppress(Exception): + if ph_analytics.is_enabled(): + ph_analytics.capture( + "anon_chat_turn_completed", + distinct_id=session_id, + properties={ + "client": "anonymous", + "outcome": anon_outcome, + "model_slug": body.model_slug, + "model_name": model_cfg.get("model_name"), + "total_tokens": accumulator.grand_total, + "prompt_tokens": accumulator.total_prompt_tokens, + "completion_tokens": accumulator.total_completion_tokens, + "cost_micros": accumulator.total_cost_micros, + }, + ) + return StreamingResponse( _generate(), media_type="text/event-stream", diff --git a/surfsense_backend/app/routes/image_generation_routes.py b/surfsense_backend/app/routes/image_generation_routes.py index 5a6c71127..e4aab3f80 100644 --- a/surfsense_backend/app/routes/image_generation_routes.py +++ b/surfsense_backend/app/routes/image_generation_routes.py @@ -17,6 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload from app.auth.context import AuthContext +from app.observability import analytics as ph_analytics from app.config import config from app.db import ( ImageGeneration, @@ -327,6 +328,20 @@ async def create_image_generation( await session.commit() await session.refresh(db_image_gen) + + # Credit-consuming deliverable; the frontend never confirms + # completion. ``error_message`` set => provider call failed. + ph_analytics.capture_for( + auth, + "image_generated", + { + "workspace_id": data.workspace_id, + "model": data.model, + "n": data.n, + "status": "error" if db_image_gen.error_message else "success", + }, + groups={"workspace": str(data.workspace_id)}, + ) return db_image_gen except HTTPException: diff --git a/surfsense_backend/app/routes/incentive_tasks_routes.py b/surfsense_backend/app/routes/incentive_tasks_routes.py index 2635df42f..a4c553097 100644 --- a/surfsense_backend/app/routes/incentive_tasks_routes.py +++ b/surfsense_backend/app/routes/incentive_tasks_routes.py @@ -15,6 +15,7 @@ from app.db import ( UserIncentiveTask, get_async_session, ) +from app.observability import analytics as ph_analytics from app.schemas.incentive_tasks import ( CompleteTaskResponse, IncentiveTaskInfo, @@ -125,6 +126,20 @@ async def complete_task( await session.commit() await session.refresh(user) + # Authoritative reward grant (migrated from earn-credits-content.tsx). + # Placed after the already-completed early-return so retries never + # double-count. + ph_analytics.capture_for( + auth, + "incentive_task_completed", + { + "task_type": task_type.value + if hasattr(task_type, "value") + else str(task_type), + "credit_micros_rewarded": credit_micros_reward, + }, + ) + return CompleteTaskResponse( success=True, message=f"Task completed! You earned ${credit_micros_reward / 1_000_000:.2f} of credit.", diff --git a/surfsense_backend/app/routes/new_chat_routes.py b/surfsense_backend/app/routes/new_chat_routes.py index 89d1cef95..71a70e9b5 100644 --- a/surfsense_backend/app/routes/new_chat_routes.py +++ b/surfsense_backend/app/routes/new_chat_routes.py @@ -51,6 +51,7 @@ from app.db import ( get_async_session, shielded_async_session, ) +from app.observability import analytics as ph_analytics from app.schemas.new_chat import ( AgentToolInfo, CancelActiveTurnResponse, @@ -823,6 +824,15 @@ async def create_thread( session.add(db_thread) await session.commit() await session.refresh(db_thread) + + # Authoritative thread creation (migrated from stream-engine/engine.ts). + # get_auth_context => covers PAT/MCP callers the frontend never sees. + ph_analytics.capture_for( + auth, + "chat_created", + {"workspace_id": db_thread.workspace_id, "chat_id": db_thread.id}, + groups={"workspace": str(db_thread.workspace_id)}, + ) return db_thread except HTTPException: diff --git a/surfsense_backend/app/routes/personal_access_tokens_routes.py b/surfsense_backend/app/routes/personal_access_tokens_routes.py index a7849a2fc..3d63703b4 100644 --- a/surfsense_backend/app/routes/personal_access_tokens_routes.py +++ b/surfsense_backend/app/routes/personal_access_tokens_routes.py @@ -8,6 +8,7 @@ from sqlalchemy.future import select from app.auth.context import AuthContext from app.config import config from app.db import PersonalAccessToken, get_async_session +from app.observability import analytics as ph_analytics from app.schemas.pat import PATCreate, PATCreated, PATRead from app.users import require_session_context from app.utils.pat import generate_pat, hash_pat, token_prefix @@ -57,6 +58,13 @@ async def create_personal_access_token( await session.commit() await session.refresh(pat) + # Leading indicator of MCP / programmatic-API adoption. + ph_analytics.capture_for( + auth, + "pat_created", + {"pat_id": pat.id, "has_expiry": pat.expires_at is not None}, + ) + return PATCreated( id=pat.id, label=pat.label, @@ -102,3 +110,5 @@ async def delete_personal_access_token( ) ) await session.commit() + + ph_analytics.capture_for(auth, "pat_revoked", {"pat_id": pat_id}) diff --git a/surfsense_backend/app/routes/public_chat_routes.py b/surfsense_backend/app/routes/public_chat_routes.py index 70f012911..f90050055 100644 --- a/surfsense_backend/app/routes/public_chat_routes.py +++ b/surfsense_backend/app/routes/public_chat_routes.py @@ -13,6 +13,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.auth.context import AuthContext from app.db import get_async_session +from app.observability import analytics as ph_analytics from app.schemas.new_chat import ( CloneResponse, PublicChatResponse, @@ -56,7 +57,20 @@ async def clone_public_chat( Creates thread and copies messages. Requires authentication. """ - return await clone_from_snapshot(session, share_token, user) + result = await clone_from_snapshot(session, share_token, user) + + # Share-link conversion — only observable server-side. + ph_analytics.capture_for( + auth, + "public_chat_cloned", + { + "workspace_id": result.workspace_id, + "chat_id": result.thread_id, + }, + groups={"workspace": str(result.workspace_id)}, + ) + + return result @router.get("/{share_token}/podcasts/{podcast_id}") diff --git a/surfsense_backend/app/routes/rbac_routes.py b/surfsense_backend/app/routes/rbac_routes.py index 2d8f58d33..8ab0e17c6 100644 --- a/surfsense_backend/app/routes/rbac_routes.py +++ b/surfsense_backend/app/routes/rbac_routes.py @@ -28,6 +28,7 @@ from app.db import ( WorkspaceRole, get_async_session, ) +from app.observability import analytics as ph_analytics from app.schemas import ( InviteAcceptRequest, InviteAcceptResponse, @@ -782,6 +783,19 @@ async def create_invite( ) db_invite = result.scalars().first() + # Authoritative invite creation (migrated from team-content.tsx). + ph_analytics.capture_for( + auth, + "workspace_invite_sent", + { + "workspace_id": workspace_id, + "role_name": db_invite.role.name if db_invite.role else None, + "has_expiry": db_invite.expires_at is not None, + "has_max_uses": db_invite.max_uses is not None, + }, + groups={"workspace": str(workspace_id)}, + ) + return db_invite except HTTPException: @@ -1091,6 +1105,16 @@ async def accept_invite( role_name = invite.role.name if invite.role else "Default" workspace_name = invite.workspace.name if invite.workspace else "" + # Authoritative join (migrated from app/invite/[invite_code]/page.tsx, + # which fired both events). workspace_name dropped — user content. + for _evt in ("workspace_invite_accepted", "workspace_user_added"): + ph_analytics.capture_for( + auth, + _evt, + {"workspace_id": invite.workspace_id, "role_name": role_name}, + groups={"workspace": str(invite.workspace_id)}, + ) + return InviteAcceptResponse( message="Successfully joined the workspace", workspace_id=invite.workspace_id, diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 4cfa78af2..ba0899db3 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -29,6 +29,7 @@ import redis from dateutil.parser import isoparse from fastapi import APIRouter, Body, Depends, HTTPException, Query from pydantic import BaseModel, Field, ValidationError +from sqlalchemy import event as sa_event from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select @@ -44,6 +45,7 @@ from app.db import ( get_async_session, ) from app.notifications.service import NotificationService +from app.observability import analytics as ph_analytics from app.observability import metrics as ot_metrics, otel as ot from app.schemas import ( GoogleDriveIndexRequest, @@ -135,6 +137,45 @@ def _get_heartbeat_key(notification_id: int) -> str: router = APIRouter() +def _connector_type_value(connector_type) -> str: + """Low-cardinality connector_type string for analytics (enum -> its value).""" + return getattr(connector_type, "value", None) or str(connector_type) + + +def _emit_connector_connected(_mapper, _connection, target) -> None: + """SQLAlchemy after_insert hook: one guard for every connector-creation path. + + Fires ``connector_connected`` for the form route here AND all 15 OAuth + callback routes (each builds ``SearchSourceConnector(...)`` inline with no + shared helper), so we don't instrument 16 call sites. ``distinct_id`` comes + off the row itself (``user_id``) — the same person the web app identifies. + + ponytail: fires during flush, so a post-flush rollback would over-count + (rare; connector creation commits immediately after add). No-op without a + PostHog key. Upgrade path: move to an after_commit collector if over-count + ever shows up in the data. + """ + if not ph_analytics.is_enabled(): + return + user_id = getattr(target, "user_id", None) + if not user_id: + return + workspace_id = getattr(target, "workspace_id", None) + ph_analytics.capture( + "connector_connected", + distinct_id=str(user_id), + properties={ + "workspace_id": workspace_id, + "connector_id": target.id, + "connector_type": _connector_type_value(target.connector_type), + }, + groups={"workspace": str(workspace_id)} if workspace_id is not None else None, + ) + + +sa_event.listen(SearchSourceConnector, "after_insert", _emit_connector_connected) + + # Use Pydantic's BaseModel here class GitHubPATRequest(BaseModel): github_pat: str = Field(..., description="GitHub Personal Access Token") @@ -244,6 +285,10 @@ async def create_search_source_connector( await session.commit() await session.refresh(db_connector) + # ``connector_connected`` is emitted by the after_insert listener below, + # so it fires once for this form route AND all 15 OAuth callback routes + # without instrumenting each — see _emit_connector_connected. + # Create periodic schedule if periodic indexing is enabled if ( db_connector.periodic_indexing_enabled @@ -679,10 +724,23 @@ async def delete_search_source_connector( # Delete the connector record workspace_id = db_connector.workspace_id + deleted_connector_type = db_connector.connector_type is_mcp = db_connector.connector_type == SearchSourceConnectorType.MCP_CONNECTOR await session.delete(db_connector) await session.commit() + # Authoritative deletion (migrated from use-connector-dialog.ts). + ph_analytics.capture_for( + auth, + "connector_deleted", + { + "workspace_id": workspace_id, + "connector_id": connector_id, + "connector_type": _connector_type_value(deleted_connector_type), + }, + groups={"workspace": str(workspace_id)}, + ) + if is_mcp: from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import ( invalidate_mcp_tools_cache, diff --git a/surfsense_backend/app/routes/stripe_routes.py b/surfsense_backend/app/routes/stripe_routes.py index 459b23d60..4da24c2af 100644 --- a/surfsense_backend/app/routes/stripe_routes.py +++ b/surfsense_backend/app/routes/stripe_routes.py @@ -27,6 +27,7 @@ from app.db import ( User, get_async_session, ) +from app.observability import analytics as ph_analytics from app.schemas.stripe import ( AutoReloadSettingsResponse, CreateAutoReloadSetupSessionRequest, @@ -47,6 +48,26 @@ logger = logging.getLogger(__name__) router = APIRouter(prefix="/stripe", tags=["stripe"]) +def _capture_credits_purchased(user: User, purchase: CreditPurchase) -> None: + """Emit ``credits_purchased`` — revenue events only exist server-side. + + Call only from the idempotent grant paths (which early-return on already + COMPLETED / non-PENDING rows) so Stripe retries never double-count. No-op + when PostHog is unconfigured. + """ + ph_analytics.capture( + "credits_purchased", + distinct_id=str(user.id), + properties={ + "credit_micros_granted": purchase.credit_micros_granted, + "quantity": purchase.quantity, + "amount_total": purchase.amount_total, + "currency": purchase.currency, + "source": purchase.source, + }, + ) + + def get_stripe_client() -> StripeClient: """Return a configured Stripe client or raise if Stripe is disabled.""" if not config.STRIPE_SECRET_KEY: @@ -309,6 +330,7 @@ async def _fulfill_completed_credit_purchase( ) await db_session.commit() + _capture_credits_purchased(user, purchase) return StripeWebhookResponse() @@ -448,6 +470,8 @@ async def _reconcile_auto_reload_payment_intent( purchase.status = CreditPurchaseStatus.FAILED await db_session.commit() + if succeeded: + _capture_credits_purchased(user, purchase) return StripeWebhookResponse() diff --git a/surfsense_backend/app/routes/workspaces_routes.py b/surfsense_backend/app/routes/workspaces_routes.py index 53d4344f7..26e08a225 100644 --- a/surfsense_backend/app/routes/workspaces_routes.py +++ b/surfsense_backend/app/routes/workspaces_routes.py @@ -15,6 +15,7 @@ from app.db import ( get_async_session, get_default_roles_config, ) +from app.observability import analytics as ph_analytics from app.routes.model_connections_routes import compute_llm_setup_status from app.schemas import ( WorkspaceApiAccessUpdate, @@ -112,6 +113,16 @@ async def create_workspace( await session.commit() await session.refresh(db_workspace) + # Authoritative creation event (migrated from the frontend + # CreateWorkspaceDialog). Workspace name is intentionally NOT sent — + # it's user content with no aggregation value. + ph_analytics.capture_for( + auth, + "workspace_created", + {"workspace_id": db_workspace.id}, + groups={"workspace": str(db_workspace.id)}, + ) + response = WorkspaceRead.model_validate(db_workspace) response.llm_setup = await compute_llm_setup_status( session, auth, db_workspace.id diff --git a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py index ec078e031..fb08215b5 100644 --- a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py @@ -1,5 +1,6 @@ """Celery tasks for connector indexing.""" +import contextlib import logging import time import traceback @@ -8,6 +9,7 @@ from collections.abc import Awaitable, Callable from celery import current_task from app.celery_app import celery_app +from app.observability import analytics as ph_analytics from app.observability import metrics as ot_metrics, otel as ot from app.tasks.celery_tasks import ( get_celery_session_maker, @@ -17,8 +19,18 @@ from app.tasks.celery_tasks import ( logger = logging.getLogger(__name__) -def run_async_celery_task[T](coro_factory: Callable[[], Awaitable[T]]) -> T: - """Run connector sync work and record aggregate connector metrics.""" +def run_async_celery_task[T]( + coro_factory: Callable[[], Awaitable[T]], + *, + workspace_id: int | None = None, + user_id: str | None = None, +) -> T: + """Run connector sync work and record aggregate connector metrics. + + When ``workspace_id``/``user_id`` are provided, also emits the PostHog + ``connector_indexing_completed``/``_failed`` outcome (the frontend only + knows indexing *started*, never whether it worked). No-op without a key. + """ task_name = getattr(current_task, "name", None) or "unknown" t0 = time.perf_counter() status = "failed" @@ -45,6 +57,29 @@ def run_async_celery_task[T](coro_factory: Callable[[], Awaitable[T]]) -> T: status=status, error_category=error_category, ) + if user_id and ph_analytics.is_enabled(): + event = ( + "connector_indexing_completed" + if status == "success" + else "connector_indexing_failed" + ) + with contextlib.suppress(Exception): + ph_analytics.capture( + event, + distinct_id=str(user_id), + properties={ + "workspace_id": workspace_id, + # ``connector_type`` is the Celery task name, e.g. + # index_notion_pages / index_github_repos. + "connector_type": task_name, + "status": status, + "error_category": error_category, + "duration_ms": int(elapsed_s * 1000), + }, + groups={"workspace": str(workspace_id)} + if workspace_id is not None + else None, + ) def _handle_greenlet_error(e: Exception, task_name: str, connector_id: int) -> None: @@ -91,7 +126,9 @@ def index_notion_pages_task( return run_async_celery_task( lambda: _index_notion_pages( connector_id, workspace_id, user_id, start_date, end_date - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) except Exception as e: _handle_greenlet_error(e, "index_notion_pages", connector_id) @@ -129,7 +166,9 @@ def index_github_repos_task( return run_async_celery_task( lambda: _index_github_repos( connector_id, workspace_id, user_id, start_date, end_date - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) @@ -164,7 +203,9 @@ def index_confluence_pages_task( return run_async_celery_task( lambda: _index_confluence_pages( connector_id, workspace_id, user_id, start_date, end_date - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) @@ -200,7 +241,9 @@ def index_google_calendar_events_task( return run_async_celery_task( lambda: _index_google_calendar_events( connector_id, workspace_id, user_id, start_date, end_date - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) except Exception as e: _handle_greenlet_error(e, "index_google_calendar_events", connector_id) @@ -238,7 +281,9 @@ def index_google_gmail_messages_task( return run_async_celery_task( lambda: _index_google_gmail_messages( connector_id, workspace_id, user_id, start_date, end_date - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) @@ -275,7 +320,9 @@ def index_google_drive_files_task( workspace_id, user_id, items_dict, - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) @@ -315,7 +362,9 @@ def index_onedrive_files_task( workspace_id, user_id, items_dict, - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) @@ -355,7 +404,9 @@ def index_dropbox_files_task( workspace_id, user_id, items_dict, - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) @@ -393,7 +444,9 @@ def index_elasticsearch_documents_task( return run_async_celery_task( lambda: _index_elasticsearch_documents( connector_id, workspace_id, user_id, start_date, end_date - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) @@ -428,7 +481,9 @@ def index_bookstack_pages_task( return run_async_celery_task( lambda: _index_bookstack_pages( connector_id, workspace_id, user_id, start_date, end_date - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) @@ -463,7 +518,9 @@ def index_composio_connector_task( return run_async_celery_task( lambda: _index_composio_connector( connector_id, workspace_id, user_id, start_date, end_date - ) + ), + workspace_id=workspace_id, + user_id=user_id, ) diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index e2ca5345e..8e20275b7 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -4,11 +4,13 @@ import asyncio import contextlib import logging import os +import time from uuid import UUID from app.celery_app import celery_app from app.config import config from app.notifications.service import NotificationService +from app.observability import analytics as ph_analytics from app.observability import metrics as ot_metrics from app.services.task_logging_service import TaskLoggingService from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task @@ -22,6 +24,44 @@ from app.tasks.document_processors import ( logger = logging.getLogger(__name__) + +def _capture_doc_processing( + status: str, + *, + user_id: str | None, + workspace_id: int, + doc_type: str, + file_size: int | None = None, + duration_ms: int | None = None, +) -> None: + """Emit ``document_processing_completed``/``_failed`` from a Celery task. + + The frontend only knows the upload POST succeeded, never whether ingestion + actually worked — this is the authoritative outcome. No-op when PostHog is + unconfigured. ``distinct_id`` is the owning user's id so it joins the same + person the web app identifies. + """ + if not ph_analytics.is_enabled() or not user_id: + return + event = ( + "document_processing_completed" + if status == "success" + else "document_processing_failed" + ) + ph_analytics.capture( + event, + distinct_id=str(user_id), + properties={ + "workspace_id": workspace_id, + "doc_type": doc_type, + "file_size": file_size, + "duration_ms": duration_ms, + "status": status, + }, + groups={"workspace": str(workspace_id)}, + ) + + # ===== Redis heartbeat for document processing tasks ===== # Same mechanism as connector indexing heartbeats (search_source_connectors_routes.py). # A background coroutine refreshes a Redis key every 60s with a 2-min TTL. @@ -268,11 +308,30 @@ def process_extension_document_task( workspace_id: ID of the workspace user_id: ID of the user """ - return run_async_celery_task( - lambda: _process_extension_document( - individual_document_dict, workspace_id, user_id + _t0 = time.perf_counter() + try: + result = run_async_celery_task( + lambda: _process_extension_document( + individual_document_dict, workspace_id, user_id + ) ) + except Exception: + _capture_doc_processing( + "failed", + user_id=user_id, + workspace_id=workspace_id, + doc_type="extension", + duration_ms=int((time.perf_counter() - _t0) * 1000), + ) + raise + _capture_doc_processing( + "success", + user_id=user_id, + workspace_id=workspace_id, + doc_type="extension", + duration_ms=int((time.perf_counter() - _t0) * 1000), ) + return result async def _process_extension_document( @@ -430,12 +489,14 @@ def process_file_upload_task( ) return + file_size: int | None = None try: file_size = os.path.getsize(file_path) logger.info(f"[process_file_upload] File size: {file_size} bytes") except Exception as e: logger.warning(f"[process_file_upload] Could not get file size: {e}") + _t0 = time.perf_counter() try: run_async_celery_task( lambda: _process_file_upload(file_path, filename, workspace_id, user_id) @@ -448,7 +509,23 @@ def process_file_upload_task( f"[process_file_upload] Task failed for {filename}: {e}\n" f"Traceback:\n{traceback.format_exc()}" ) + _capture_doc_processing( + "failed", + user_id=user_id, + workspace_id=workspace_id, + doc_type="file_upload", + file_size=file_size, + duration_ms=int((time.perf_counter() - _t0) * 1000), + ) raise + _capture_doc_processing( + "success", + user_id=user_id, + workspace_id=workspace_id, + doc_type="file_upload", + file_size=file_size, + duration_ms=int((time.perf_counter() - _t0) * 1000), + ) async def _process_file_upload( @@ -682,6 +759,7 @@ def process_file_upload_with_document_task( ) return + _t0 = time.perf_counter() try: run_async_celery_task( lambda: _process_file_with_document( @@ -702,7 +780,21 @@ def process_file_upload_with_document_task( f"[process_file_upload_with_document] Task failed for {filename}: {e}\n" f"Traceback:\n{traceback.format_exc()}" ) + _capture_doc_processing( + "failed", + user_id=user_id, + workspace_id=workspace_id, + doc_type="file_upload_2phase", + duration_ms=int((time.perf_counter() - _t0) * 1000), + ) raise + _capture_doc_processing( + "success", + user_id=user_id, + workspace_id=workspace_id, + doc_type="file_upload_2phase", + duration_ms=int((time.perf_counter() - _t0) * 1000), + ) async def _mark_document_failed(document_id: int, reason: str): diff --git a/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py b/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py index 4f934cc27..c91d3031c 100644 --- a/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py @@ -12,6 +12,7 @@ from app.agents.video_presentation.state import State as VideoPresentationState from app.celery_app import celery_app from app.config import config as app_config from app.db import VideoPresentation, VideoPresentationStatus +from app.observability import analytics as ph_analytics from app.services.billable_calls import ( BillingSettlementError, QuotaInsufficientError, @@ -239,6 +240,20 @@ async def _generate_video_presentation( logger.info(f"Successfully generated video presentation: {video_pres.id}") + # Credit-consuming deliverable — the frontend never confirms + # completion. Attributed to the workspace owner resolved above. + if owner_user_id: + ph_analytics.capture( + "video_presentation_generated", + distinct_id=str(owner_user_id), + properties={ + "workspace_id": workspace_id, + "video_presentation_id": video_pres.id, + "slide_count": len(serializable_slides), + }, + groups={"workspace": str(workspace_id)}, + ) + return { "status": "ready", "video_presentation_id": video_pres.id, diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py index 4451b4b37..a6544b810 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py @@ -97,6 +97,10 @@ from app.tasks.chat.streaming.flows.shared.rate_limit_recovery import ( log_rate_limit_recovered, reroute_to_next_auto_pin, ) +from app.tasks.chat.streaming.flows.shared.analytics import ( + build_llm_callback_handler, + capture_chat_turn_completed, +) from app.tasks.chat.streaming.flows.shared.span import ( close_chat_request_span, open_chat_request_span, @@ -487,6 +491,23 @@ async def stream_new_chat( "recursion_limit": 10_000, } + # PostHog LLM analytics: attach a callback so the full agent trace + # tree (LLM calls, tools, subagents) is captured per turn. No-op when + # PostHog is unconfigured. + _llm_handler = build_llm_callback_handler( + distinct_id=user_id, + trace_id=stream_result.turn_id, + properties={ + "workspace_id": workspace_id, + "chat_id": chat_id, + "$ai_session_id": str(chat_id), + "flow": flow, + }, + groups={"workspace": str(workspace_id)}, + ) + if _llm_handler is not None: + config["callbacks"] = [_llm_handler] + # --- Block 4: First SSE frames --- for sse in iter_initial_frames( @@ -830,6 +851,27 @@ async def stream_new_chat( log_prefix="stream_new_chat", ) + # Authoritative server-side product event. Inside the shield so it + # survives client-disconnect cancellation (abandoned tabs still + # produce a turn event), and while ``stream_result`` is still live + # (it's dropped to None below for GC). + capture_chat_turn_completed( + flow=flow, + outcome=chat_outcome, + error_category=chat_error_category, + workspace_id=workspace_id, + chat_id=chat_id, + user_id=user_id, + auth_context=auth_context, + agent_mode=chat_agent_mode, + client_platform=fs_platform, + filesystem_mode=fs_mode, + turn_id=getattr(stream_result, "turn_id", None), + request_id=request_id, + duration_ms=int((time.perf_counter() - _t_total) * 1000), + accumulator=accumulator, + ) + # Persist any sandbox-produced files to local storage so they remain # downloadable after the Daytona sandbox auto-deletes. if stream_result and stream_result.sandbox_files: diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py index 8b7956f4c..8dd749aee 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py @@ -76,6 +76,10 @@ from app.tasks.chat.streaming.flows.shared.rate_limit_recovery import ( log_rate_limit_recovered, reroute_to_next_auto_pin, ) +from app.tasks.chat.streaming.flows.shared.analytics import ( + build_llm_callback_handler, + capture_chat_turn_completed, +) from app.tasks.chat.streaming.flows.shared.span import ( close_chat_request_span, open_chat_request_span, @@ -387,6 +391,22 @@ async def stream_resume_chat( "recursion_limit": 10_000, } + # PostHog LLM analytics — same trace as the original turn's + # conversation via ``$ai_session_id``. No-op when unconfigured. + _llm_handler = build_llm_callback_handler( + distinct_id=user_id, + trace_id=stream_result.turn_id, + properties={ + "workspace_id": workspace_id, + "chat_id": chat_id, + "$ai_session_id": str(chat_id), + "flow": "resume", + }, + groups={"workspace": str(workspace_id)}, + ) + if _llm_handler is not None: + config["callbacks"] = [_llm_handler] + # --- First SSE frames --- for sse in iter_initial_frames( @@ -599,6 +619,25 @@ async def stream_resume_chat( log_prefix="stream_resume", ) + # Authoritative server-side product event (see new_chat + # orchestrator for rationale). ``flow="resume"``. + capture_chat_turn_completed( + flow="resume", + outcome=chat_outcome, + error_category=chat_error_category, + workspace_id=workspace_id, + chat_id=chat_id, + user_id=user_id, + auth_context=auth_context, + agent_mode=chat_agent_mode, + client_platform=fs_platform, + filesystem_mode=fs_mode, + turn_id=getattr(stream_result, "turn_id", None), + request_id=request_id, + duration_ms=int((time.perf_counter() - _t_total) * 1000), + accumulator=accumulator, + ) + # Release the lock from the original interrupted turn or any # re-interrupt/bailout. Skip on ``BusyError`` (lock not held here). if not busy_error_raised: diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/analytics.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/analytics.py new file mode 100644 index 000000000..2d311c26b --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/analytics.py @@ -0,0 +1,117 @@ +"""PostHog chat-turn analytics for streaming flows. + +Emits a single authoritative ``chat_turn_completed`` product event per turn, +shared by the new-chat and resume orchestrators so every chat source (web, +desktop, PAT scripts, gateway, automations) is tracked identically — +including sources the frontend can never observe. No-op when PostHog is +unconfigured. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from app.config import config +from app.observability import analytics + +if TYPE_CHECKING: + from app.auth.context import AuthContext + from app.services.token_tracking_service import TurnTokenAccumulator + +logger = logging.getLogger(__name__) + + +def build_llm_callback_handler( + *, + distinct_id: str | None, + trace_id: str | None, + properties: dict[str, Any] | None = None, + groups: dict[str, str] | None = None, +) -> Any | None: + """Build a PostHog LangChain ``CallbackHandler`` for a chat turn. + + Attaching this to the LangGraph ``config["callbacks"]`` captures the full + agent trace tree ($ai_trace / $ai_span / $ai_generation) — every LLM call, + tool, subagent, and retriever — joined to the ``chat_turn_completed`` event + via ``trace_id`` (the turn id) and grouped per conversation via + ``$ai_session_id`` (in ``properties``). + + Returns ``None`` when PostHog is disabled or the package/handler is + unavailable, so callers can simply skip attaching callbacks. ``privacy_mode`` + (default on) suppresses prompt/completion bodies — chat content includes + users' private documents. + """ + client_obj = analytics.get_client() + if client_obj is None or not distinct_id: + return None + + try: + from posthog.ai.langchain import CallbackHandler + + return CallbackHandler( + client=client_obj, + distinct_id=distinct_id, + trace_id=trace_id, + properties=properties or {}, + privacy_mode=config.POSTHOG_AI_PRIVACY_MODE, + groups=groups, + ) + except Exception: + logger.debug("PostHog LLM callback handler unavailable", exc_info=True) + return None + + +def capture_chat_turn_completed( + *, + flow: str, + outcome: str, + error_category: str | None, + workspace_id: int, + chat_id: int, + user_id: str | None, + auth_context: AuthContext | None, + agent_mode: str, + client_platform: str, + filesystem_mode: str, + turn_id: str | None, + request_id: str | None, + duration_ms: int, + accumulator: TurnTokenAccumulator, +) -> None: + """Capture ``chat_turn_completed``. Best-effort; never raises.""" + if not analytics.is_enabled() or not user_id: + return + + props: dict[str, Any] = { + "flow": flow, + "outcome": outcome, + "error_category": error_category, + "workspace_id": workspace_id, + "chat_id": chat_id, + "agent_mode": agent_mode, + "client_platform": client_platform, + "filesystem_mode": filesystem_mode, + "turn_id": turn_id, + "request_id": request_id, + "duration_ms": duration_ms, + # Cost is micro-USD (integer), matching TurnTokenAccumulator; do not + # convert to float dollars. + "total_tokens": accumulator.grand_total, + "prompt_tokens": accumulator.total_prompt_tokens, + "completion_tokens": accumulator.total_completion_tokens, + "cost_micros": accumulator.total_cost_micros, + } + groups = {"workspace": str(workspace_id)} + + if auth_context is not None: + analytics.capture_for( + auth_context, "chat_turn_completed", props, groups=groups + ) + else: + analytics.capture( + "chat_turn_completed", + distinct_id=user_id, + properties=props, + groups=groups, + ) diff --git a/surfsense_backend/app/users.py b/surfsense_backend/app/users.py index dbec93876..633b08443 100644 --- a/surfsense_backend/app/users.py +++ b/surfsense_backend/app/users.py @@ -20,6 +20,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.auth.context import AuthContext from app.auth.session_cookies import access_expires_at, write_session from app.config import config +from app.observability import analytics as ph_analytics from app.db import ( Prompt, User, @@ -144,6 +145,9 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]): except Exception as e: logger.warning(f"Failed to update last_login for user {user.id}: {e}") + # Authoritative login event (vs. the frontend's optimistic capture). + ph_analytics.capture("auth_login_success", distinct_id=str(user.id)) + async def on_after_register(self, user: User, request: Request | None = None): """ Called after a user registers. Creates a default workspace for the user @@ -206,6 +210,17 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]): logger.info( f"Created default workspace (ID: {default_workspace.id}) for user {user.id}" ) + + # Authoritative registration + auto-created default workspace. + ph_analytics.capture( + "auth_registration_success", distinct_id=str(user.id) + ) + ph_analytics.capture( + "workspace_created", + distinct_id=str(user.id), + properties={"client": "auto_register"}, + groups={"workspace": str(default_workspace.id)}, + ) except Exception as e: logger.error(f"Failed to create default workspace for user {user.id}: {e}") @@ -338,6 +353,13 @@ async def get_auth_context( FastAPI-Users still handles JWT mechanics; PATs are resolved here so RBAC receives the full SurfSense principal instead of a bare User. """ + def _stash(ctx: AuthContext) -> AuthContext: + # Expose the resolved principal on request.state so downstream + # middleware (e.g. PostHog pat_api_request attribution) can read it + # without re-resolving auth. + request.state.auth_context = ctx + return ctx + auth_header = request.headers.get("Authorization") if auth_header: scheme, _, credential = auth_header.partition(" ") @@ -348,7 +370,7 @@ async def get_auth_context( pat = await resolve_pat(session, token) if pat and pat.user and pat.user.is_active: maybe_touch_last_used(pat) - return AuthContext.pat_auth(pat.user, pat) + return _stash(AuthContext.pat_auth(pat.user, pat)) if is_bearer and _token_meets_epoch(token): try: @@ -358,7 +380,7 @@ async def get_auth_context( user = None if user and user.is_active: - return AuthContext.session(user) + return _stash(AuthContext.session(user)) cookie_token = request.cookies.get(config.SESSION_COOKIE_NAME) if cookie_token and _token_meets_epoch(cookie_token): @@ -369,7 +391,7 @@ async def get_auth_context( user = None if user and user.is_active: - return AuthContext.session(user) + return _stash(AuthContext.session(user)) raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index bd346b306..955178320 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -86,6 +86,7 @@ dependencies = [ "python-telegram-bot>=22.7", "croniter>=2.0.0", "scrapling[fetchers]>=0.4.11", + "posthog>=6.0.0", ] [project.optional-dependencies] diff --git a/surfsense_backend/tests/unit/observability/test_analytics.py b/surfsense_backend/tests/unit/observability/test_analytics.py new file mode 100644 index 000000000..4cacb23c0 --- /dev/null +++ b/surfsense_backend/tests/unit/observability/test_analytics.py @@ -0,0 +1,149 @@ +"""Unit tests for the PostHog analytics wrapper. + +Covers the two properties the rest of the codebase relies on: +1. Opt-in no-op: with no client configured, every entry point is silent and + never raises (analytics must never break a request). +2. Correct stamping when a client IS present: ``source="backend"`` and + ``disable_geoip=True`` on every capture, plus ``auth_method`` / ``client`` + derived from the ``AuthContext`` principal by ``capture_for``. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from app.observability import analytics + +pytestmark = pytest.mark.unit + + +class _FakeClient: + """Records capture()/group_identify() calls instead of hitting the network.""" + + def __init__(self) -> None: + self.captures: list[dict] = [] + self.groups: list[dict] = [] + + def capture(self, event, *, distinct_id, properties, groups, disable_geoip): + self.captures.append( + { + "event": event, + "distinct_id": distinct_id, + "properties": properties, + "groups": groups, + "disable_geoip": disable_geoip, + } + ) + + def group_identify(self, *, group_type, group_key, properties): + self.groups.append( + {"group_type": group_type, "group_key": group_key, "properties": properties} + ) + + +@pytest.fixture(autouse=True) +def _reset_module_state(): + """Isolate the module-level lazy-client singleton between tests.""" + orig_client = analytics._client + orig_attempted = analytics._init_attempted + yield + analytics._client = orig_client + analytics._init_attempted = orig_attempted + + +def _use_fake_client() -> _FakeClient: + """Inject a fake client, bypassing lazy init (no posthog import, no key).""" + fake = _FakeClient() + analytics._client = fake + analytics._init_attempted = True + return fake + + +def _disable_client() -> None: + analytics._client = None + analytics._init_attempted = True + + +# ---- No-op behaviour when disabled ------------------------------------------------- + + +def test_capture_is_noop_without_client(): + _disable_client() + assert analytics.is_enabled() is False + # Must not raise even though there is no client. + analytics.capture("some_event", distinct_id="u1", properties={"a": 1}) + + +def test_capture_for_is_noop_without_client(): + _disable_client() + auth = SimpleNamespace(method="session", source=None, user=SimpleNamespace(id="u1")) + analytics.capture_for(auth, "some_event", {"a": 1}) # no raise + + +def test_shutdown_is_noop_without_client(): + _disable_client() + analytics.shutdown() # no raise + + +# ---- Stamping when a client is present --------------------------------------------- + + +def test_capture_stamps_source_and_disables_geoip(): + fake = _use_fake_client() + analytics.capture( + "chat_turn_completed", + distinct_id="user-123", + properties={"workspace_id": 7}, + groups={"workspace": "7"}, + ) + + assert len(fake.captures) == 1 + call = fake.captures[0] + assert call["event"] == "chat_turn_completed" + assert call["distinct_id"] == "user-123" + # source is always stamped so backend vs frontend events are separable. + assert call["properties"]["source"] == "backend" + assert call["properties"]["workspace_id"] == 7 + assert call["groups"] == {"workspace": "7"} + # Without this the server IP would overwrite each person's real location. + assert call["disable_geoip"] is True + + +def test_capture_never_raises_when_client_errors(): + class _Boom(_FakeClient): + def capture(self, *a, **k): + raise RuntimeError("network down") + + boom = _Boom() + analytics._client = boom + analytics._init_attempted = True + # Swallowed like the frontend safeCapture — analytics never breaks a request. + analytics.capture("evt", distinct_id="u1") + + +@pytest.mark.parametrize( + ("method", "source", "expected_client"), + [ + ("session", None, "web"), + ("pat", None, "pat"), + ("system", "gateway", "gateway"), + ("system", None, "system"), + ], +) +def test_capture_for_stamps_auth_method_and_client(method, source, expected_client): + fake = _use_fake_client() + auth = SimpleNamespace( + method=method, source=source, user=SimpleNamespace(id="user-abc") + ) + + analytics.capture_for(auth, "workspace_created", {"workspace_id": 1}) + + assert len(fake.captures) == 1 + props = fake.captures[0]["properties"] + assert fake.captures[0]["distinct_id"] == "user-abc" + assert props["auth_method"] == method + assert props["client"] == expected_client + assert props["source"] == "backend" # capture_for delegates to capture + assert props["workspace_id"] == 1 diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index 3a27770b6..5f8ec906a 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -48,6 +48,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -94,6 +97,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -140,6 +146,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -186,6 +195,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -232,6 +244,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -278,6 +293,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -324,6 +342,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -370,6 +391,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -416,6 +440,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -462,6 +489,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -523,6 +553,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -569,6 +603,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -615,6 +652,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -661,6 +701,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -707,6 +750,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -753,6 +799,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -799,6 +848,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -845,6 +897,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -891,6 +946,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -937,6 +995,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -983,6 +1044,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1044,6 +1108,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1090,6 +1158,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1136,6 +1207,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1182,6 +1256,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1228,6 +1305,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1274,6 +1354,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1320,6 +1403,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1366,6 +1452,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1412,6 +1501,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1458,6 +1550,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1504,6 +1599,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1565,6 +1663,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1611,6 +1713,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1702,6 +1807,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1748,6 +1859,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1794,6 +1908,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1885,6 +2002,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -1976,6 +2099,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -2022,6 +2151,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", ] conflicts = [[ @@ -2557,6 +2689,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845 }, ] +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148 }, +] + [[package]] name = "banks" version = "2.4.1" @@ -8650,6 +8791,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424 }, ] +[[package]] +name = "posthog" +version = "7.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backoff" }, + { name = "distro" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/f0/3af875ac3fd5863ed4874c9618d85eab3f7fd24b395827d99da0ef90aca6/posthog-7.28.0.tar.gz", hash = "sha256:9e048dee58f27373db622c0744be30c1a2b7f1df31049956d6341c9646fb3833", size = 356360 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/8b/d41d98e64bd6ce650d45690cffc718274b1eda65e3bfd79575a1cf95b45c/posthog-7.28.0-py3-none-any.whl", hash = "sha256:4cff10062807bbd8ae6ec2804a71584831a75b390ce7ba3e736bf4cc0fb25d28", size = 425147 }, +] + [[package]] name = "preshed" version = "3.0.13" @@ -10926,6 +11082,7 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "opentelemetry-semantic-conventions" }, { name = "pgvector" }, + { name = "posthog" }, { name = "psycopg", extra = ["binary", "pool"] }, { name = "pyarrow" }, { name = "pyjwt" }, @@ -11041,6 +11198,7 @@ requires-dist = [ { name = "opentelemetry-sdk", specifier = ">=1.40.0" }, { name = "opentelemetry-semantic-conventions", specifier = ">=0.61b0" }, { name = "pgvector", specifier = ">=0.3.6" }, + { name = "posthog", specifier = ">=6.0.0" }, { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.3.2" }, { name = "pyarrow", specifier = ">=15.0.0,<19.0.0" }, { name = "pyjwt", specifier = ">=2.12.0" }, diff --git a/surfsense_mcp/mcp_server/core/client.py b/surfsense_mcp/mcp_server/core/client.py index d65b47b7e..2bdf405dc 100644 --- a/surfsense_mcp/mcp_server/core/client.py +++ b/surfsense_mcp/mcp_server/core/client.py @@ -37,7 +37,11 @@ class SurfSenseClient: self._fallback_api_key = fallback_api_key self._http = httpx.AsyncClient( base_url=api_base, - headers={"Accept": "application/json"}, + # ``X-SurfSense-Client`` lets the backend distinguish PAT traffic + # originating from this MCP server vs. raw PAT scripts, so + # "documents added via MCP" / "searches via MCP" are queryable. + # Server-to-server, so no CORS implications. + headers={"Accept": "application/json", "X-SurfSense-Client": "mcp"}, timeout=timeout, ) diff --git a/surfsense_web/app/(home)/login/LocalLoginForm.tsx b/surfsense_web/app/(home)/login/LocalLoginForm.tsx index dd415e10f..e60463b9d 100644 --- a/surfsense_web/app/(home)/login/LocalLoginForm.tsx +++ b/surfsense_web/app/(home)/login/LocalLoginForm.tsx @@ -13,7 +13,7 @@ import { Spinner } from "@/components/ui/spinner"; import { getAuthErrorDetails, isNetworkError } from "@/lib/auth-errors"; import { getPostLoginRedirectPath } from "@/lib/auth-utils"; import { ValidationError } from "@/lib/error"; -import { trackLoginAttempt, trackLoginFailure, trackLoginSuccess } from "@/lib/posthog/events"; +import { trackLoginAttempt, trackLoginFailure } from "@/lib/posthog/events"; export function LocalLoginForm() { const t = useTranslations("auth"); @@ -45,8 +45,8 @@ export function LocalLoginForm() { grant_type: "password", }); - // Track successful login - trackLoginSuccess("local"); + // auth_login_success is now emitted server-side + // (UserManager.on_after_login) — authoritative vs. optimistic. // Small delay to show success message setTimeout(() => { diff --git a/surfsense_web/app/(home)/register/page.tsx b/surfsense_web/app/(home)/register/page.tsx index 571103e79..5c91c7064 100644 --- a/surfsense_web/app/(home)/register/page.tsx +++ b/surfsense_web/app/(home)/register/page.tsx @@ -15,11 +15,7 @@ import { Spinner } from "@/components/ui/spinner"; import { useSession } from "@/hooks/use-session"; import { getAuthErrorDetails, isNetworkError, shouldRetry } from "@/lib/auth-errors"; import { AppError, ValidationError } from "@/lib/error"; -import { - trackRegistrationAttempt, - trackRegistrationFailure, - trackRegistrationSuccess, -} from "@/lib/posthog/events"; +import { trackRegistrationAttempt, trackRegistrationFailure } from "@/lib/posthog/events"; import { AmbientBackground } from "../login/AmbientBackground"; export default function RegisterPage() { @@ -81,8 +77,8 @@ export default function RegisterPage() { is_verified: false, }); - // Track successful registration - trackRegistrationSuccess(); + // auth_registration_success is now emitted server-side + // (UserManager.on_after_register) — authoritative vs. optimistic. // Success toast toast.success(t("register_success"), { diff --git a/surfsense_web/app/dashboard/[workspace_id]/team/team-content.tsx b/surfsense_web/app/dashboard/[workspace_id]/team/team-content.tsx index 571512305..dc45462e8 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/team/team-content.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/team/team-content.tsx @@ -96,7 +96,6 @@ import type { Role } from "@/contracts/types/roles.types"; import { invitesApiService } from "@/lib/apis/invites-api.service"; import { rolesApiService } from "@/lib/apis/roles-api.service"; import { formatRelativeDate } from "@/lib/format-date"; -import { trackWorkspaceInviteSent, trackWorkspaceUsersViewed } from "@/lib/posthog/events"; import { cacheKeys } from "@/lib/query-client/cache-keys"; import { cn } from "@/lib/utils"; @@ -226,12 +225,7 @@ export function TeamContent({ workspaceId }: TeamContentProps) { const canPrev = pageIndex > 0; const canNext = displayEnd < totalItems; - useEffect(() => { - if (members.length > 0 && !membersLoading) { - const ownerCount = members.filter((m) => m.is_owner).length; - trackWorkspaceUsersViewed(workspaceId, members.length, ownerCount); - } - }, [members, membersLoading, workspaceId]); + // workspace_users_viewed removed — redundant with $pageview. if (accessLoading || membersLoading) { return ( @@ -342,11 +336,7 @@ export function TeamContent({ workspaceId }: TeamContentProps) { Invite members ) : ( - + )} {invitesLoading ? (
); } @@ -190,6 +195,7 @@ export function SchemaForm({ disabled, getFieldOptions, fieldErrors, + fieldWarnings, }: SchemaFormProps) { const [showAdvanced, setShowAdvanced] = useState(false); @@ -209,6 +215,7 @@ export function SchemaForm({ onChange={(value) => onChange(field.name, value)} disabled={disabled} error={fieldErrors?.[field.name]} + warning={fieldWarnings?.[field.name]} options={getFieldOptions?.(field)} /> ))} @@ -239,6 +246,7 @@ export function SchemaForm({ onChange={(value) => onChange(field.name, value)} disabled={disabled} error={fieldErrors?.[field.name]} + warning={fieldWarnings?.[field.name]} options={getFieldOptions?.(field)} /> ))} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx index b7fbae91c..5a4540bcf 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/webcrawler-config.tsx @@ -8,8 +8,18 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Textarea } from "@/components/ui/textarea"; +import { isHttpUrl } from "@/lib/url"; +import { cn } from "@/lib/utils"; import type { ConnectorConfigProps } from "../index"; +/** Return the malformed (non-http/https) lines from a newline-separated URL list. */ +function invalidUrlLines(value: string): string[] { + return value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line && !isHttpUrl(line)); +} + export const WebcrawlerConfig: FC = ({ connector, onConfigChange }) => { // Initialize with existing config values const existingApiKey = (connector.config?.FIRECRAWL_API_KEY as string | undefined) || ""; @@ -19,6 +29,8 @@ export const WebcrawlerConfig: FC = ({ connector, onConfig const [initialUrls, setInitialUrls] = useState(existingUrls); const [showApiKey, setShowApiKey] = useState(false); + const invalidUrls = invalidUrlLines(initialUrls); + const handleApiKeyChange = (value: string) => { setApiKey(value); if (onConfigChange) { @@ -108,11 +120,21 @@ export const WebcrawlerConfig: FC = ({ connector, onConfig placeholder="https://example.com https://docs.example.com https://blog.example.com" value={initialUrls} onChange={(e) => handleUrlsChange(e.target.value)} - className="min-h-[100px] font-mono text-xs sm:text-sm bg-slate-400/5 dark:bg-white/5 border-slate-400/20 resize-none" + aria-invalid={invalidUrls.length > 0} + className={cn( + "min-h-[100px] font-mono text-xs sm:text-sm bg-slate-400/5 dark:bg-white/5 border-slate-400/20 resize-none", + invalidUrls.length > 0 && "border-destructive" + )} /> -

- Enter URLs to crawl (one per line). You can add more URLs later. -

+ {invalidUrls.length > 0 ? ( +

+ Not valid http(s) URLs: {invalidUrls.join(", ")} +

+ ) : ( +

+ Enter URLs to crawl (one per line). You can add more URLs later. +

+ )}
{/* Info Alert */} diff --git a/surfsense_web/lib/url.ts b/surfsense_web/lib/url.ts index 7bb0488d0..c351d1b18 100644 --- a/surfsense_web/lib/url.ts +++ b/surfsense_web/lib/url.ts @@ -12,3 +12,13 @@ export function tryGetHostname(url: string): string | undefined { return undefined; } } + +/** True when the value parses as an http(s) URL — mirrors the backend's boundary rule. */ +export function isHttpUrl(value: string): boolean { + try { + const { protocol } = new URL(value); + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } +} From 0118744c20380e0efc1cb82c83572b73df21fb55 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 19:39:15 +0200 Subject: [PATCH 167/217] fix: strip 'body' root from validation error summary --- surfsense_backend/app/app.py | 7 ++- .../tests/unit/test_error_contract.py | 44 ++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index de3b066f6..9f88fe02a 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -243,7 +243,12 @@ def _validation_error_handler( ] def _segment(field: dict[str, Any]) -> str: - loc = " -> ".join(field["loc"]) + # Drop the "body" request root so model-level errors read as a plain + # sentence and field errors read as "field -> sub", not "body -> field". + path = field["loc"] + if path and path[0] == "body": + path = path[1:] + loc = " -> ".join(path) return f"{loc}: {field['msg']}" if loc else field["msg"] summary = "; ".join(_segment(f) for f in fields) diff --git a/surfsense_backend/tests/unit/test_error_contract.py b/surfsense_backend/tests/unit/test_error_contract.py index ec8021290..e1e160232 100644 --- a/surfsense_backend/tests/unit/test_error_contract.py +++ b/surfsense_backend/tests/unit/test_error_contract.py @@ -14,6 +14,7 @@ import json import pytest from fastapi import HTTPException +from pydantic import BaseModel, model_validator from starlette.testclient import TestClient from app.exceptions import ( @@ -32,6 +33,23 @@ from app.exceptions import ( pytestmark = pytest.mark.unit +class _RequireOne(BaseModel): + """Body model with a cross-field rule, used to test the 422 summary. + + Defined at module scope (not inside the app factory) so FastAPI's ``Body`` + ``TypeAdapter`` can resolve the forward reference. + """ + + a: str | None = None + b: str | None = None + + @model_validator(mode="after") + def _at_least_one(self): + if not self.a and not self.b: + raise ValueError("Provide at least one of 'a' or 'b'.") + return self + + # --------------------------------------------------------------------------- # Helpers - lightweight FastAPI app that re-uses the real global handlers # --------------------------------------------------------------------------- @@ -39,7 +57,7 @@ pytestmark = pytest.mark.unit def _make_test_app(): """Build a minimal FastAPI app with the same handlers as the real one.""" - from fastapi import FastAPI + from fastapi import Body, FastAPI from fastapi.exceptions import RequestValidationError from pydantic import BaseModel @@ -124,6 +142,10 @@ def _make_test_app(): async def validated(item: Item): return item.model_dump() + @app.post("/require-one") + async def require_one(payload: _RequireOne = Body(...)): + return payload.model_dump() + return app @@ -277,6 +299,26 @@ class TestValidationErrorHandler: body = _assert_envelope(resp, 422) assert body["error"]["code"] == "VALIDATION_ERROR" + def test_field_error_drops_body_root(self, client): + # The "body" request root is noise in the human summary; the field name + # is kept so clients can still attach the error inline. + resp = client.post("/require-one", json={"a": 123}) + body = _assert_envelope(resp, 422) + msg = body["error"]["message"] + assert "body" not in msg + assert "a:" in msg + assert body["error"]["fields"][0]["loc"] == ["body", "a"] + + def test_model_level_error_reads_as_sentence(self, client): + # A cross-field rule (loc == ["body"]) must read as a plain sentence, + # not "body: ...", while fields still carry the location for clients. + resp = client.post("/require-one", json={}) + body = _assert_envelope(resp, 422) + assert body["error"]["message"] == ( + "Validation failed: Provide at least one of 'a' or 'b'." + ) + assert body["error"]["fields"][0]["loc"] == ["body"] + # --------------------------------------------------------------------------- # SurfSenseError class hierarchy unit tests From 55676cfe942f1b84b1735a6c8f268d0e1d3362df Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 19:39:15 +0200 Subject: [PATCH 168/217] fix: skip logging and telemetry for handled 4xx errors --- surfsense_web/lib/apis/base-api.service.ts | 46 ++++++++++++++-------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts index 872c885d9..867dea93e 100644 --- a/surfsense_web/lib/apis/base-api.service.ts +++ b/surfsense_web/lib/apis/base-api.service.ts @@ -293,24 +293,36 @@ class BaseApiService { ); } - console.error("Request failed:", JSON.stringify(error)); - if (!(error instanceof AuthenticationError)) { - import("posthog-js") - .then(({ default: posthog }) => { - posthog.captureException(error, { - api_url: url, - api_method: options?.method ?? "GET", - ...(error instanceof AppError && { - status_code: error.status, - status_text: error.statusText, - error_code: error.code, - request_id: error.requestId, - }), + // Handled client errors (validation, credits, auth, not-found, ...) are + // expected outcomes surfaced to the user via typed errors + toasts — not + // app faults. Skip logging/telemetry for them so they don't spam the + // console, PostHog, or Next.js's dev error overlay. + const isHandledClientError = + error instanceof AppError && + typeof error.status === "number" && + error.status >= 400 && + error.status < 500; + + if (!isHandledClientError) { + console.error("Request failed:", JSON.stringify(error)); + if (!(error instanceof AuthenticationError)) { + import("posthog-js") + .then(({ default: posthog }) => { + posthog.captureException(error, { + api_url: url, + api_method: options?.method ?? "GET", + ...(error instanceof AppError && { + status_code: error.status, + status_text: error.statusText, + error_code: error.code, + request_id: error.requestId, + }), + }); + }) + .catch(() => { + console.error("Failed to capture exception in PostHog"); }); - }) - .catch(() => { - console.error("Failed to capture exception in PostHog"); - }); + } } throw error; } From 44149323466b0ad46f0cae6da3e66dc4c755d2f4 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 23 Jul 2026 19:39:15 +0200 Subject: [PATCH 169/217] feat: surface playground run errors inline and persist toasts --- .../components/playground-runner.tsx | 14 +++++++++++-- .../playground/components/schema-form.tsx | 21 ++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx index 6fbf7b98b..6bee3e4e5 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx @@ -209,11 +209,21 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn } if (run.status === "error") { - setFieldErrors(fieldErrorsFromError(run.error)); + const nextFieldErrors = fieldErrorsFromError(run.error); + setFieldErrors(nextFieldErrors); const key = `${run.runId ?? "run"}:error`; if (notifiedRunRef.current === key) return; notifiedRunRef.current = key; - toast.error(getRunErrorMessage(run.error)); + // Field-level failures are shown inline (the form reveals + focuses the + // first one), so only toast global failures that have no field to anchor to. + // Keep run errors until dismissed — they're actionable and shouldn't + // vanish before the user reads them. + if (Object.keys(nextFieldErrors).length === 0) { + toast.error(getRunErrorMessage(run.error), { + duration: Number.POSITIVE_INFINITY, + closeButton: true, + }); + } } }, [run.status, run.runId, run.error]); diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx index c86dd9839..b8eb72c66 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/schema-form.tsx @@ -1,7 +1,7 @@ "use client"; import { ChevronDown } from "lucide-react"; -import { useMemo, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { Badge } from "@/components/ui/badge"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; @@ -205,6 +205,25 @@ export function SchemaForm({ return { primary: primaryFields, advanced: advancedFields }; }, [fields]); + // First invalid field in display order; drives reveal + focus below. + const firstErrorName = useMemo( + () => (fieldErrors ? fields.find((f) => fieldErrors[f.name])?.name : undefined), + [fields, fieldErrors] + ); + + // Reveal the section holding an invalid field, then move focus to it, so an + // error is never left hidden inside the collapsed "Advanced" group. + useEffect(() => { + if (!firstErrorName) return; + if (advanced.some((f) => f.name === firstErrorName)) setShowAdvanced(true); + const raf = requestAnimationFrame(() => { + const el = document.getElementById(`field-${firstErrorName}`); + el?.focus(); + el?.scrollIntoView({ block: "center", behavior: "smooth" }); + }); + return () => cancelAnimationFrame(raf); + }, [firstErrorName, advanced]); + return (
{primary.map((field) => ( From 9c16b7f40748b105d81591fe3b24d69e11f5bcf6 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:37:16 +0530 Subject: [PATCH 170/217] feat(connectors): add dedicated connectors route with sidebar panel --- .../[workspace_id]/connectors/page.tsx | 5 + .../connector-popup/connectors-panel.tsx | 548 ++++++++++++++++++ 2 files changed, 553 insertions(+) create mode 100644 surfsense_web/app/dashboard/[workspace_id]/connectors/page.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connectors-panel.tsx diff --git a/surfsense_web/app/dashboard/[workspace_id]/connectors/page.tsx b/surfsense_web/app/dashboard/[workspace_id]/connectors/page.tsx new file mode 100644 index 000000000..d43a54d3d --- /dev/null +++ b/surfsense_web/app/dashboard/[workspace_id]/connectors/page.tsx @@ -0,0 +1,5 @@ +import { ConnectorsSection } from "@/components/assistant-ui/connector-popup/connectors-panel"; + +export default function ConnectorsPage() { + return ; +} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connectors-panel.tsx b/surfsense_web/components/assistant-ui/connector-popup/connectors-panel.tsx new file mode 100644 index 000000000..4e0e8b4b1 --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connectors-panel.tsx @@ -0,0 +1,548 @@ +"use client"; + +import { useAtomValue } from "jotai"; +import { ChevronRight, LayoutGrid, Search, TriangleAlert, X } from "lucide-react"; +import { type ReactNode, useMemo, useState } from "react"; +import { statusInboxItemsAtom } from "@/atoms/inbox/status-inbox.atom"; +import { activeWorkspaceIdAtom } from "@/atoms/workspaces/workspace-query.atoms"; +import { Button } from "@/components/ui/button"; +import { + Drawer, + DrawerContent, + DrawerHandle, + DrawerTitle, + DrawerTrigger, +} from "@/components/ui/drawer"; +import { Separator } from "@/components/ui/separator"; +import { Spinner } from "@/components/ui/spinner"; +import { EnumConnectorName } from "@/contracts/enums/connector"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; +import { connectorIndexingMetadata } from "@/contracts/types/inbox.types"; +import { useConnectorsSync } from "@/hooks/use-connectors-sync"; +import { useZeroDocumentTypeCounts } from "@/hooks/use-zero-document-type-counts"; +import { cn } from "@/lib/utils"; +import { ConnectorConnectView } from "./connector-configs/views/connector-connect-view"; +import { ConnectorEditView } from "./connector-configs/views/connector-edit-view"; +import { IndexingConfigurationView } from "./connector-configs/views/indexing-configuration-view"; +import { COMPOSIO_CONNECTORS, getConnectorTitle, OAUTH_CONNECTORS } from "./constants/connector-constants"; +import { useConnectorDialog } from "./hooks/use-connector-dialog"; +import { useIndexingConnectors } from "./hooks/use-indexing-connectors"; +import { AllConnectorsTab } from "./tabs/all-connectors-tab"; +import { ConnectorAccountsListView } from "./views/connector-accounts-list-view"; +import { YouTubeCrawlerView } from "./views/youtube-crawler-view"; + +/** Health of a connected connector type, derived from indexing + inbox state. */ +type ConnectorHealth = "syncing" | "failed" | "ok"; + +interface ConnectedRow { + type: string; + title: string; + connectors: SearchSourceConnector[]; + accountCount: number; + health: ConnectorHealth; + errorMessage?: string; +} + +/** + * Connector management surface (single `/connectors` route) rendered inside the + * workspace panel. Stateful master–detail: + * - sub-rail: Overview + a flat "Your connectors" list (only what's connected, + * each with a live status glyph) + an "Add MCP server" action; + * - detail pane: the flat catalog (search + cards) OR one of the existing flow + * views (connect / edit / indexing / accounts / YouTube), reused verbatim + * from the former dialog. + * + * Unconnected connectors are one click from the catalog cards, so they never + * clutter the rail. The hook's internal `isOpen`/`connectorDialogOpenAtom` is + * inert on a route and ignored. + */ +export function ConnectorsSection() { + const workspaceId = useAtomValue(activeWorkspaceIdAtom); + const documentTypeCounts = useZeroDocumentTypeCounts(workspaceId); + const statusInboxItems = useAtomValue(statusInboxItemsAtom); + const [drawerOpen, setDrawerOpen] = useState(false); + + const inboxItems = useMemo( + () => statusInboxItems.filter((item) => item.type === "connector_indexing"), + [statusInboxItems] + ); + + const { + connectingId, + searchQuery, + indexingConfig, + indexingConnector, + indexingConnectorConfig, + editingConnector, + connectingConnectorType, + isCreatingConnector, + startDate, + endDate, + isStartingIndexing, + isSaving, + isDisconnecting, + periodicEnabled, + frequencyMinutes, + enableVisionLlm, + allConnectors, + viewingAccountsType, + viewingMCPList, + isYouTubeView, + isFromOAuth, + setSearchQuery, + setStartDate, + setEndDate, + setPeriodicEnabled, + setFrequencyMinutes, + setEnableVisionLlm, + handleOpenChange, + handleConnectOAuth, + handleConnectNonOAuth, + handleCreateWebcrawler, + handleCreateYouTubeCrawler, + handleSubmitConnectForm, + handleStartIndexing, + handleSkipIndexing, + handleStartEdit, + handleSaveConnector, + handleDisconnectConnector, + handleBackFromEdit, + handleBackFromConnect, + handleBackFromYouTube, + handleViewAccountsList, + handleBackFromAccountsList, + handleViewMCPList, + handleBackFromMCPList, + handleAddNewMCPFromList, + handleQuickIndexConnector, + connectorConfig, + setConnectorConfig, + setIndexingConnectorConfig, + setConnectorName, + } = useConnectorDialog(); + + const { + connectors: connectorsFromSync = [], + loading: connectorsLoading, + error: connectorsError, + refreshConnectors: refreshConnectorsSync, + } = useConnectorsSync(workspaceId); + + const useSyncData = connectorsFromSync.length > 0 || (connectorsLoading && !connectorsError); + const connectors = (useSyncData ? connectorsFromSync : allConnectors || []) as SearchSourceConnector[]; + + const refreshConnectors = async () => { + if (useSyncData) { + await refreshConnectorsSync(); + } + }; + + const { indexingConnectorIds, startIndexing, stopIndexing } = useIndexingConnectors( + connectors, + inboxItems + ); + + const connectedTypes = useMemo( + () => new Set(connectors.map((c) => c.connector_type)), + [connectors] + ); + + // Latest indexing status per connector id, parsed from the status inbox. Used + // to drive the "Your connectors" status glyphs (syncing / failed). + const statusByConnectorId = useMemo(() => { + const map = new Map(); + for (const item of inboxItems) { + const parsed = connectorIndexingMetadata.safeParse(item.metadata); + if (!parsed.success) continue; + const at = item.updated_at ?? item.created_at; + const prev = map.get(parsed.data.connector_id); + if (!prev || at > prev.at) { + map.set(parsed.data.connector_id, { + status: parsed.data.status, + error: parsed.data.error_message ?? undefined, + at, + }); + } + } + return map; + }, [inboxItems]); + + // Flat "Your connectors" rows: one per connected type, alphabetized, with a + // derived health (syncing > failed > ok) and account count. + const connectedRows = useMemo(() => { + const byType = new Map(); + for (const c of connectors) { + const arr = byType.get(c.connector_type) ?? []; + arr.push(c); + byType.set(c.connector_type, arr); + } + return [...byType.entries()] + .map(([type, list]) => { + const ids = list.map((c) => c.id); + const syncing = ids.some( + (id) => indexingConnectorIds.has(id) || statusByConnectorId.get(id)?.status === "in_progress" + ); + const failedEntry = syncing + ? undefined + : ids.map((id) => statusByConnectorId.get(id)).find((s) => s?.status === "failed"); + return { + type, + title: + type === EnumConnectorName.MCP_CONNECTOR ? "MCP Servers" : getConnectorTitle(type), + connectors: list, + accountCount: list.length, + health: syncing ? "syncing" : failedEntry ? "failed" : "ok", + errorMessage: failedEntry?.error, + } satisfies ConnectedRow; + }) + .sort((a, b) => a.title.localeCompare(b.title)); + }, [connectors, indexingConnectorIds, statusByConnectorId]); + + if (!workspaceId) return null; + + const activeConnectorType = editingConnector + ? editingConnector.connector_type + : connectingConnectorType || + viewingAccountsType?.connectorType || + (viewingMCPList ? EnumConnectorName.MCP_CONNECTOR : null); + + const flowView = ((): ReactNode => { + if (isYouTubeView) { + return ; + } + if (viewingMCPList) { + return ( + + ); + } + if (viewingAccountsType) { + return ( + { + const oauthConnector = + OAUTH_CONNECTORS.find( + (c) => c.connectorType === viewingAccountsType.connectorType + ) || + COMPOSIO_CONNECTORS.find( + (c) => c.connectorType === viewingAccountsType.connectorType + ); + if (oauthConnector) { + handleConnectOAuth(oauthConnector); + } + }} + isConnecting={connectingId !== null} + /> + ); + } + if (connectingConnectorType) { + return ( + handleSubmitConnectForm(formData, startIndexing)} + onBack={handleBackFromConnect} + isSubmitting={isCreatingConnector} + /> + ); + } + if (editingConnector) { + return ( + c.id === editingConnector.id)?.last_indexed_at ?? + editingConnector.last_indexed_at, + }} + startDate={startDate} + endDate={endDate} + periodicEnabled={periodicEnabled} + frequencyMinutes={frequencyMinutes} + enableVisionLlm={enableVisionLlm} + isSaving={isSaving} + isDisconnecting={isDisconnecting} + isIndexing={indexingConnectorIds.has(editingConnector.id)} + workspaceId={workspaceId?.toString()} + onStartDateChange={setStartDate} + onEndDateChange={setEndDate} + onPeriodicEnabledChange={setPeriodicEnabled} + onFrequencyChange={setFrequencyMinutes} + onEnableVisionLlmChange={setEnableVisionLlm} + onSave={() => { + startIndexing(editingConnector.id); + handleSaveConnector(() => refreshConnectors()); + }} + onDisconnect={() => handleDisconnectConnector(() => refreshConnectors())} + onBack={handleBackFromEdit} + onQuickIndex={(() => { + const cfg = connectorConfig || editingConnector.config; + const isDriveOrOneDrive = + editingConnector.connector_type === "GOOGLE_DRIVE_CONNECTOR" || + editingConnector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" || + editingConnector.connector_type === "ONEDRIVE_CONNECTOR" || + editingConnector.connector_type === "DROPBOX_CONNECTOR"; + const hasDriveItems = isDriveOrOneDrive + ? ((cfg?.selected_folders as unknown[]) ?? []).length > 0 || + ((cfg?.selected_files as unknown[]) ?? []).length > 0 + : true; + if (!hasDriveItems) return undefined; + return () => { + startIndexing(editingConnector.id); + handleQuickIndexConnector( + editingConnector.id, + editingConnector.connector_type, + stopIndexing, + startDate, + endDate + ); + }; + })()} + onConfigChange={setConnectorConfig} + onNameChange={setConnectorName} + /> + ); + } + if (indexingConfig) { + return ( + { + if (indexingConfig.connectorId) { + startIndexing(indexingConfig.connectorId); + } + handleStartIndexing(() => refreshConnectors()); + }} + onSkip={handleSkipIndexing} + /> + ); + } + return null; + })(); + + const isFlowActive = flowView !== null; + + const detailTitle = + isFlowActive && activeConnectorType ? getConnectorTitle(activeConnectorType) : "Overview"; + + const hasConnected = connectedRows.length > 0; + + // Rail navigation: clear any open flow first (handleOpenChange(false) is the + // hook's full reset), then apply the target management action. + const resetFlow = () => handleOpenChange(false); + const goOverview = () => { + setDrawerOpen(false); + resetFlow(); + }; + const manageRow = (row: ConnectedRow) => { + setDrawerOpen(false); + resetFlow(); + if (row.type === EnumConnectorName.MCP_CONNECTOR) { + handleViewMCPList(); + return; + } + if (row.accountCount > 1) { + handleViewAccountsList(row.type, row.title); + return; + } + handleStartEdit(row.connectors[0]); + }; + const addMcpServer = () => { + setDrawerOpen(false); + resetFlow(); + handleConnectNonOAuth?.(EnumConnectorName.MCP_CONNECTOR); + }; + + const navBtnClass = (active: boolean) => + cn( + "inline-flex w-full items-center justify-start gap-3 rounded-md px-3 py-2.5 text-left text-sm font-medium transition-colors duration-150 focus:outline-none", + active + ? "bg-accent text-accent-foreground" + : "text-muted-foreground hover:bg-accent hover:text-accent-foreground" + ); + + const railLabel = (text: string) => ( +

{text}

+ ); + + const renderNav = () => ( + <> + + + {hasConnected && ( + <> + + {railLabel("Your connectors")} +
+ {connectedRows.map((row) => { + const isActive = isFlowActive && activeConnectorType === row.type; + return ( + + ); + })} +
+ + )} + + + + + ); + + const catalog = ( +
+
+
+ + setSearchQuery(e.target.value)} + /> + {searchQuery && ( + + )} +
+
+ + +
+ ); + + return ( +
+
+

+ Connectors +

+ + + + + + + + Connectors navigation + + + +
+ +
+
+

{detailTitle}

+ +
+
{isFlowActive ? flowView : catalog}
+
+
+ ); +} From e705a73ae952f25744e24b805f482ebbfb0d57c5 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:37:26 +0530 Subject: [PATCH 171/217] feat(sidebar): add Connectors nav item to workspace sidebar --- .../layout/providers/LayoutDataProvider.tsx | 14 ++++++++++++-- .../components/layout/ui/sidebar/Sidebar.tsx | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 511c095e5..cfd970f13 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -2,7 +2,7 @@ import { useQuery } from "@tanstack/react-query"; import { useAtomValue, useSetAtom } from "jotai"; -import { AlarmClock, AlertTriangle, Shapes, SquareTerminal } from "lucide-react"; +import { AlarmClock, AlertTriangle, Shapes, SquareTerminal, Unplug } from "lucide-react"; import { useParams, usePathname, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useTheme } from "next-themes"; @@ -300,6 +300,7 @@ export function LayoutDataProvider({ const isAutomationsActive = pathname?.includes("/automations") === true; const isArtifactsActive = pathname?.endsWith("/artifacts") === true; const isPlaygroundRoute = pathname?.includes("/playground") === true; + const isConnectorsRoute = pathname?.includes("/connectors") === true; const navItems: NavItem[] = useMemo( () => ( @@ -316,6 +317,12 @@ export function LayoutDataProvider({ icon: Shapes, isActive: isArtifactsActive, }, + { + title: "Connectors", + url: `/dashboard/${workspaceId}/connectors`, + icon: Unplug, + isActive: isConnectorsRoute, + }, { title: "Playground", url: `/dashboard/${workspaceId}/playground`, @@ -324,7 +331,7 @@ export function LayoutDataProvider({ }, ] as (NavItem | null)[] ).filter((item): item is NavItem => item !== null), - [workspaceId, isAutomationsActive, isArtifactsActive, isPlaygroundRoute] + [workspaceId, isAutomationsActive, isArtifactsActive, isConnectorsRoute, isPlaygroundRoute] ); // Handlers @@ -631,6 +638,7 @@ export function LayoutDataProvider({ const isAutomationsPage = pathname?.includes("/automations") === true; const isArtifactsPage = pathname?.endsWith("/artifacts") === true; const isPlaygroundPage = pathname?.includes("/playground") === true; + const isConnectorsPage = pathname?.includes("/connectors") === true; const isAllChatsPage = pathname?.endsWith("/chats") === true; const handleChatsClick = useCallback(() => { router.push(`/dashboard/${workspaceId}/chats`); @@ -650,6 +658,7 @@ export function LayoutDataProvider({ isAutomationsPage || isArtifactsPage || isPlaygroundPage || + isConnectorsPage || isAllChatsPage; return ( @@ -702,6 +711,7 @@ export function LayoutDataProvider({ isAutomationsPage || isArtifactsPage || isPlaygroundPage || + isConnectorsPage || isAllChatsPage ? "items-start justify-center px-6 py-8 md:px-10 md:pb-10 md:pt-16" : undefined diff --git a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx index e24523448..0d280dc28 100644 --- a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx @@ -140,6 +140,10 @@ export function Sidebar({ () => navItems.find((item) => item.url.endsWith("/artifacts")), [navItems] ); + const connectorsItem = useMemo( + () => navItems.find((item) => item.url.endsWith("/connectors")), + [navItems] + ); const playgroundItem = useMemo( () => navItems.find((item) => item.url.endsWith("/playground")), [navItems] @@ -150,6 +154,7 @@ export function Sidebar({ (item) => !item.url.endsWith("/automations") && !item.url.endsWith("/artifacts") && + !item.url.endsWith("/connectors") && !item.url.endsWith("/playground") ), [navItems] @@ -254,6 +259,16 @@ export function Sidebar({ tooltipContent={isCollapsed ? artifactsItem.title : undefined} /> )} + {connectorsItem && ( + onNavItemClick?.(connectorsItem)} + isCollapsed={isCollapsed} + isActive={connectorsItem.isActive} + tooltipContent={isCollapsed ? connectorsItem.title : undefined} + /> + )} {playgroundItem && ( Date: Thu, 23 Jul 2026 23:37:30 +0530 Subject: [PATCH 172/217] fix(connectors): land OAuth callback on connectors panel to resume flow --- .../app/dashboard/[workspace_id]/connectors/callback/route.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/surfsense_web/app/dashboard/[workspace_id]/connectors/callback/route.ts b/surfsense_web/app/dashboard/[workspace_id]/connectors/callback/route.ts index 09a2c2f2c..857075e58 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/connectors/callback/route.ts +++ b/surfsense_web/app/dashboard/[workspace_id]/connectors/callback/route.ts @@ -19,7 +19,9 @@ export async function GET( const response = new NextResponse(null, { status: 302, headers: { - Location: `/dashboard/${workspace_id}/new-chat`, + // Land on the connectors panel so `useConnectorDialog` (mounted there) + // consumes the result cookie and continues the indexing/edit flow. + Location: `/dashboard/${workspace_id}/connectors`, }, }); response.cookies.set(OAUTH_RESULT_COOKIE, result, { From 132fd1e5d63a379062fee838f87cec46e20d6572 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:37:40 +0530 Subject: [PATCH 173/217] refactor(connectors): remove legacy ConnectorIndicator modal --- .../[workspace_id]/client-layout.tsx | 2 - .../assistant-ui/connector-popup.tsx | 388 ------------------ .../assistant-ui/connector-popup/index.ts | 3 - 3 files changed, 393 deletions(-) delete mode 100644 surfsense_web/components/assistant-ui/connector-popup.tsx diff --git a/surfsense_web/app/dashboard/[workspace_id]/client-layout.tsx b/surfsense_web/app/dashboard/[workspace_id]/client-layout.tsx index 54fc9d000..fb76af227 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/client-layout.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/client-layout.tsx @@ -8,7 +8,6 @@ import { useEffect } from "react"; import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom"; import { llmSetupStatusAtomFamily } from "@/atoms/model-connections/model-connections-query.atoms"; import { activeWorkspaceIdAtom } from "@/atoms/workspaces/workspace-query.atoms"; -import { ConnectorIndicator } from "@/components/assistant-ui/connector-popup"; import { DocumentUploadDialogProvider } from "@/components/assistant-ui/document-upload-popup"; import { LayoutDataProvider } from "@/components/layout"; import { OnboardingTour } from "@/components/onboarding-tour"; @@ -169,7 +168,6 @@ export function DashboardClientLayout({ initialPlaygroundSidebarCollapsed={initialPlaygroundSidebarCollapsed} > {children} - ); diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx deleted file mode 100644 index a8bfc29bd..000000000 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ /dev/null @@ -1,388 +0,0 @@ -"use client"; - -import { useAtomValue } from "jotai"; -import { forwardRef, useEffect, useImperativeHandle, useMemo, useState } from "react"; -import { createPortal } from "react-dom"; -import { statusInboxItemsAtom } from "@/atoms/inbox/status-inbox.atom"; -import { activeWorkspaceIdAtom } from "@/atoms/workspaces/workspace-query.atoms"; -import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog"; -import { Tabs, TabsContent } from "@/components/ui/tabs"; -import type { SearchSourceConnector } from "@/contracts/types/connector.types"; -import { useConnectorsSync } from "@/hooks/use-connectors-sync"; -import { PICKER_CLOSE_EVENT, PICKER_OPEN_EVENT } from "@/hooks/use-google-picker"; -import { useZeroDocumentTypeCounts } from "@/hooks/use-zero-document-type-counts"; -import { ConnectorDialogHeader } from "./connector-popup/components/connector-dialog-header"; -import { ConnectorConnectView } from "./connector-popup/connector-configs/views/connector-connect-view"; -import { ConnectorEditView } from "./connector-popup/connector-configs/views/connector-edit-view"; -import { IndexingConfigurationView } from "./connector-popup/connector-configs/views/indexing-configuration-view"; -import { - COMPOSIO_CONNECTORS, - OAUTH_CONNECTORS, -} from "./connector-popup/constants/connector-constants"; -import { useConnectorDialog } from "./connector-popup/hooks/use-connector-dialog"; -import { useIndexingConnectors } from "./connector-popup/hooks/use-indexing-connectors"; -import { ActiveConnectorsTab } from "./connector-popup/tabs/active-connectors-tab"; -import { AllConnectorsTab } from "./connector-popup/tabs/all-connectors-tab"; -import { ConnectorAccountsListView } from "./connector-popup/views/connector-accounts-list-view"; -import { YouTubeCrawlerView } from "./connector-popup/views/youtube-crawler-view"; - -export interface ConnectorIndicatorHandle { - open: () => void; -} - -interface ConnectorIndicatorProps { - showTrigger?: boolean; -} - -export const ConnectorIndicator = forwardRef( - (_props, ref) => { - const workspaceId = useAtomValue(activeWorkspaceIdAtom); - - // Real-time document type counts via Zero (updates instantly as docs are indexed) - const documentTypeCounts = useZeroDocumentTypeCounts(workspaceId); - // Read status inbox items from shared atom (populated by LayoutDataProvider) - // instead of creating a duplicate useInbox("status") hook. - const statusInboxItems = useAtomValue(statusInboxItemsAtom); - const inboxItems = useMemo( - () => statusInboxItems.filter((item) => item.type === "connector_indexing"), - [statusInboxItems] - ); - - // Use the custom hook for dialog state management - const { - isOpen, - activeTab, - connectingId, - isScrolled, - searchQuery, - indexingConfig, - indexingConnector, - indexingConnectorConfig, - editingConnector, - connectingConnectorType, - isCreatingConnector, - startDate, - endDate, - isStartingIndexing, - isSaving, - isDisconnecting, - periodicEnabled, - frequencyMinutes, - enableVisionLlm, - allConnectors, - viewingAccountsType, - viewingMCPList, - isYouTubeView, - isFromOAuth, - setSearchQuery, - setStartDate, - setEndDate, - setPeriodicEnabled, - setFrequencyMinutes, - setEnableVisionLlm, - handleOpenChange, - handleTabChange, - handleScroll, - handleConnectOAuth, - handleConnectNonOAuth, - handleCreateWebcrawler, - handleCreateYouTubeCrawler, - handleSubmitConnectForm, - handleStartIndexing, - handleSkipIndexing, - handleStartEdit, - handleSaveConnector, - handleDisconnectConnector, - handleBackFromEdit, - handleBackFromConnect, - handleBackFromYouTube, - handleViewAccountsList, - handleBackFromAccountsList, - handleBackFromMCPList, - handleAddNewMCPFromList, - handleQuickIndexConnector, - connectorConfig, - setConnectorConfig, - setIndexingConnectorConfig, - setConnectorName, - } = useConnectorDialog(); - - const [pickerOpen, setPickerOpen] = useState(false); - useEffect(() => { - const onOpen = () => setPickerOpen(true); - const onClose = () => setPickerOpen(false); - window.addEventListener(PICKER_OPEN_EVENT, onOpen); - window.addEventListener(PICKER_CLOSE_EVENT, onClose); - return () => { - window.removeEventListener(PICKER_OPEN_EVENT, onOpen); - window.removeEventListener(PICKER_CLOSE_EVENT, onClose); - }; - }, []); - - const { - connectors: connectorsFromSync = [], - loading: connectorsLoading, - error: connectorsError, - refreshConnectors: refreshConnectorsSync, - } = useConnectorsSync(workspaceId); - - const useSyncData = connectorsFromSync.length > 0 || (connectorsLoading && !connectorsError); - const connectors = useSyncData ? connectorsFromSync : allConnectors || []; - - const refreshConnectors = async () => { - if (useSyncData) { - await refreshConnectorsSync(); - } - }; - - // Track indexing state locally - clears automatically when last_indexed_at changes via real-time sync - // Also clears when failed notifications are detected - const { indexingConnectorIds, startIndexing, stopIndexing } = useIndexingConnectors( - connectors as SearchSourceConnector[], - inboxItems - ); - - // Get document types that have documents in the workspace - const activeDocumentTypes = documentTypeCounts - ? Object.entries(documentTypeCounts).filter(([, count]) => count > 0) - : []; - - const hasConnectors = connectors.length > 0; - const hasSources = hasConnectors || activeDocumentTypes.length > 0; - const totalSourceCount = connectors.length + activeDocumentTypes.length; - - const activeConnectorsCount = connectors.length; - - // Check which connectors are already connected - // Real-time connector updates via Zero sync - const connectedTypes = new Set( - (connectors || []).map((c: SearchSourceConnector) => c.connector_type) - ); - - useImperativeHandle(ref, () => ({ - open: () => handleOpenChange(true), - })); - - if (!workspaceId) return null; - - return ( - - {isOpen && - createPortal( - - ); - } -); - -ConnectorIndicator.displayName = "ConnectorIndicator"; diff --git a/surfsense_web/components/assistant-ui/connector-popup/index.ts b/surfsense_web/components/assistant-ui/connector-popup/index.ts index f3209a777..8fd895e71 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/index.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/index.ts @@ -1,6 +1,3 @@ -// Main component export -export { ConnectorIndicator } from "../connector-popup"; - // Sub-components (if needed for external use) export { ConnectorCard } from "./components/connector-card"; export { ConnectorDialogHeader } from "./components/connector-dialog-header"; From 915ebcf2e74d2e3e9d521ed4357af54200e35309 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:37:48 +0530 Subject: [PATCH 174/217] refactor(chat): open connectors page from composer/banner and restyle tool-group headings --- .../components/assistant-ui/thread.tsx | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 73d31e11d..120af4bfe 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -44,7 +44,6 @@ import { clearPremiumAlertForThreadAtom, premiumAlertByThreadAtom, } from "@/atoms/chat/premium-alert.atom"; -import { connectorDialogOpenAtom } from "@/atoms/connector-dialog/connector-dialog.atoms"; import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { membersAtom } from "@/atoms/members/members-query.atoms"; import { llmSetupStatusAtomFamily } from "@/atoms/model-connections/model-connections-query.atoms"; @@ -328,7 +327,9 @@ const ConnectToolsBanner: FC<{ onVisibleChange?: (visible: boolean) => void; }> = ({ isThreadEmpty, onVisibleChange }) => { const { data: connectors } = useAtomValue(connectorsAtom); - const setConnectorDialogOpen = useSetAtom(connectorDialogOpenAtom); + const router = useRouter(); + const params = useParams(); + const bannerWorkspaceId = getWorkspaceIdNumber(params); const [dismissed, setDismissed] = useState(() => { if (typeof window === "undefined") return false; return localStorage.getItem(BANNER_DISMISSED_KEY) === "true"; @@ -371,7 +372,9 @@ const ConnectToolsBanner: FC<{ variant="ghost" size="sm" className="h-7 min-w-0 cursor-pointer justify-start gap-2 rounded-md px-0 text-[13px] font-normal text-muted-foreground select-none hover:bg-transparent hover:text-foreground" - onClick={() => setConnectorDialogOpen(true)} + onClick={() => { + if (bannerWorkspaceId) router.push(`/dashboard/${bannerWorkspaceId}/connectors`); + }} > Connect your tools @@ -1152,7 +1155,11 @@ const ComposerAction: FC = ({ onChatModelSelected, }) => { const mentionedDocuments = useAtomValue(mentionedDocumentsAtom); - const setConnectorDialogOpen = useSetAtom(connectorDialogOpenAtom); + const router = useRouter(); + const openConnectors = useCallback( + () => router.push(`/dashboard/${workspaceId}/connectors`), + [router, workspaceId] + ); const [toolsPopoverOpen, setToolsPopoverOpen] = useState(false); const [openConnectorSubmenu, setOpenConnectorSubmenu] = useState(null); const [expandedConnectorGroups, setExpandedConnectorGroups] = useState>( @@ -1302,7 +1309,7 @@ const ComposerAction: FC = ({ Upload Files - setConnectorDialogOpen(true)}> + MCP Connectors @@ -1327,7 +1334,7 @@ const ComposerAction: FC = ({
{regularToolGroups.map((group) => (
-
+
{group.label}
{group.tools.map((tool) => { @@ -1354,7 +1361,7 @@ const ComposerAction: FC = ({ ))} {connectorToolGroups.length > 0 && (
-
+
Connector Actions
{connectorToolGroups.map((group) => { @@ -1435,7 +1442,7 @@ const ComposerAction: FC = ({ )} {otherToolGroup && (
-
+
{otherToolGroup.label}
{otherToolGroup.tools.map((tool) => { @@ -1522,7 +1529,7 @@ const ComposerAction: FC = ({ Take a screenshot - setConnectorDialogOpen(true)}> + MCP Connectors @@ -1546,7 +1553,7 @@ const ComposerAction: FC = ({ > {regularToolGroups.map((group) => (
-
+
{group.label}
{group.tools.map((tool) => { @@ -1581,7 +1588,7 @@ const ComposerAction: FC = ({ ))} {connectorToolGroups.length > 0 && (
-
+
Connector Actions
{connectorToolGroups.map((group) => { @@ -1667,7 +1674,7 @@ const ComposerAction: FC = ({ )} {otherToolGroup && (
-
+
{otherToolGroup.label}
{otherToolGroup.tools.map((tool) => { From 6c1c75a3e1fc5a869a56c595db4e2b5d68f25394 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:37:51 +0530 Subject: [PATCH 175/217] feat(connector-icon): mask MCP icon to inherit currentColor --- surfsense_web/contracts/enums/connectorIcons.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index 81696729d..1e76d7e45 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -74,7 +74,17 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas case EnumConnectorName.CIRCLEBACK_CONNECTOR: return Circleback; case EnumConnectorName.MCP_CONNECTOR: - return MCP; + // Masked so the black glyph inherits currentColor (white in dark mode). + return ( +