diff --git a/surfsense_backend/alembic/versions/144_add_automation_tables.py b/surfsense_backend/alembic/versions/144_add_automation_tables.py new file mode 100644 index 000000000..8d836095d --- /dev/null +++ b/surfsense_backend/alembic/versions/144_add_automation_tables.py @@ -0,0 +1,179 @@ +"""Add automation tables (automations, automation_triggers, automation_runs) + +Revision ID: 144 +Revises: 143 +Create Date: 2026-05-26 + +Adds the three tables that back the v1 automation engine, plus the +three PostgreSQL ENUM types they reference. Matches the SQLAlchemy +models under ``app.automations.persistence.models`` and the v1 data +model in ``automation-design-plan.md`` §9. + +v1 ships these three tables only. ``domain_events`` is deferred to +Phase 3 with the event trigger; ``mcp_connections`` / ``mcp_tools`` +are deferred to Phase 4 with the MCP integration. +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "144" +down_revision: str | None = "143" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ENUM types (PostgreSQL requires types created before tables that use them) + op.execute( + """ + CREATE TYPE automation_status AS ENUM ( + 'active', 'paused', 'archived' + ); + """ + ) + op.execute( + """ + CREATE TYPE automation_trigger_type AS ENUM ( + 'schedule', 'manual' + ); + """ + ) + op.execute( + """ + CREATE TYPE automation_run_status AS ENUM ( + 'pending', 'running', 'succeeded', 'failed', + 'cancelled', 'timed_out' + ); + """ + ) + + # automations — the editable, versioned automation definition + op.execute( + """ + CREATE TABLE automations ( + id SERIAL PRIMARY KEY, + search_space_id INTEGER NOT NULL + REFERENCES searchspaces(id) ON DELETE CASCADE, + created_by_user_id UUID + REFERENCES "user"(id) ON DELETE SET NULL, + name VARCHAR(200) NOT NULL, + description TEXT, + status automation_status NOT NULL DEFAULT 'active', + definition JSONB NOT NULL, + version INTEGER NOT NULL DEFAULT 1, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() + ); + """ + ) + op.execute( + "CREATE INDEX ix_automations_search_space_id ON automations(search_space_id);" + ) + op.execute( + "CREATE INDEX ix_automations_created_by_user_id ON automations(created_by_user_id);" + ) + op.execute("CREATE INDEX ix_automations_status ON automations(status);") + op.execute("CREATE INDEX ix_automations_created_at ON automations(created_at);") + op.execute("CREATE INDEX ix_automations_updated_at ON automations(updated_at);") + + # automation_triggers — one row per (automation, trigger-instance) pair + op.execute( + """ + CREATE TABLE automation_triggers ( + id SERIAL PRIMARY KEY, + automation_id INTEGER NOT NULL + REFERENCES automations(id) ON DELETE CASCADE, + type automation_trigger_type NOT NULL, + params JSONB NOT NULL, + static_inputs JSONB NOT NULL DEFAULT '{}'::jsonb, + enabled BOOLEAN NOT NULL DEFAULT true, + last_fired_at TIMESTAMP WITH TIME ZONE, + next_fire_at TIMESTAMP WITH TIME ZONE, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() + ); + """ + ) + op.execute( + "CREATE INDEX ix_automation_triggers_automation_id ON automation_triggers(automation_id);" + ) + op.execute( + "CREATE INDEX ix_automation_triggers_type ON automation_triggers(type);" + ) + op.execute( + "CREATE INDEX ix_automation_triggers_enabled ON automation_triggers(enabled);" + ) + op.execute( + "CREATE INDEX ix_automation_triggers_created_at ON automation_triggers(created_at);" + ) + # Partial index for the schedule tick: only enabled schedule triggers + # with a scheduled next fire are ever scanned for due rows. + op.execute( + """ + CREATE INDEX ix_automation_triggers_due + ON automation_triggers (next_fire_at) + WHERE enabled = true + AND type = 'schedule' + AND next_fire_at IS NOT NULL; + """ + ) + + # automation_runs — the immutable per-fire execution record + op.execute( + """ + CREATE TABLE automation_runs ( + id SERIAL PRIMARY KEY, + automation_id INTEGER NOT NULL + REFERENCES automations(id) ON DELETE CASCADE, + trigger_id INTEGER + REFERENCES automation_triggers(id) ON DELETE SET NULL, + status automation_run_status NOT NULL DEFAULT 'pending', + definition_snapshot JSONB NOT NULL, + inputs JSONB NOT NULL DEFAULT '{}'::jsonb, + step_results JSONB NOT NULL DEFAULT '[]'::jsonb, + output JSONB, + artifacts JSONB NOT NULL DEFAULT '[]'::jsonb, + error JSONB, + started_at TIMESTAMP WITH TIME ZONE, + finished_at TIMESTAMP WITH TIME ZONE, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() + ); + """ + ) + op.execute( + "CREATE INDEX ix_automation_runs_automation_id ON automation_runs(automation_id);" + ) + op.execute( + "CREATE INDEX ix_automation_runs_trigger_id ON automation_runs(trigger_id);" + ) + op.execute("CREATE INDEX ix_automation_runs_status ON automation_runs(status);") + op.execute( + "CREATE INDEX ix_automation_runs_created_at ON automation_runs(created_at);" + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_automation_runs_created_at;") + op.execute("DROP INDEX IF EXISTS ix_automation_runs_status;") + op.execute("DROP INDEX IF EXISTS ix_automation_runs_trigger_id;") + op.execute("DROP INDEX IF EXISTS ix_automation_runs_automation_id;") + op.execute("DROP TABLE IF EXISTS automation_runs;") + + op.execute("DROP INDEX IF EXISTS ix_automation_triggers_due;") + op.execute("DROP INDEX IF EXISTS ix_automation_triggers_created_at;") + op.execute("DROP INDEX IF EXISTS ix_automation_triggers_enabled;") + op.execute("DROP INDEX IF EXISTS ix_automation_triggers_type;") + op.execute("DROP INDEX IF EXISTS ix_automation_triggers_automation_id;") + op.execute("DROP TABLE IF EXISTS automation_triggers;") + + op.execute("DROP INDEX IF EXISTS ix_automations_updated_at;") + op.execute("DROP INDEX IF EXISTS ix_automations_created_at;") + op.execute("DROP INDEX IF EXISTS ix_automations_status;") + op.execute("DROP INDEX IF EXISTS ix_automations_created_by_user_id;") + op.execute("DROP INDEX IF EXISTS ix_automations_search_space_id;") + op.execute("DROP TABLE IF EXISTS automations;") + + op.execute("DROP TYPE IF EXISTS automation_run_status;") + op.execute("DROP TYPE IF EXISTS automation_trigger_type;") + op.execute("DROP TYPE IF EXISTS automation_status;") diff --git a/surfsense_backend/alembic/versions/145_add_automations_permissions_to_roles.py b/surfsense_backend/alembic/versions/145_add_automations_permissions_to_roles.py new file mode 100644 index 000000000..779656b44 --- /dev/null +++ b/surfsense_backend/alembic/versions/145_add_automations_permissions_to_roles.py @@ -0,0 +1,87 @@ +"""Add automations permissions to existing Editor/Viewer roles + +Revision ID: 145 +Revises: 144 +Create Date: 2026-05-27 + +Owners already have ``*`` and need no backfill. Custom (non-system) roles +are left untouched on purpose: workspace admins manage those explicitly. +""" + +from collections.abc import Sequence + +from sqlalchemy import text + +from alembic import op + +revision: str = "145" +down_revision: str | None = "144" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +_EDITOR_PERMISSIONS = ( + "automations:create", + "automations:read", + "automations:update", + "automations:execute", +) +_VIEWER_PERMISSIONS = ("automations:read",) + + +def upgrade(): + connection = op.get_bind() + + for permission in _EDITOR_PERMISSIONS: + connection.execute( + text( + """ + UPDATE search_space_roles + SET permissions = array_append(permissions, :permission) + WHERE name = 'Editor' + AND NOT (:permission = ANY(permissions)) + """ + ), + {"permission": permission}, + ) + + for permission in _VIEWER_PERMISSIONS: + connection.execute( + text( + """ + UPDATE search_space_roles + SET permissions = array_append(permissions, :permission) + WHERE name = 'Viewer' + AND NOT (:permission = ANY(permissions)) + """ + ), + {"permission": permission}, + ) + + +def downgrade(): + connection = op.get_bind() + + for permission in _EDITOR_PERMISSIONS: + connection.execute( + text( + """ + UPDATE search_space_roles + SET permissions = array_remove(permissions, :permission) + WHERE name = 'Editor' + """ + ), + {"permission": permission}, + ) + + for permission in _VIEWER_PERMISSIONS: + connection.execute( + text( + """ + UPDATE search_space_roles + SET permissions = array_remove(permissions, :permission) + WHERE name = 'Viewer' + """ + ), + {"permission": permission}, + ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/__init__.py new file mode 100644 index 000000000..30699a4a1 --- /dev/null +++ b/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/__init__.py @@ -0,0 +1 @@ +"""``create_automation`` — description + few-shot examples.""" diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/description.md b/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/description.md new file mode 100644 index 000000000..ce6562c97 --- /dev/null +++ b/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/description.md @@ -0,0 +1,34 @@ +- `create_automation` — Draft and author a new automation. You describe the + user's intent; a focused drafter inside the tool turns it into the full + automation JSON; the user sees a preview on an approval card and chooses + approve or reject. All three phases happen in a single tool call. + - Call when the user wants SurfSense to do something on its own: anything + recurring or scheduled ("every morning…", "each Monday…", "weekly + recap…"). + - Args: + - `intent` (string): restate the user's request **concretely**, in one + paragraph. Cover three things: + - **What** should run (the action: summarize, recap, post, draft, …). + - **When** it should run (schedule + timezone if the user mentioned one; + otherwise leave the timezone for the drafter to default to UTC). + - **Static values** the automation needs (folder ids, channel names, + project keys, parent page ids, …) — list them with their values. + If the user did NOT supply one the automation needs, say so + explicitly ("the Notion parent page id was not specified") so the + drafter leaves a placeholder. + - Do NOT prompt the user to confirm before calling — the approval card + IS the confirmation. The card shows a structured preview plus the raw + JSON; it offers approve/reject only. If the user wants changes after + seeing the draft, they reply in chat and you call this tool again with + a refined `intent` — that's the edit path. + - Returns: + - `{status: "saved", automation_id, name}` — confirm briefly to the + user ("Saved as automation #N — runs ."). Don't dump JSON back. + - `{status: "rejected", message}` — the user declined on the card. + Acknowledge once ("Understood, I didn't create it.") and stop. Do + NOT retry or pitch variants without a fresh user request. + - `{status: "invalid", issues, raw?}` — drafting/validation failed + before the card was shown. Read the issues, refine your `intent` + with the missing details, call again. + - `{status: "error", message}` — surface the message verbatim and + offer to retry. diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/example.md b/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/example.md new file mode 100644 index 000000000..19311bef0 --- /dev/null +++ b/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/example.md @@ -0,0 +1,13 @@ + +user: "Every weekday at 9am, summarize new documents in folder 12 and post the summary to Slack channel #daily-digest." +→ create_automation(intent="Every weekday at 09:00 UTC, summarize documents added to folder_id=12 since the last run, then post the summary to Slack channel '#daily-digest'. Static inputs: folder_id=12, slack_channel='#daily-digest'.") +tool returns: {"status": "saved", "automation_id": 42, "name": "Daily folder 12 digest"} +(Reply briefly: "Saved as automation #42 — runs weekdays at 9am UTC.") + + + +user: "Once a week on Mondays at 7am Paris time, draft a Notion page recapping last week's Jira tickets in project CORE." +→ create_automation(intent="Every Monday at 07:00 Europe/Paris, read last week's Jira issues in project CORE, then draft a Notion page recapping them. Static inputs: jira_project_key='CORE'. The user did NOT specify which Notion page the recap should sit under — leave notion_parent_page_id as a placeholder.") +tool returns: {"status": "saved", "automation_id": 51, "name": "Weekly CORE Jira recap"} +(Reply: "Saved as automation #51. I left the Notion parent page id as a placeholder — set it on the automation before next Monday.") + diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/__init__.py new file mode 100644 index 000000000..d47bbac7e --- /dev/null +++ b/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/__init__.py @@ -0,0 +1,7 @@ +"""``create_automation`` — author + persist an automation via a HITL card.""" + +from __future__ import annotations + +from .create import create_create_automation_tool + +__all__ = ["create_create_automation_tool"] diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/create.py b/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/create.py new file mode 100644 index 000000000..173d302e5 --- /dev/null +++ b/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/create.py @@ -0,0 +1,208 @@ +"""``create_automation`` — NL intent → drafted JSON → HITL approval card → persisted. + +Single tool that: + +1. Drafts a structured automation from the user's intent via a focused sub-LLM + (system prompt in :mod:`.prompt`). +2. Surfaces the validated draft in a HITL approval card + (``action_type="automation_create"``). +3. On approval, validates the (possibly edited) payload again and persists + it via :class:`AutomationService`. + +The main agent only restates the user's request as a single ``intent`` string. +The drafting sub-LLM owns the JSON shape; the HITL card is the user's review. +""" + +from __future__ import annotations + +import json +import logging +import re +from typing import Any +from uuid import UUID + +from fastapi import HTTPException +from langchain.tools import ToolRuntime +from langchain_core.messages import HumanMessage +from langchain_core.tools import tool +from pydantic import ValidationError + +from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( + request_approval, +) +from app.automations.schemas.api import AutomationCreate +from app.automations.services.automation import AutomationService +from app.db import User, async_session_maker +from app.utils.content_utils import extract_text_content + +from .prompt import build_draft_prompt + +logger = logging.getLogger(__name__) + +_JSON_FENCE = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL) + + +def create_create_automation_tool( + *, + search_space_id: int, + user_id: str | UUID, + llm: Any, +): + """Factory for the ``create_automation`` tool. + + ``search_space_id`` is injected from the chat session (the model never + has to guess it). ``llm`` is the drafting sub-model — we reuse the main + agent's LLM and tag the call so it's identifiable in traces. A fresh + ``AsyncSession`` is opened per call to avoid stale sessions on + compiled-agent cache hits (same pattern as the Notion / memory tools). + """ + uid = UUID(user_id) if isinstance(user_id, str) else user_id + + @tool + async def create_automation(intent: str, runtime: ToolRuntime) -> dict[str, Any]: + """Draft + save an automation from a natural-language intent. + + Use this when the user wants SurfSense to do something on its own + on a schedule (e.g. "every morning summarize folder 12 to Slack"). + Restate the user's request as ONE concrete ``intent`` string: what + should run, when, and which static values (folder ids, channel + names, …) it needs. + + The tool drafts the full automation JSON internally, shows the user + a structured preview on an approval card, and persists on approval. + The card supports approve/reject only — if the user wants edits + after seeing the draft, they say so in chat and you call this tool + again with a refined intent. Do NOT prompt the user to confirm + before calling — the card IS the confirmation. + + Args: + intent: Concrete restatement of the user's request. Include + the schedule (with timezone if mentioned), the action to + take, and any static values. Example: "Every weekday at + 09:00 UTC, summarize new docs added to folder_id=12 since + the last run, then post the summary to Slack channel + '#daily-digest'." + + Returns: + ``{"status": "saved", "automation_id": int, "name": str}`` on + approval + save. + ``{"status": "rejected", "message": "..."}`` when the user + declines on the card. + ``{"status": "invalid", "issues": [...], "raw": ...}`` when + the drafter produced output that did not validate (call again + with a more precise intent). + ``{"status": "error", "message": "..."}`` on drafter or + persistence failure. + + IMPORTANT: when status is ``"rejected"`` the user explicitly + declined. Acknowledge once and stop — do NOT retry or pitch + variants without a fresh user request. + """ + # --- 1. Draft via sub-LLM --- + prompt = build_draft_prompt(search_space_id=search_space_id, intent=intent) + try: + response = await llm.ainvoke( + [HumanMessage(content=prompt)], + config={"tags": ["surfsense:internal", "automation-draft"]}, + ) + except Exception as exc: + logger.exception("create_automation drafting LLM call failed") + return {"status": "error", "message": f"drafting failed: {exc}"} + + raw_text = extract_text_content(response.content).strip() + draft = _extract_json(raw_text) + if draft is None: + return { + "status": "invalid", + "issues": ["model output was not parseable JSON"], + "raw": raw_text, + } + + # search_space_id is injected here so the sub-LLM never has to guess. + draft["search_space_id"] = search_space_id + try: + validated_draft = AutomationCreate.model_validate(draft) + except ValidationError as exc: + return { + "status": "invalid", + "issues": _format_validation_issues(exc), + "raw": draft, + } + + # --- 2. HITL approval card --- + try: + card_params = validated_draft.model_dump(mode="json", by_alias=True) + # search_space_id is session-scoped, not user-editable. + card_params.pop("search_space_id", None) + + result = request_approval( + action_type="automation_create", + tool_name="create_automation", + params=card_params, + context={"search_space_id": search_space_id}, + tool_call_id=runtime.tool_call_id, + ) + + if result.rejected: + return { + "status": "rejected", + "message": "User declined. Do not retry or suggest alternatives.", + } + + # --- 3. Persist (re-validate in case the user edited) --- + final_payload = {**result.params, "search_space_id": search_space_id} + try: + final_validated = AutomationCreate.model_validate(final_payload) + except ValidationError as exc: + return { + "status": "invalid", + "issues": _format_validation_issues(exc), + } + + async with async_session_maker() as session: + user = await session.get(User, uid) + if user is None: + return { + "status": "error", + "message": "user not found in this session", + } + service = AutomationService(session=session, user=user) + created = await service.create(final_validated) + return { + "status": "saved", + "automation_id": created.id, + "name": created.name, + } + + except HTTPException as exc: + return {"status": "error", "message": exc.detail} + except Exception as exc: + from langgraph.errors import GraphInterrupt + + if isinstance(exc, GraphInterrupt): + raise + logger.exception("create_automation failed") + return {"status": "error", "message": f"persistence failed: {exc}"} + + return create_automation + + +def _extract_json(text: str) -> dict[str, Any] | None: + """Pull a JSON object out of the model response, tolerating ``` fences.""" + if not text: + return None + candidate = text + fence_match = _JSON_FENCE.search(text) + if fence_match: + candidate = fence_match.group(1) + try: + parsed = json.loads(candidate) + except json.JSONDecodeError: + return None + return parsed if isinstance(parsed, dict) else None + + +def _format_validation_issues(exc: ValidationError) -> list[str]: + return [ + f"{'.'.join(str(p) for p in err['loc'])}: {err['msg']}" for err in exc.errors() + ] diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/prompt.py b/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/prompt.py new file mode 100644 index 000000000..45870e768 --- /dev/null +++ b/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/prompt.py @@ -0,0 +1,179 @@ +"""System prompt for the drafting sub-LLM inside ``create_automation``. + +Converts a natural-language ``intent`` into a structured ``AutomationCreate`` +JSON object. That object becomes the payload the HITL approval card surfaces. + +Scope split: + Real automation JSONs live here — this is the graph that *generates* + the JSON. The main agent's prompt fragments (``description.md`` / + ``example.md``) only carry intent-string examples; the main agent + never sees the schema. + +Layout: + The prompt is concatenated from four format-safe pieces. ``_HEADER`` / + ``_FOOTER`` carry the only ``str.format`` placeholders; ``_SCHEMA`` and + ``_FEW_SHOTS`` are plain strings so their JSON literals (and the + ``{{ inputs.X }}`` Jinja references in queries) can stay readable + without doubled-brace escaping. + +Catalog handling: + v1 hard-codes the action/trigger catalog (one action, one trigger). + When new types ship, swap the inline lines for a render-time pull + from ``app.automations.actions`` / ``app.automations.triggers`` via + lazy imports inside :func:`build_draft_prompt` so this module never + participates in the ``multi_agent_chat`` import cycle. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +_HEADER = """\ +You are the SurfSense automation drafter. Convert the user intent below +into a SINGLE JSON object matching the AutomationCreate schema. Output +ONLY that JSON object — no prose, no markdown fence, no commentary. + +Current UTC time (for cron context): {now} +Target search_space_id: {search_space_id} +""" + + +_SCHEMA = """ +Required JSON shape: +{ + "name": "<1-200 char identifier>", + "description": "", + "definition": { + "schema_version": "1.0", + "name": "", + "goal": "", + "plan": [ + { + "step_id": "", + "action": "agent_task", + "params": { + "query": "", + "auto_approve_all": true + } + } + ], + "metadata": {"tags": ["..."]} + }, + "triggers": [ + { + "type": "schedule", + "params": {"cron": "<5-field cron>", "timezone": ""}, + "static_inputs": {"": , ...}, + "enabled": true + } + ] +} + +v1 catalog (only these are valid): +- Actions: agent_task — params: query (string, Jinja), auto_approve_all (bool). +- Triggers: schedule — params: cron (5-field), timezone (IANA, e.g. "UTC", + "Europe/Paris"). Has static_inputs (object). + +Conventions: +- Whatever the plan references via {{ inputs.X }} MUST appear either in a + trigger's static_inputs OR in definition.inputs.schema_.properties so the + executor can resolve it at fire time. +- static_inputs carries values that stay the same across every fire + (folder ids, channel names, project keys, parent page ids). Put them on + the trigger that supplies them, not in the plan. +- If the user did NOT supply a value the plan needs, put "REPLACE_ME" in + static_inputs. Do NOT invent ids, channels, or paths. +- Cron is 5-field (minute hour day-of-month month day-of-week). Use the + timezone the user mentioned; default "UTC" when unspecified. +- Templating variables available at fire time: inputs.* (merged + static_inputs + runtime), inputs.fired_at, inputs.last_fired_at. +""" + + +_FEW_SHOTS = """ +Few-shot examples (intent → JSON output): + +### Example 1 — schedule with all static values supplied +intent: "Every weekday at 09:00 UTC, summarize documents added to folder_id=12 since the last run, then post the summary to Slack channel '#daily-digest'. Static inputs: folder_id=12, slack_channel='#daily-digest'." +output: +{ + "name": "Daily folder 12 digest", + "description": "Weekday 09:00 UTC summary of folder 12 documents posted to #daily-digest", + "definition": { + "schema_version": "1.0", + "name": "Daily folder 12 digest", + "goal": "Summarize new docs in folder 12 since the last run and post to #daily-digest", + "plan": [ + { + "step_id": "summarize_and_post", + "action": "agent_task", + "params": { + "query": "Summarize documents added to folder {{ inputs.folder_id }} since {{ inputs.last_fired_at or 'yesterday' }}, then send the summary to Slack channel {{ inputs.slack_channel }}.", + "auto_approve_all": true + } + } + ], + "metadata": {"tags": ["daily", "digest", "slack"]} + }, + "triggers": [ + { + "type": "schedule", + "params": {"cron": "0 9 * * 1-5", "timezone": "UTC"}, + "static_inputs": {"folder_id": 12, "slack_channel": "#daily-digest"}, + "enabled": true + } + ] +} + +### Example 2 — schedule with a missing value (REPLACE_ME placeholder) +intent: "Every Monday at 07:00 Europe/Paris, read last week's Jira issues in project CORE, then draft a Notion page recapping them. Static inputs: jira_project_key='CORE'. The user did NOT specify the Notion parent page id — leave it as a placeholder." +output: +{ + "name": "Weekly CORE Jira recap", + "description": "Monday 07:00 Europe/Paris recap of last week's CORE Jira issues, drafted to Notion", + "definition": { + "schema_version": "1.0", + "name": "Weekly CORE Jira recap", + "goal": "Recap last week's CORE Jira issues into a Notion page", + "plan": [ + { + "step_id": "recap", + "action": "agent_task", + "params": { + "query": "List Jira issues in project {{ inputs.jira_project_key }} updated in the 7 days before {{ inputs.fired_at }}. Draft a Notion page under parent id {{ inputs.notion_parent_page_id }} titled 'CORE recap — week of {{ inputs.fired_at }}'.", + "auto_approve_all": true + } + } + ], + "metadata": {"tags": ["weekly", "recap", "jira", "notion"]} + }, + "triggers": [ + { + "type": "schedule", + "params": {"cron": "0 7 * * 1", "timezone": "Europe/Paris"}, + "static_inputs": {"jira_project_key": "CORE", "notion_parent_page_id": "REPLACE_ME"}, + "enabled": true + } + ] +} +""" + + +_FOOTER = """ +User intent: +{intent} +""" + + +def build_draft_prompt(*, search_space_id: int, intent: str) -> str: + """Render the drafting sub-LLM system prompt for the given intent.""" + return ( + _HEADER.format( + now=datetime.now(UTC).isoformat(timespec="seconds"), + search_space_id=search_space_id, + ) + + _SCHEMA + + _FEW_SHOTS + + _FOOTER.format(intent=intent.strip()) + ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/index.py b/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/index.py index 5d309261c..88509eda7 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/index.py +++ b/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/index.py @@ -10,6 +10,7 @@ MAIN_AGENT_SURFSENSE_TOOL_NAMES_ORDERED: tuple[str, ...] = ( "web_search", "scrape_webpage", "update_memory", + "create_automation", ) MAIN_AGENT_SURFSENSE_TOOL_NAMES: frozenset[str] = frozenset( diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/request.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/request.py index 8729ea85b..2f7e3cd35 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/request.py +++ b/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/request.py @@ -49,6 +49,7 @@ def request_approval( params: dict[str, Any], context: dict[str, Any] | None = None, trusted_tools: list[str] | None = None, + tool_call_id: str | None = None, ) -> HITLResult: """Pause the graph for user approval and return the user's decision. @@ -64,6 +65,10 @@ def request_approval( forwarded verbatim to the FE for richer card chrome. trusted_tools: Per-session allowlist; when ``tool_name`` is in it the interrupt is skipped and the tool runs immediately. + tool_call_id: Caller's LangChain tool-call id. Required for tools + running directly on the main agent; subagent-mounted tools omit + it (the ``task`` chokepoint stamps it on re-raise — see + :mod:`...checkpointed_subagent_middleware.propagation`). Returns: :class:`HITLResult` with ``rejected=True`` if the user declined or @@ -90,6 +95,8 @@ def request_approval( interrupt_type=action_type, context=context, ) + if tool_call_id: + payload["tool_call_id"] = tool_call_id approval = interrupt(payload) parsed = parse_lc_envelope(approval) diff --git a/surfsense_backend/app/agents/new_chat/tools/registry.py b/surfsense_backend/app/agents/new_chat/tools/registry.py index b842d7a20..8c263ca20 100644 --- a/surfsense_backend/app/agents/new_chat/tools/registry.py +++ b/surfsense_backend/app/agents/new_chat/tools/registry.py @@ -150,6 +150,28 @@ class ToolDefinition: reverse: Callable[[dict[str, Any], Any], dict[str, Any]] | None = None +# ============================================================================= +# Deferred-import factories +# ============================================================================= +# Used for tools whose impls live under ``multi_agent_chat``. Importing those +# at module-load time would cycle (``multi_agent_chat`` middleware imports +# this registry). The import inside the factory runs only when +# ``build_tools`` is called, by which point ``multi_agent_chat`` is fully +# initialised. + + +def _build_create_automation_tool(deps: dict[str, Any]) -> BaseTool: + from app.agents.multi_agent_chat.main_agent.tools.automation import ( + create_create_automation_tool, + ) + + return create_create_automation_tool( + search_space_id=deps["search_space_id"], + user_id=deps["user_id"], + llm=deps["llm"], + ) + + # ============================================================================= # Built-in Tools Registry # ============================================================================= @@ -261,6 +283,21 @@ BUILTIN_TOOLS: list[ToolDefinition] = [ requires=["db_session", "search_space_id", "user_id"], ), # ========================================================================= + # AUTOMATION AUTHORING - single HITL tool. The tool takes an NL ``intent`` + # from the main agent, drafts the full AutomationCreate JSON via a focused + # sub-LLM, surfaces it on an approval card, and persists on approval. The + # factory defers its import because the impl lives under ``multi_agent_chat`` + # and that package transitively pulls this registry via middleware; + # deferring to ``build_tools`` call-time breaks the cycle without a + # parallel registry. + # ========================================================================= + ToolDefinition( + name="create_automation", + description="Draft an automation from an NL intent; user approves the card; tool saves", + factory=_build_create_automation_tool, + requires=["search_space_id", "user_id", "llm"], + ), + # ========================================================================= # MEMORY TOOL - single update_memory, private or team by thread_visibility # ========================================================================= ToolDefinition( diff --git a/surfsense_backend/app/automations/__init__.py b/surfsense_backend/app/automations/__init__.py new file mode 100644 index 000000000..a4ce8ecc9 --- /dev/null +++ b/surfsense_backend/app/automations/__init__.py @@ -0,0 +1,5 @@ +"""Automations engine — see automation-design-plan.md.""" + +from __future__ import annotations + +__all__: list[str] = [] diff --git a/surfsense_backend/app/automations/actions/__init__.py b/surfsense_backend/app/automations/actions/__init__.py new file mode 100644 index 000000000..9ef091cb3 --- /dev/null +++ b/surfsense_backend/app/automations/actions/__init__.py @@ -0,0 +1,24 @@ +"""Actions domain: registry surface + built-in action packages. + +Each action lives in its own subpackage (``agent_task/``, ...) and self-registers +at import time via its ``definition`` module. Side-effect imports below ensure +the registry is populated whenever anyone touches the actions package. +""" + +from __future__ import annotations + +from .store import all_actions, get_action, register_action +from .types import ActionContext, ActionDefinition, ActionHandler, ActionHandlerFactory + +__all__ = [ + "ActionContext", + "ActionDefinition", + "ActionHandler", + "ActionHandlerFactory", + "all_actions", + "get_action", + "register_action", +] + +# Built-in actions self-register at import time. +from . import agent_task # noqa: E402, F401 diff --git a/surfsense_backend/app/automations/actions/agent_task/__init__.py b/surfsense_backend/app/automations/actions/agent_task/__init__.py new file mode 100644 index 000000000..308812211 --- /dev/null +++ b/surfsense_backend/app/automations/actions/agent_task/__init__.py @@ -0,0 +1,15 @@ +"""``agent_task`` action: spin up multi_agent_chat for one rendered query. + +Imports ``definition`` for its side-effect (self-registration on the actions +registry) and re-exports ``build_handler`` for direct consumers. +""" + +from __future__ import annotations + +from .factory import build_handler +from .params import AgentTaskActionParams + +__all__ = ["AgentTaskActionParams", "build_handler"] + +# Side-effect: register on the actions store. +from . import definition # noqa: E402, F401 diff --git a/surfsense_backend/app/automations/actions/agent_task/auto_decide.py b/surfsense_backend/app/automations/actions/agent_task/auto_decide.py new file mode 100644 index 000000000..357eeb565 --- /dev/null +++ b/surfsense_backend/app/automations/actions/agent_task/auto_decide.py @@ -0,0 +1,39 @@ +"""Synthesize HITL decisions for every pending interrupt (approve-all or reject-all).""" + +from __future__ import annotations + +from typing import Any + + +def build_auto_decisions( + state: Any, decision: str +) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]: + """Return ``(lg_resume_map, surfsense_resume_value)`` covering every pending interrupt. + + ``lg_resume_map`` is keyed by ``Interrupt.id`` for ``Command(resume=...)``; + ``surfsense_resume_value`` is keyed by ``tool_call_id`` for the subagent + middleware bridge. Action count is read from ``value.action_requests`` when + present and falls back to ``1`` for wrapped scalar interrupts. + """ + lg_resume_map: dict[str, dict[str, Any]] = {} + routed: dict[str, dict[str, Any]] = {} + + for interrupt_obj in getattr(state, "interrupts", ()) or (): + value = getattr(interrupt_obj, "value", None) + if not isinstance(value, dict): + continue + interrupt_id = getattr(interrupt_obj, "id", None) + if not isinstance(interrupt_id, str): + continue + + action_requests = value.get("action_requests") + count = len(action_requests) if isinstance(action_requests, list) else 1 + decisions = [{"type": decision} for _ in range(count)] + + lg_resume_map[interrupt_id] = {"decisions": decisions} + + tool_call_id = value.get("tool_call_id") + if isinstance(tool_call_id, str): + routed[tool_call_id] = {"decisions": decisions} + + return lg_resume_map, routed diff --git a/surfsense_backend/app/automations/actions/agent_task/definition.py b/surfsense_backend/app/automations/actions/agent_task/definition.py new file mode 100644 index 000000000..7d14dc49e --- /dev/null +++ b/surfsense_backend/app/automations/actions/agent_task/definition.py @@ -0,0 +1,18 @@ +"""``agent_task`` ``ActionDefinition`` registration.""" + +from __future__ import annotations + +from ..store import register_action +from ..types import ActionDefinition +from .factory import build_handler +from .params import AgentTaskActionParams + +AGENT_TASK_ACTION = ActionDefinition( + type="agent_task", + name="Agent task", + description="Run a multi_agent_chat turn from an automation step.", + params_model=AgentTaskActionParams, + build_handler=build_handler, +) + +register_action(AGENT_TASK_ACTION) diff --git a/surfsense_backend/app/automations/actions/agent_task/dependencies.py b/surfsense_backend/app/automations/actions/agent_task/dependencies.py new file mode 100644 index 000000000..79107cd65 --- /dev/null +++ b/surfsense_backend/app/automations/actions/agent_task/dependencies.py @@ -0,0 +1,75 @@ +"""Build the per-invocation dependencies the multi_agent_chat factory needs.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from langgraph.checkpoint.memory import InMemorySaver +from sqlalchemy.ext.asyncio import AsyncSession + +from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle +from app.tasks.chat.streaming.flows.shared.pre_stream_setup import ( + setup_connector_and_firecrawl, +) + + +class DependencyError(Exception): + """An external dependency (LLM config, connector service, ...) refused to load.""" + + +@dataclass(frozen=True, slots=True) +class AgentDependencies: + """Everything ``create_multi_agent_chat_deep_agent`` needs from the environment.""" + + llm: Any + agent_config: Any + connector_service: Any + firecrawl_api_key: str | None + checkpointer: Any + + +async def build_dependencies( + *, + session: AsyncSession, + search_space_id: int, +) -> AgentDependencies: + """Load the LLM bundle, connector service, and a per-invoke in-memory checkpointer. + + Uses the search space's default LLM config (``config_id=-1``). Per-step + model overrides land in a future iteration alongside the ``model`` param. + """ + llm, agent_config, err = await load_llm_bundle( + session, config_id=-1, search_space_id=search_space_id + ) + if err is not None or llm is None: + raise DependencyError(err or "failed to load default LLM config") + + connector_service, firecrawl_api_key = await setup_connector_and_firecrawl( + session, search_space_id=search_space_id + ) + # Quick fix: use an in-memory checkpointer for automation runs. + # + # The shared Postgres checkpointer caches DB connections in a + # module-level pool. Each cached connection is bound to the asyncio + # loop that opened it. Celery throws away the loop after every task, + # so the pool ends up full of connections pointing to a dead loop, + # and the next Celery task (running on a fresh loop) can't use any + # of them — it hangs 30s and fails with + # `PoolTimeout: couldn't get a connection after 30.00 sec`. + # + # InMemorySaver has no cached connections, no loop binding — each + # Celery task creates one and drops it on exit. + # + # TODO(checkpointer): proper fix is to dispose the checkpointer + # pool around each Celery task in `run_async_celery_task`, the same + # way `_dispose_shared_db_engine` already does for the SQLAlchemy + # pool. Then this site can switch back to the shared checkpointer. + checkpointer = InMemorySaver() + return AgentDependencies( + llm=llm, + agent_config=agent_config, + connector_service=connector_service, + firecrawl_api_key=firecrawl_api_key, + checkpointer=checkpointer, + ) diff --git a/surfsense_backend/app/automations/actions/agent_task/factory.py b/surfsense_backend/app/automations/actions/agent_task/factory.py new file mode 100644 index 000000000..18a408e13 --- /dev/null +++ b/surfsense_backend/app/automations/actions/agent_task/factory.py @@ -0,0 +1,23 @@ +"""Bind ``ActionContext`` to a callable that runs one ``agent_task`` step.""" + +from __future__ import annotations + +from typing import Any + +from ..types import ActionContext, ActionHandler +from .invoke import run_agent_task +from .params import AgentTaskActionParams + + +def build_handler(ctx: ActionContext) -> ActionHandler: + """Return a handler closure that validates params and runs the agent task.""" + + async def handle(params: dict[str, Any]) -> dict[str, Any]: + validated = AgentTaskActionParams.model_validate(params) + return await run_agent_task( + ctx=ctx, + query=validated.query, + auto_approve_all=validated.auto_approve_all, + ) + + return handle diff --git a/surfsense_backend/app/automations/actions/agent_task/finalize.py b/surfsense_backend/app/automations/actions/agent_task/finalize.py new file mode 100644 index 000000000..d5f1f95f6 --- /dev/null +++ b/surfsense_backend/app/automations/actions/agent_task/finalize.py @@ -0,0 +1,44 @@ +"""Extract the agent's final assistant text from the terminal invoke result.""" + +from __future__ import annotations + +from typing import Any + +from langchain_core.messages import AIMessage + + +def extract_final_assistant_message(result: Any) -> str | None: + """Return the last ``AIMessage`` text content, or ``None`` if there isn't one. + + Multi-part messages (content lists) are flattened by concatenating ``text`` + parts in order. Non-string content (tool calls, images) is skipped. + """ + if not isinstance(result, dict): + return None + messages = result.get("messages") + if not isinstance(messages, list): + return None + + for msg in reversed(messages): + if not isinstance(msg, AIMessage): + continue + return _content_to_text(msg.content) + return None + + +def _content_to_text(content: Any) -> str | None: + if isinstance(content, str): + text = content.strip() + return text or None + if isinstance(content, list): + parts: list[str] = [] + for part in content: + if isinstance(part, str): + parts.append(part) + elif isinstance(part, dict) and part.get("type") == "text": + text = part.get("text") + if isinstance(text, str): + parts.append(text) + joined = "".join(parts).strip() + return joined or None + return None diff --git a/surfsense_backend/app/automations/actions/agent_task/invoke.py b/surfsense_backend/app/automations/actions/agent_task/invoke.py new file mode 100644 index 000000000..a37e9beed --- /dev/null +++ b/surfsense_backend/app/automations/actions/agent_task/invoke.py @@ -0,0 +1,98 @@ +"""Run one ``agent_task`` invocation: ainvoke + auto-decision resume loop.""" + +from __future__ import annotations + +import time +import uuid +from typing import Any + +from langchain_core.messages import HumanMessage +from langgraph.types import Command + +from app.agents.multi_agent_chat import create_multi_agent_chat_deep_agent +from app.db import ChatVisibility, async_session_maker + +from ..types import ActionContext + +from .auto_decide import build_auto_decisions +from .dependencies import build_dependencies +from .finalize import extract_final_assistant_message + +# Cap on HITL resume iterations. The agent should not need this many turns in one +# step; treat overshoot as a runaway and fail the step. +_MAX_RESUMES = 50 + + +async def run_agent_task( + *, + ctx: ActionContext, + query: str, + auto_approve_all: bool, +) -> dict[str, Any]: + """Invoke multi_agent_chat for one rendered query and return its outcome. + + Opens its own DB session so the executor's bookkeeping session isn't tied + up for the entire invocation. The LangGraph ``thread_id`` (a fresh UUID) + is returned as ``agent_session_id`` for later inspection. + """ + agent_session_id = str(uuid.uuid4()) + user_id = str(ctx.creator_user_id) if ctx.creator_user_id else None + decision = "approve" if auto_approve_all else "reject" + + async with async_session_maker() as agent_session: + deps = await build_dependencies( + session=agent_session, + search_space_id=ctx.search_space_id, + ) + + agent = await create_multi_agent_chat_deep_agent( + llm=deps.llm, + search_space_id=ctx.search_space_id, + db_session=agent_session, + connector_service=deps.connector_service, + checkpointer=deps.checkpointer, + user_id=user_id, + thread_id=None, + agent_config=deps.agent_config, + firecrawl_api_key=deps.firecrawl_api_key, + thread_visibility=ChatVisibility.PRIVATE, + ) + + request_id = f"automation:{ctx.run_id}:{ctx.step_id}" + turn_id = f"{request_id}:{int(time.time() * 1000)}" + input_state: dict[str, Any] = { + "messages": [HumanMessage(content=query)], + "search_space_id": ctx.search_space_id, + "request_id": request_id, + "turn_id": turn_id, + } + config: dict[str, Any] = { + "configurable": { + "thread_id": agent_session_id, + "request_id": request_id, + "turn_id": turn_id, + }, + "recursion_limit": 10_000, + } + + result = await agent.ainvoke(input_state, config=config) + + resumes = 0 + while True: + state = await agent.aget_state(config) + if not getattr(state, "interrupts", None): + break + if resumes >= _MAX_RESUMES: + raise RuntimeError( + f"agent_task exceeded {_MAX_RESUMES} HITL resume iterations" + ) + lg_resume_map, routed = build_auto_decisions(state, decision) + config["configurable"]["surfsense_resume_value"] = routed + result = await agent.ainvoke(Command(resume=lg_resume_map), config=config) + resumes += 1 + + return { + "agent_session_id": agent_session_id, + "final_message": extract_final_assistant_message(result), + "resumes": resumes, + } diff --git a/surfsense_backend/app/automations/actions/agent_task/params.py b/surfsense_backend/app/automations/actions/agent_task/params.py new file mode 100644 index 000000000..b0e99a78b --- /dev/null +++ b/surfsense_backend/app/automations/actions/agent_task/params.py @@ -0,0 +1,21 @@ +"""``AgentTaskActionParams`` — params for the ``agent_task`` action type.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + + +class AgentTaskActionParams(BaseModel): + """Run a multi_agent_chat turn from an automation step.""" + + model_config = ConfigDict(extra="forbid") + + query: str = Field( + ..., + min_length=1, + description="User query for the agent; rendered at execute time.", + ) + auto_approve_all: bool = Field( + default=False, + description="If true, every HITL approval is auto-approved; otherwise rejected.", + ) diff --git a/surfsense_backend/app/automations/actions/store.py b/surfsense_backend/app/automations/actions/store.py new file mode 100644 index 000000000..eff66c4c7 --- /dev/null +++ b/surfsense_backend/app/automations/actions/store.py @@ -0,0 +1,23 @@ +"""In-memory action registry. Populated once at process startup.""" + +from __future__ import annotations + +from .types import ActionDefinition + +_REGISTRY: dict[str, ActionDefinition] = {} + + +def register_action(action: ActionDefinition) -> None: + """Register an action. Raises on duplicate type.""" + if action.type in _REGISTRY: + raise ValueError(f"Action already registered: {action.type!r}") + _REGISTRY[action.type] = action + + +def get_action(action_type: str) -> ActionDefinition | None: + return _REGISTRY.get(action_type) + + +def all_actions() -> dict[str, ActionDefinition]: + """Defensive snapshot of the registry.""" + return dict(_REGISTRY) diff --git a/surfsense_backend/app/automations/actions/types.py b/surfsense_backend/app/automations/actions/types.py new file mode 100644 index 000000000..2c4ffad8d --- /dev/null +++ b/surfsense_backend/app/automations/actions/types.py @@ -0,0 +1,40 @@ +"""``ActionDefinition``, ``ActionContext``, and handler/factory signatures.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any +from uuid import UUID + +from pydantic import BaseModel +from sqlalchemy.ext.asyncio import AsyncSession + + +@dataclass(frozen=True, slots=True) +class ActionContext: + """Per-invocation dependencies bound to an action handler at execute time.""" + + session: AsyncSession + run_id: int + step_id: str + search_space_id: int + creator_user_id: UUID | None + + +ActionHandler = Callable[[dict[str, Any]], Awaitable[Any]] +ActionHandlerFactory = Callable[[ActionContext], ActionHandler] + + +@dataclass(frozen=True, slots=True) +class ActionDefinition: + type: str + name: str + description: str + params_model: type[BaseModel] + build_handler: ActionHandlerFactory + + @property + def params_schema(self) -> dict[str, Any]: + """JSON Schema (draft 2020-12) derived from ``params_model``.""" + return self.params_model.model_json_schema() diff --git a/surfsense_backend/app/automations/api/__init__.py b/surfsense_backend/app/automations/api/__init__.py new file mode 100644 index 000000000..a18e91a95 --- /dev/null +++ b/surfsense_backend/app/automations/api/__init__.py @@ -0,0 +1,16 @@ +"""HTTP layer for the automations feature.""" + +from __future__ import annotations + +from fastapi import APIRouter + +from .automation import router as automation_router +from .run import router as run_router +from .trigger import router as trigger_router + +router = APIRouter() +router.include_router(automation_router) +router.include_router(trigger_router) +router.include_router(run_router) + +__all__ = ["router"] diff --git a/surfsense_backend/app/automations/api/automation.py b/surfsense_backend/app/automations/api/automation.py new file mode 100644 index 000000000..b67f0af09 --- /dev/null +++ b/surfsense_backend/app/automations/api/automation.py @@ -0,0 +1,80 @@ +"""HTTP routes for the ``Automation`` resource.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Query, status + +from app.automations.schemas.api import ( + AutomationCreate, + AutomationDetail, + AutomationList, + AutomationSummary, + AutomationUpdate, +) +from app.automations.services import AutomationService, get_automation_service + +router = APIRouter() + + +@router.post( + "/automations", + response_model=AutomationDetail, + status_code=status.HTTP_201_CREATED, +) +async def create_automation( + payload: AutomationCreate, + service: AutomationService = Depends(get_automation_service), +) -> AutomationDetail: + """Create an automation, optionally with initial triggers (atomic).""" + automation = await service.create(payload) + return AutomationDetail.model_validate(automation) + + +@router.get("/automations", response_model=AutomationList) +async def list_automations( + search_space_id: int = Query(...), + limit: int = Query(default=50, ge=1, le=200), + offset: int = Query(default=0, ge=0), + service: AutomationService = Depends(get_automation_service), +) -> AutomationList: + """List automations in a search space.""" + items, total = await service.list( + search_space_id=search_space_id, limit=limit, offset=offset + ) + return AutomationList( + items=[AutomationSummary.model_validate(a) for a in items], + total=total, + ) + + +@router.get("/automations/{automation_id}", response_model=AutomationDetail) +async def get_automation( + automation_id: int, + service: AutomationService = Depends(get_automation_service), +) -> AutomationDetail: + """Get one automation with its definition and triggers.""" + automation = await service.get(automation_id) + return AutomationDetail.model_validate(automation) + + +@router.patch("/automations/{automation_id}", response_model=AutomationDetail) +async def update_automation( + automation_id: int, + patch: AutomationUpdate, + service: AutomationService = Depends(get_automation_service), +) -> AutomationDetail: + """Partially update an automation. Triggers are managed separately.""" + automation = await service.update(automation_id, patch) + return AutomationDetail.model_validate(automation) + + +@router.delete( + "/automations/{automation_id}", + status_code=status.HTTP_204_NO_CONTENT, +) +async def delete_automation( + automation_id: int, + service: AutomationService = Depends(get_automation_service), +) -> None: + """Delete an automation; triggers and runs are removed by FK cascade.""" + await service.delete(automation_id) diff --git a/surfsense_backend/app/automations/api/run.py b/surfsense_backend/app/automations/api/run.py new file mode 100644 index 000000000..b662a5943 --- /dev/null +++ b/surfsense_backend/app/automations/api/run.py @@ -0,0 +1,44 @@ +"""HTTP routes for automation run history.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Query + +from app.automations.schemas.api import RunDetail, RunList, RunSummary +from app.automations.services import RunService, get_run_service + +router = APIRouter() + + +@router.get( + "/automations/{automation_id}/runs", + response_model=RunList, +) +async def list_runs( + automation_id: int, + limit: int = Query(default=50, ge=1, le=200), + offset: int = Query(default=0, ge=0), + service: RunService = Depends(get_run_service), +) -> RunList: + """List run history for an automation, newest first.""" + items, total = await service.list( + automation_id=automation_id, limit=limit, offset=offset + ) + return RunList( + items=[RunSummary.model_validate(r) for r in items], + total=total, + ) + + +@router.get( + "/automations/{automation_id}/runs/{run_id}", + response_model=RunDetail, +) +async def get_run( + automation_id: int, + run_id: int, + service: RunService = Depends(get_run_service), +) -> RunDetail: + """Get the full record of a single run, including step results and artifacts.""" + run = await service.get(automation_id=automation_id, run_id=run_id) + return RunDetail.model_validate(run) diff --git a/surfsense_backend/app/automations/api/trigger.py b/surfsense_backend/app/automations/api/trigger.py new file mode 100644 index 000000000..40e47a86b --- /dev/null +++ b/surfsense_backend/app/automations/api/trigger.py @@ -0,0 +1,55 @@ +"""HTTP routes for triggers attached to an automation.""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, status + +from app.automations.schemas.api import TriggerCreate, TriggerDetail, TriggerUpdate +from app.automations.services import TriggerService, get_trigger_service + +router = APIRouter() + + +@router.post( + "/automations/{automation_id}/triggers", + response_model=TriggerDetail, + status_code=status.HTTP_201_CREATED, +) +async def add_trigger( + automation_id: int, + payload: TriggerCreate, + service: TriggerService = Depends(get_trigger_service), +) -> TriggerDetail: + """Attach a new trigger to an automation.""" + trigger = await service.add(automation_id=automation_id, payload=payload) + return TriggerDetail.model_validate(trigger) + + +@router.patch( + "/automations/{automation_id}/triggers/{trigger_id}", + response_model=TriggerDetail, +) +async def update_trigger( + automation_id: int, + trigger_id: int, + patch: TriggerUpdate, + service: TriggerService = Depends(get_trigger_service), +) -> TriggerDetail: + """Toggle ``enabled`` or replace ``params``. Trigger type is immutable.""" + trigger = await service.update( + automation_id=automation_id, trigger_id=trigger_id, patch=patch + ) + return TriggerDetail.model_validate(trigger) + + +@router.delete( + "/automations/{automation_id}/triggers/{trigger_id}", + status_code=status.HTTP_204_NO_CONTENT, +) +async def remove_trigger( + automation_id: int, + trigger_id: int, + service: TriggerService = Depends(get_trigger_service), +) -> None: + """Detach a trigger from an automation.""" + await service.remove(automation_id=automation_id, trigger_id=trigger_id) diff --git a/surfsense_backend/app/automations/dispatch/__init__.py b/surfsense_backend/app/automations/dispatch/__init__.py new file mode 100644 index 000000000..be8a36581 --- /dev/null +++ b/surfsense_backend/app/automations/dispatch/__init__.py @@ -0,0 +1,8 @@ +"""Generic dispatch primitives shared across trigger types.""" + +from __future__ import annotations + +from .errors import DispatchError +from .run import dispatch_run + +__all__ = ["DispatchError", "dispatch_run"] diff --git a/surfsense_backend/app/automations/dispatch/errors.py b/surfsense_backend/app/automations/dispatch/errors.py new file mode 100644 index 000000000..75640a987 --- /dev/null +++ b/surfsense_backend/app/automations/dispatch/errors.py @@ -0,0 +1,7 @@ +"""Dispatch errors raised when a fire request cannot be turned into a run.""" + +from __future__ import annotations + + +class DispatchError(Exception): + """A dispatch could not proceed (missing trigger, invalid inputs, ...).""" diff --git a/surfsense_backend/app/automations/dispatch/run.py b/surfsense_backend/app/automations/dispatch/run.py new file mode 100644 index 000000000..02d0b0356 --- /dev/null +++ b/surfsense_backend/app/automations/dispatch/run.py @@ -0,0 +1,83 @@ +"""Generic run dispatch: validate, snapshot, persist, enqueue. Shared by every trigger.""" + +from __future__ import annotations + +from typing import Any + +import jsonschema +from sqlalchemy.ext.asyncio import AsyncSession + +from app.automations.persistence.enums.run_status import RunStatus +from app.automations.persistence.models.automation import Automation +from app.automations.persistence.models.run import AutomationRun +from app.automations.persistence.models.trigger import AutomationTrigger +from app.automations.schemas.definition.envelope import AutomationDefinition +from app.automations.tasks.execute_run import automation_run_execute + +from .errors import DispatchError + + +async def dispatch_run( + *, + session: AsyncSession, + automation: Automation, + trigger: AutomationTrigger, + runtime_inputs: dict[str, Any] | None = None, +) -> AutomationRun: + """Validate, snapshot the definition, persist an ``AutomationRun``, enqueue execution. + + Final inputs = ``trigger.static_inputs`` merged with ``runtime_inputs``, + static winning on key collision. The merged dict is validated against + ``automation.definition.inputs.schema_`` and stored on the run. + + Callers (trigger-specific adapters) are responsible for resolving + ``automation`` and ``trigger`` and for the trigger-side ``ACTIVE`` / + ``enabled`` guards. This function only handles what's identical across + every trigger type. + """ + try: + definition = AutomationDefinition.model_validate(automation.definition) + except Exception as exc: + raise DispatchError(f"invalid automation definition: {exc}") from exc + + merged_inputs = {**(runtime_inputs or {}), **(trigger.static_inputs or {})} + validated_inputs = _validate_inputs(definition, merged_inputs) + snapshot = definition.model_dump(mode="json", by_alias=True) + + run = AutomationRun( + automation_id=automation.id, + trigger_id=trigger.id, + status=RunStatus.PENDING, + definition_snapshot=snapshot, + inputs=validated_inputs, + step_results=[], + artifacts=[], + ) + session.add(run) + await session.commit() + await session.refresh(run) + + automation_run_execute.apply_async( + args=[run.id], + time_limit=definition.execution.timeout_seconds, + ) + return run + + +def _validate_inputs( + definition: AutomationDefinition, inputs: dict[str, Any] +) -> dict[str, Any]: + """Validate merged inputs against the optional declared schema. + + No declared schema → pass through (runtime inputs like ``fired_at`` / + ``last_fired_at`` and trigger ``static_inputs`` must still reach the + template context). Returning ``{}`` here strips them and makes Jinja + blow up on any ``{{ inputs.* }}`` reference. + """ + if definition.inputs is None or not definition.inputs.schema_: + return inputs + try: + jsonschema.validate(instance=inputs, schema=definition.inputs.schema_) + except jsonschema.ValidationError as exc: + raise DispatchError(f"inputs: {exc.message}") from exc + return inputs diff --git a/surfsense_backend/app/automations/persistence/__init__.py b/surfsense_backend/app/automations/persistence/__init__.py new file mode 100644 index 000000000..b10aef03d --- /dev/null +++ b/surfsense_backend/app/automations/persistence/__init__.py @@ -0,0 +1,15 @@ +"""Models and enums for the automation tables.""" + +from __future__ import annotations + +from .enums import AutomationStatus, RunStatus, TriggerType +from .models import Automation, AutomationRun, AutomationTrigger + +__all__ = [ + "Automation", + "AutomationRun", + "AutomationStatus", + "AutomationTrigger", + "RunStatus", + "TriggerType", +] diff --git a/surfsense_backend/app/automations/persistence/enums/__init__.py b/surfsense_backend/app/automations/persistence/enums/__init__.py new file mode 100644 index 000000000..6c2cfcf1f --- /dev/null +++ b/surfsense_backend/app/automations/persistence/enums/__init__.py @@ -0,0 +1,13 @@ +"""Enums for the automation tables.""" + +from __future__ import annotations + +from .automation_status import AutomationStatus +from .run_status import RunStatus +from .trigger_type import TriggerType + +__all__ = [ + "AutomationStatus", + "RunStatus", + "TriggerType", +] diff --git a/surfsense_backend/app/automations/persistence/enums/automation_status.py b/surfsense_backend/app/automations/persistence/enums/automation_status.py new file mode 100644 index 000000000..aff6f4683 --- /dev/null +++ b/surfsense_backend/app/automations/persistence/enums/automation_status.py @@ -0,0 +1,11 @@ +"""Automation lifecycle status.""" + +from __future__ import annotations + +from enum import StrEnum + + +class AutomationStatus(StrEnum): + ACTIVE = "active" # eligible to fire + PAUSED = "paused" # kept, but triggers don't fire + ARCHIVED = "archived" # read-only history diff --git a/surfsense_backend/app/automations/persistence/enums/run_status.py b/surfsense_backend/app/automations/persistence/enums/run_status.py new file mode 100644 index 000000000..64dcd49e8 --- /dev/null +++ b/surfsense_backend/app/automations/persistence/enums/run_status.py @@ -0,0 +1,14 @@ +"""AutomationRun state machine: pending → running → (succeeded|failed|cancelled|timed_out).""" + +from __future__ import annotations + +from enum import StrEnum + + +class RunStatus(StrEnum): + PENDING = "pending" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + CANCELLED = "cancelled" + TIMED_OUT = "timed_out" diff --git a/surfsense_backend/app/automations/persistence/enums/trigger_type.py b/surfsense_backend/app/automations/persistence/enums/trigger_type.py new file mode 100644 index 000000000..a583b1bd6 --- /dev/null +++ b/surfsense_backend/app/automations/persistence/enums/trigger_type.py @@ -0,0 +1,15 @@ +"""Trigger-kind discriminator. + +v1 only registers ``schedule``. ``manual`` is reserved in the enum (mirrors the +postgres enum) but is intentionally unregistered pending a redesign of the +"Run now" UX. +""" + +from __future__ import annotations + +from enum import StrEnum + + +class TriggerType(StrEnum): + SCHEDULE = "schedule" + MANUAL = "manual" diff --git a/surfsense_backend/app/automations/persistence/models/__init__.py b/surfsense_backend/app/automations/persistence/models/__init__.py new file mode 100644 index 000000000..8b985f025 --- /dev/null +++ b/surfsense_backend/app/automations/persistence/models/__init__.py @@ -0,0 +1,13 @@ +"""Models, one per table.""" + +from __future__ import annotations + +from .automation import Automation +from .run import AutomationRun +from .trigger import AutomationTrigger + +__all__ = [ + "Automation", + "AutomationRun", + "AutomationTrigger", +] diff --git a/surfsense_backend/app/automations/persistence/models/automation.py b/surfsense_backend/app/automations/persistence/models/automation.py new file mode 100644 index 000000000..cb0b2ed31 --- /dev/null +++ b/surfsense_backend/app/automations/persistence/models/automation.py @@ -0,0 +1,81 @@ +"""``automations`` table — editable, versioned automation definition.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from sqlalchemy import ( + TIMESTAMP, + Column, + Enum as SQLAlchemyEnum, + ForeignKey, + Integer, + String, + Text, +) +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import relationship + +from app.db import BaseModel, TimestampMixin + +from ..enums.automation_status import AutomationStatus + + +class Automation(BaseModel, TimestampMixin): + __tablename__ = "automations" + + search_space_id = Column( + Integer, + ForeignKey("searchspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + + created_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("user.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + name = Column(String(200), nullable=False) + description = Column(Text, nullable=True) + + status = Column( + SQLAlchemyEnum( + AutomationStatus, + name="automation_status", + values_callable=lambda x: [e.value for e in x], + ), + nullable=False, + default=AutomationStatus.ACTIVE, + server_default=AutomationStatus.ACTIVE.value, + index=True, + ) + + definition = Column(JSONB, nullable=False) + + version = Column(Integer, nullable=False, default=1, server_default="1") + + updated_at = Column( + TIMESTAMP(timezone=True), + nullable=False, + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + index=True, + ) + + search_space = relationship("SearchSpace", back_populates="automations") + created_by = relationship("User", back_populates="automations") + triggers = relationship( + "AutomationTrigger", + back_populates="automation", + cascade="all, delete-orphan", + passive_deletes=True, + ) + runs = relationship( + "AutomationRun", + back_populates="automation", + cascade="all, delete-orphan", + passive_deletes=True, + ) diff --git a/surfsense_backend/app/automations/persistence/models/run.py b/surfsense_backend/app/automations/persistence/models/run.py new file mode 100644 index 000000000..262e4c2bf --- /dev/null +++ b/surfsense_backend/app/automations/persistence/models/run.py @@ -0,0 +1,66 @@ +"""``automation_runs`` table — immutable per-fire execution record.""" + +from __future__ import annotations + +from sqlalchemy import ( + TIMESTAMP, + Column, + Enum as SQLAlchemyEnum, + ForeignKey, + Integer, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import relationship + +from app.db import BaseModel, TimestampMixin + +from ..enums.run_status import RunStatus + + +class AutomationRun(BaseModel, TimestampMixin): + __tablename__ = "automation_runs" + + automation_id = Column( + Integer, + ForeignKey("automations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + + trigger_id = Column( + Integer, + ForeignKey("automation_triggers.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + status = Column( + SQLAlchemyEnum( + RunStatus, + name="automation_run_status", + values_callable=lambda x: [e.value for e in x], + ), + nullable=False, + default=RunStatus.PENDING, + server_default=RunStatus.PENDING.value, + index=True, + ) + + # locked at fire time so historical runs always show the exact code path + definition_snapshot = Column(JSONB, nullable=False) + + # merged & validated inputs the run was dispatched with + # (trigger.static_inputs ∪ producer runtime data, static wins on collision) + inputs = Column(JSONB, nullable=False, server_default="{}") + # one entry per executed step; agent_task entries carry their own + # `agent_session_id` inside their entry + step_results = Column(JSONB, nullable=False, server_default="[]") + output = Column(JSONB, nullable=True) + artifacts = Column(JSONB, nullable=False, server_default="[]") + error = Column(JSONB, nullable=True) + + started_at = Column(TIMESTAMP(timezone=True), nullable=True) + finished_at = Column(TIMESTAMP(timezone=True), nullable=True) + + automation = relationship("Automation", back_populates="runs") + trigger = relationship("AutomationTrigger", back_populates="runs") diff --git a/surfsense_backend/app/automations/persistence/models/trigger.py b/surfsense_backend/app/automations/persistence/models/trigger.py new file mode 100644 index 000000000..de1078acf --- /dev/null +++ b/surfsense_backend/app/automations/persistence/models/trigger.py @@ -0,0 +1,67 @@ +"""``automation_triggers`` table — one row per (automation, trigger-instance) pair.""" + +from __future__ import annotations + +from sqlalchemy import ( + TIMESTAMP, + Boolean, + Column, + Enum as SQLAlchemyEnum, + ForeignKey, + Integer, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import relationship + +from app.db import BaseModel, TimestampMixin + +from ..enums.trigger_type import TriggerType + + +class AutomationTrigger(BaseModel, TimestampMixin): + __tablename__ = "automation_triggers" + + automation_id = Column( + Integer, + ForeignKey("automations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + + type = Column( + SQLAlchemyEnum( + TriggerType, + name="automation_trigger_type", + values_callable=lambda x: [e.value for e in x], + ), + nullable=False, + index=True, + ) + + params = Column(JSONB, nullable=False) + + # Per-attachment domain values merged into every dispatched run's inputs. + # Static wins over runtime data on key collision. + static_inputs = Column(JSONB, nullable=False, server_default="{}") + + enabled = Column( + Boolean, + nullable=False, + default=True, + server_default="true", + index=True, + ) + + last_fired_at = Column(TIMESTAMP(timezone=True), nullable=True) + + # Precomputed next fire moment in UTC; advanced after each fire by the + # schedule tick. NULL means the trigger has never been scheduled (the + # tick self-heals on first sight). + next_fire_at = Column(TIMESTAMP(timezone=True), nullable=True) + + automation = relationship("Automation", back_populates="triggers") + runs = relationship( + "AutomationRun", + back_populates="trigger", + passive_deletes=True, + ) diff --git a/surfsense_backend/app/automations/runtime/__init__.py b/surfsense_backend/app/automations/runtime/__init__.py new file mode 100644 index 000000000..0650882b2 --- /dev/null +++ b/surfsense_backend/app/automations/runtime/__init__.py @@ -0,0 +1,7 @@ +"""Automation run executor: plan walker, step dispatch, retries, persistence.""" + +from __future__ import annotations + +from .executor import execute_run + +__all__ = ["execute_run"] diff --git a/surfsense_backend/app/automations/runtime/executor.py b/surfsense_backend/app/automations/runtime/executor.py new file mode 100644 index 000000000..b8a377e5b --- /dev/null +++ b/surfsense_backend/app/automations/runtime/executor.py @@ -0,0 +1,124 @@ +"""Walk an ``AutomationRun``'s snapshot plan to terminal state.""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.automations.persistence.enums.run_status import RunStatus +from app.automations.persistence.models.run import AutomationRun +from app.automations.actions.types import ActionContext +from app.automations.schemas.definition.envelope import AutomationDefinition +from app.automations.schemas.definition.plan_step import PlanStep +from app.automations.templating import build_run_context + +from . import repository +from .step import execute_step + + +async def execute_run(session: AsyncSession, run_id: int) -> None: + """Load run ``run_id`` and execute its snapshot plan to a terminal state.""" + run = await repository.load_run(session, run_id) + if run is None: + raise ValueError(f"automation_run {run_id} not found") + + if run.status != RunStatus.PENDING: + return + + try: + definition = AutomationDefinition.model_validate(run.definition_snapshot) + except Exception as exc: + await repository.mark_failed( + session, + run, + {"message": f"definition_snapshot invalid: {exc}", "type": type(exc).__name__}, + ) + await session.commit() + return + + await repository.mark_running(session, run) + await session.commit() + + step_outputs: dict[str, Any] = {} + + for step in definition.plan: + template_ctx = _build_template_ctx(run, step_outputs) + action_ctx = _build_action_ctx(session, run, step) + result = await execute_step( + step=step, + template_context=template_ctx, + action_context=action_ctx, + default_max_retries=definition.execution.max_retries, + default_retry_backoff=definition.execution.retry_backoff, + default_timeout_seconds=definition.execution.timeout_seconds, + ) + await repository.append_step_result(session, run, result) + await session.commit() + + if result["status"] == "failed": + await _run_on_failure(session, run, definition) + await repository.mark_failed(session, run, result.get("error")) + await session.commit() + return + + if result["status"] == "succeeded": + step_outputs[step.output_as or step.step_id] = result.get("result") + + await repository.mark_succeeded(session, run) + await session.commit() + + +async def _run_on_failure( + session: AsyncSession, + run: AutomationRun, + definition: AutomationDefinition, +) -> None: + """Run the on_failure steps. Their failures don't recurse into more on_failure.""" + if not definition.execution.on_failure: + return + template_ctx = _build_template_ctx(run, step_outputs={}) + for step in definition.execution.on_failure: + action_ctx = _build_action_ctx(session, run, step) + result = await execute_step( + step=step, + template_context=template_ctx, + action_context=action_ctx, + default_max_retries=definition.execution.max_retries, + default_retry_backoff=definition.execution.retry_backoff, + default_timeout_seconds=definition.execution.timeout_seconds, + ) + await repository.append_step_result(session, run, result) + await session.commit() + + +def _build_template_ctx(run: AutomationRun, step_outputs: dict[str, Any]) -> dict[str, Any]: + automation = run.automation + trigger = run.trigger + return build_run_context( + run_id=run.id, + automation_id=run.automation_id, + automation_name=automation.name if automation else None, + automation_version=automation.version if automation else None, + search_space_id=automation.search_space_id if automation else None, + creator_id=automation.created_by_user_id if automation else None, + trigger_id=run.trigger_id, + trigger_type=trigger.type.value if trigger else None, + started_at=run.started_at, + attempt=1, + inputs=run.inputs or {}, + step_outputs=step_outputs, + ) + + +def _build_action_ctx( + session: AsyncSession, run: AutomationRun, step: PlanStep +) -> ActionContext: + automation = run.automation + return ActionContext( + session=session, + run_id=run.id, + step_id=step.step_id, + search_space_id=automation.search_space_id, + creator_user_id=automation.created_by_user_id, + ) diff --git a/surfsense_backend/app/automations/runtime/repository.py b/surfsense_backend/app/automations/runtime/repository.py new file mode 100644 index 000000000..a8bdbc55a --- /dev/null +++ b/surfsense_backend/app/automations/runtime/repository.py @@ -0,0 +1,62 @@ +"""Persistence operations on ``AutomationRun``. Pure SQL, no business logic.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.automations.persistence.enums.run_status import RunStatus +from app.automations.persistence.models.run import AutomationRun + + +async def load_run(session: AsyncSession, run_id: int) -> AutomationRun | None: + """Load a run with its automation and trigger eagerly loaded.""" + stmt = ( + select(AutomationRun) + .where(AutomationRun.id == run_id) + .options( + selectinload(AutomationRun.automation), + selectinload(AutomationRun.trigger), + ) + ) + result = await session.execute(stmt) + return result.scalar_one_or_none() + + +async def mark_running(session: AsyncSession, run: AutomationRun) -> None: + run.status = RunStatus.RUNNING + run.started_at = datetime.now(UTC) + await session.flush() + + +async def mark_succeeded(session: AsyncSession, run: AutomationRun) -> None: + run.status = RunStatus.SUCCEEDED + run.finished_at = datetime.now(UTC) + await session.flush() + + +async def mark_failed( + session: AsyncSession, + run: AutomationRun, + error: dict[str, Any] | None, +) -> None: + run.status = RunStatus.FAILED + run.finished_at = datetime.now(UTC) + run.error = error + await session.flush() + + +async def append_step_result( + session: AsyncSession, + run: AutomationRun, + step_result: dict[str, Any], +) -> None: + """Append one step result. Reassigns the list so SQLAlchemy detects the change.""" + current = list(run.step_results or []) + current.append(step_result) + run.step_results = current + await session.flush() diff --git a/surfsense_backend/app/automations/runtime/retries.py b/surfsense_backend/app/automations/runtime/retries.py new file mode 100644 index 000000000..d5bfb15ca --- /dev/null +++ b/surfsense_backend/app/automations/runtime/retries.py @@ -0,0 +1,36 @@ +"""Retry policy enforcement for action handlers.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable + + +async def with_retries[T]( + coro_factory: Callable[[], Awaitable[T]], + *, + max_retries: int, + backoff: str, + timeout: int | None, +) -> tuple[T, int]: + """Call ``coro_factory`` up to ``1 + max_retries`` times. Return ``(result, attempts)``.""" + total = 1 + max(0, max_retries) + for attempt in range(1, total + 1): + try: + coro = coro_factory() + if timeout is not None and timeout > 0: + return await asyncio.wait_for(coro, timeout=timeout), attempt + return await coro, attempt + except Exception: + if attempt >= total: + raise + await asyncio.sleep(_backoff_seconds(backoff, attempt)) + raise RuntimeError("with_retries exhausted without raising or returning") + + +def _backoff_seconds(strategy: str, attempt: int) -> float: + if strategy == "exponential": + return float(2 ** (attempt - 1)) + if strategy == "linear": + return float(attempt) + return 0.0 diff --git a/surfsense_backend/app/automations/runtime/step.py b/surfsense_backend/app/automations/runtime/step.py new file mode 100644 index 000000000..ac18b5e1f --- /dev/null +++ b/surfsense_backend/app/automations/runtime/step.py @@ -0,0 +1,96 @@ +"""Execute one plan step: when-predicate, params render, handler dispatch, retries.""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import UTC, datetime +from typing import Any + +from app.automations.actions import get_action +from app.automations.actions.types import ActionContext +from app.automations.schemas.definition.plan_step import PlanStep +from app.automations.templating import evaluate_predicate, render_value + +from .retries import with_retries + + +async def execute_step( + *, + step: PlanStep, + template_context: Mapping[str, Any], + action_context: ActionContext, + default_max_retries: int, + default_retry_backoff: str, + default_timeout_seconds: int, +) -> dict[str, Any]: + """Run one step and return its structured result entry.""" + started_at = datetime.now(UTC) + + if step.when is not None: + try: + should_run = evaluate_predicate(step.when, template_context) + except Exception as exc: + return _result(step, "failed", started_at, attempts=0, error=_error(exc, "when")) + if not should_run: + return _result(step, "skipped", started_at, attempts=0) + + try: + resolved_params = render_value(step.params, template_context) + except Exception as exc: + return _result(step, "failed", started_at, attempts=0, error=_error(exc, "render")) + + action = get_action(step.action) + if action is None: + return _result( + step, + "failed", + started_at, + attempts=0, + error={"message": f"action not registered: {step.action}", "type": "ActionNotFound"}, + ) + + handler = action.build_handler(action_context) + + max_retries = step.max_retries if step.max_retries is not None else default_max_retries + timeout = step.timeout_seconds or default_timeout_seconds + + try: + result, attempts = await with_retries( + lambda: handler(resolved_params), + max_retries=max_retries, + backoff=default_retry_backoff, + timeout=timeout, + ) + except Exception as exc: + return _result(step, "failed", started_at, attempts=max_retries + 1, error=_error(exc)) + + return _result(step, "succeeded", started_at, attempts=attempts, result=result) + + +def _result( + step: PlanStep, + status: str, + started_at: datetime, + *, + attempts: int, + result: Any = None, + error: dict[str, Any] | None = None, +) -> dict[str, Any]: + entry: dict[str, Any] = { + "step_id": step.step_id, + "action": step.action, + "status": status, + "started_at": started_at.isoformat(), + "finished_at": datetime.now(UTC).isoformat(), + "attempts": attempts, + } + if result is not None: + entry["result"] = result + if error is not None: + entry["error"] = error + return entry + + +def _error(exc: Exception, phase: str | None = None) -> dict[str, Any]: + msg = f"{phase}: {exc}" if phase else str(exc) + return {"message": msg, "type": type(exc).__name__} diff --git a/surfsense_backend/app/automations/schemas/__init__.py b/surfsense_backend/app/automations/schemas/__init__.py new file mode 100644 index 000000000..2e2d60f12 --- /dev/null +++ b/surfsense_backend/app/automations/schemas/__init__.py @@ -0,0 +1,27 @@ +"""Schemas for the automation definition envelope. + +Per-action and per-trigger params schemas live with the action/trigger +implementations (``app.automations.actions..params`` / +``app.automations.triggers..params``); only the cross-cutting envelope +lives here. +""" + +from __future__ import annotations + +from .definition import ( + AutomationDefinition, + Execution, + Inputs, + Metadata, + PlanStep, + TriggerSpec, +) + +__all__ = [ + "AutomationDefinition", + "Execution", + "Inputs", + "Metadata", + "PlanStep", + "TriggerSpec", +] diff --git a/surfsense_backend/app/automations/schemas/api/__init__.py b/surfsense_backend/app/automations/schemas/api/__init__.py new file mode 100644 index 000000000..f49e5c589 --- /dev/null +++ b/surfsense_backend/app/automations/schemas/api/__init__.py @@ -0,0 +1,27 @@ +"""Request/response schemas for the automations HTTP layer.""" + +from __future__ import annotations + +from .automation import ( + AutomationCreate, + AutomationDetail, + AutomationList, + AutomationSummary, + AutomationUpdate, +) +from .run import RunDetail, RunList, RunSummary +from .trigger import TriggerCreate, TriggerDetail, TriggerUpdate + +__all__ = [ + "AutomationCreate", + "AutomationDetail", + "AutomationList", + "AutomationSummary", + "AutomationUpdate", + "RunDetail", + "RunList", + "RunSummary", + "TriggerCreate", + "TriggerDetail", + "TriggerUpdate", +] diff --git a/surfsense_backend/app/automations/schemas/api/automation.py b/surfsense_backend/app/automations/schemas/api/automation.py new file mode 100644 index 000000000..c1defd417 --- /dev/null +++ b/surfsense_backend/app/automations/schemas/api/automation.py @@ -0,0 +1,64 @@ +"""Request/response schemas for the ``Automation`` resource.""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field + +from app.automations.persistence.enums.automation_status import AutomationStatus +from app.automations.schemas.definition import AutomationDefinition + +from .trigger import TriggerCreate, TriggerDetail + + +class AutomationCreate(BaseModel): + """Create an automation, optionally with initial triggers (atomic).""" + + model_config = ConfigDict(extra="forbid") + + search_space_id: int + name: str = Field(..., min_length=1, max_length=200) + description: str | None = None + definition: AutomationDefinition + triggers: list[TriggerCreate] = Field(default_factory=list) + + +class AutomationUpdate(BaseModel): + """Partial update of an automation. Triggers are managed separately.""" + + model_config = ConfigDict(extra="forbid") + + name: str | None = Field(default=None, min_length=1, max_length=200) + description: str | None = None + status: AutomationStatus | None = None + definition: AutomationDefinition | None = None + + +class AutomationSummary(BaseModel): + """Lightweight automation view for list endpoints.""" + + model_config = ConfigDict(from_attributes=True) + + id: int + search_space_id: int + name: str + description: str | None = None + status: AutomationStatus + version: int + created_at: datetime + updated_at: datetime + + +class AutomationDetail(AutomationSummary): + """Full automation view including definition and attached triggers.""" + + definition: AutomationDefinition + triggers: list[TriggerDetail] = Field(default_factory=list) + + +class AutomationList(BaseModel): + """Paginated list of automations.""" + + items: list[AutomationSummary] + total: int diff --git a/surfsense_backend/app/automations/schemas/api/run.py b/surfsense_backend/app/automations/schemas/api/run.py new file mode 100644 index 000000000..3f6eaab82 --- /dev/null +++ b/surfsense_backend/app/automations/schemas/api/run.py @@ -0,0 +1,42 @@ +"""Response schemas for run sub-resources.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, ConfigDict + +from app.automations.persistence.enums.run_status import RunStatus + + +class RunSummary(BaseModel): + """Lightweight run view for list endpoints.""" + + model_config = ConfigDict(from_attributes=True) + + id: int + automation_id: int + trigger_id: int | None = None + status: RunStatus + started_at: datetime | None = None + finished_at: datetime | None = None + created_at: datetime + + +class RunDetail(RunSummary): + """Full run view including snapshot, results and artifacts.""" + + definition_snapshot: dict[str, Any] + inputs: dict[str, Any] + step_results: list[dict[str, Any]] + output: dict[str, Any] | None = None + artifacts: list[dict[str, Any]] + error: dict[str, Any] | None = None + + +class RunList(BaseModel): + """Paginated list of runs.""" + + items: list[RunSummary] + total: int diff --git a/surfsense_backend/app/automations/schemas/api/trigger.py b/surfsense_backend/app/automations/schemas/api/trigger.py new file mode 100644 index 000000000..35176fb9f --- /dev/null +++ b/surfsense_backend/app/automations/schemas/api/trigger.py @@ -0,0 +1,46 @@ +"""Request/response schemas for trigger sub-resources.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from app.automations.persistence.enums.trigger_type import TriggerType + + +class TriggerCreate(BaseModel): + """Attach a trigger to an automation.""" + + model_config = ConfigDict(extra="forbid") + + type: TriggerType + params: dict[str, Any] = Field(default_factory=dict) + static_inputs: dict[str, Any] = Field(default_factory=dict) + enabled: bool = True + + +class TriggerUpdate(BaseModel): + """Partial update of an existing trigger.""" + + model_config = ConfigDict(extra="forbid") + + enabled: bool | None = None + params: dict[str, Any] | None = None + static_inputs: dict[str, Any] | None = None + + +class TriggerDetail(BaseModel): + """Trigger as returned to clients.""" + + model_config = ConfigDict(from_attributes=True) + + id: int + type: TriggerType + params: dict[str, Any] + static_inputs: dict[str, Any] + enabled: bool + last_fired_at: datetime | None = None + next_fire_at: datetime | None = None + created_at: datetime diff --git a/surfsense_backend/app/automations/schemas/definition/__init__.py b/surfsense_backend/app/automations/schemas/definition/__init__.py new file mode 100644 index 000000000..3fb0a739b --- /dev/null +++ b/surfsense_backend/app/automations/schemas/definition/__init__.py @@ -0,0 +1,19 @@ +"""Automation definition envelope and its components.""" + +from __future__ import annotations + +from .envelope import AutomationDefinition +from .execution import Execution +from .inputs import Inputs +from .metadata import Metadata +from .plan_step import PlanStep +from .trigger_spec import TriggerSpec + +__all__ = [ + "AutomationDefinition", + "Execution", + "Inputs", + "Metadata", + "PlanStep", + "TriggerSpec", +] diff --git a/surfsense_backend/app/automations/schemas/definition/envelope.py b/surfsense_backend/app/automations/schemas/definition/envelope.py new file mode 100644 index 000000000..f919b2abb --- /dev/null +++ b/surfsense_backend/app/automations/schemas/definition/envelope.py @@ -0,0 +1,26 @@ +"""``AutomationDefinition`` — top-level envelope persisted in ``automations.definition``.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + +from .execution import Execution +from .inputs import Inputs +from .metadata import Metadata +from .plan_step import PlanStep +from .trigger_spec import TriggerSpec + + +class AutomationDefinition(BaseModel): + """Top-level shape of an automation.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: str = "1.0" + name: str = Field(..., min_length=1, max_length=200) + goal: str | None = None + inputs: Inputs | None = None + triggers: list[TriggerSpec] = Field(default_factory=list) + plan: list[PlanStep] = Field(..., min_length=1) + execution: Execution = Field(default_factory=Execution) + metadata: Metadata = Field(default_factory=Metadata) diff --git a/surfsense_backend/app/automations/schemas/definition/execution.py b/surfsense_backend/app/automations/schemas/definition/execution.py new file mode 100644 index 000000000..61861f8d8 --- /dev/null +++ b/surfsense_backend/app/automations/schemas/definition/execution.py @@ -0,0 +1,22 @@ +"""``Execution`` — automation-wide execution defaults (overridable per step).""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from .plan_step import PlanStep + + +class Execution(BaseModel): + model_config = ConfigDict(extra="forbid") + + timeout_seconds: int = Field(default=600, gt=0, description="Wall-clock cap for the run.") + max_retries: int = Field(default=2, ge=0, description="Per-step retry budget.") + retry_backoff: Literal["exponential", "linear", "none"] = "exponential" + concurrency: Literal["drop_if_running", "queue", "always"] = "drop_if_running" + on_failure: list[PlanStep] = Field( + default_factory=list, + description="Steps run when the main plan fails after retries.", + ) diff --git a/surfsense_backend/app/automations/schemas/definition/inputs.py b/surfsense_backend/app/automations/schemas/definition/inputs.py new file mode 100644 index 000000000..619fd16cd --- /dev/null +++ b/surfsense_backend/app/automations/schemas/definition/inputs.py @@ -0,0 +1,21 @@ +"""``Inputs`` — JSON Schema for inputs an automation accepts at fire time.""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class Inputs(BaseModel): + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + serialize_by_alias=True, + ) + + schema_: dict[str, Any] = Field( + ..., + alias="schema", + description="JSON Schema (draft 2020-12) for accepted inputs.", + ) diff --git a/surfsense_backend/app/automations/schemas/definition/metadata.py b/surfsense_backend/app/automations/schemas/definition/metadata.py new file mode 100644 index 000000000..3ac341d2e --- /dev/null +++ b/surfsense_backend/app/automations/schemas/definition/metadata.py @@ -0,0 +1,11 @@ +"""``Metadata`` — free-form metadata on a definition. Extra keys allowed.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + + +class Metadata(BaseModel): + model_config = ConfigDict(extra="allow") + + tags: list[str] = Field(default_factory=list) diff --git a/surfsense_backend/app/automations/schemas/definition/plan_step.py b/surfsense_backend/app/automations/schemas/definition/plan_step.py new file mode 100644 index 000000000..5d16f1f3e --- /dev/null +++ b/surfsense_backend/app/automations/schemas/definition/plan_step.py @@ -0,0 +1,28 @@ +"""``PlanStep`` — one step in the sequential plan.""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class PlanStep(BaseModel): + model_config = ConfigDict(extra="forbid") + + step_id: str = Field(..., min_length=1, description="Unique within the plan.") + action: str = Field(..., min_length=1, description="Action type; resolved via registry.") + when: str | None = Field( + default=None, + description="Optional predicate; step is skipped when falsy.", + ) + params: dict[str, Any] = Field( + default_factory=dict, + description="Action-type-specific params; rendered at execute time.", + ) + output_as: str | None = Field( + default=None, + description="Bind step output under this name. Defaults to step_id.", + ) + max_retries: int | None = Field(default=None, ge=0) + timeout_seconds: int | None = Field(default=None, gt=0) diff --git a/surfsense_backend/app/automations/schemas/definition/trigger_spec.py b/surfsense_backend/app/automations/schemas/definition/trigger_spec.py new file mode 100644 index 000000000..a359a2f63 --- /dev/null +++ b/surfsense_backend/app/automations/schemas/definition/trigger_spec.py @@ -0,0 +1,17 @@ +"""``TriggerSpec`` — one entry in the definition's ``triggers[]`` array.""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class TriggerSpec(BaseModel): + model_config = ConfigDict(extra="forbid") + + type: str = Field(..., min_length=1, description="Trigger type; resolved via registry.") + params: dict[str, Any] = Field( + default_factory=dict, + description="Type-specific params; validated against the trigger's schema.", + ) diff --git a/surfsense_backend/app/automations/services/__init__.py b/surfsense_backend/app/automations/services/__init__.py new file mode 100644 index 000000000..597aca98a --- /dev/null +++ b/surfsense_backend/app/automations/services/__init__.py @@ -0,0 +1,16 @@ +"""Services for the automations HTTP layer (one service per resource).""" + +from __future__ import annotations + +from .automation import AutomationService, get_automation_service +from .run import RunService, get_run_service +from .trigger import TriggerService, get_trigger_service + +__all__ = [ + "AutomationService", + "RunService", + "TriggerService", + "get_automation_service", + "get_run_service", + "get_trigger_service", +] diff --git a/surfsense_backend/app/automations/services/automation.py b/surfsense_backend/app/automations/services/automation.py new file mode 100644 index 000000000..9140da3b5 --- /dev/null +++ b/surfsense_backend/app/automations/services/automation.py @@ -0,0 +1,172 @@ +"""``AutomationService`` — orchestration for the ``Automation`` resource.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from fastapi import Depends, HTTPException +from pydantic import ValidationError +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.automations.schemas.api import ( + AutomationCreate, + AutomationUpdate, + TriggerCreate, +) +from app.automations.persistence.enums.trigger_type import TriggerType +from app.automations.persistence.models.automation import Automation +from app.automations.persistence.models.trigger import AutomationTrigger +from app.automations.triggers import get_trigger +from app.automations.triggers.schedule import compute_next_fire_at +from app.db import Permission, User, get_async_session +from app.users import current_active_user +from app.utils.rbac import check_permission + + +class AutomationService: + """Lifecycle of the ``Automation`` resource.""" + + def __init__(self, *, session: AsyncSession, user: User) -> None: + self.session = session + self.user = user + + async def create(self, payload: AutomationCreate) -> Automation: + """Create an automation and its initial triggers in one transaction.""" + await self._authorize(payload.search_space_id, Permission.AUTOMATIONS_CREATE.value) + + automation = Automation( + search_space_id=payload.search_space_id, + created_by_user_id=self.user.id, + name=payload.name, + description=payload.description, + definition=payload.definition.model_dump(mode="json", by_alias=True), + version=1, + ) + for spec in payload.triggers: + automation.triggers.append(_build_trigger(spec)) + + self.session.add(automation) + await self.session.commit() + return await self._get_with_triggers_or_raise(automation.id) + + async def list( + self, + *, + search_space_id: int, + limit: int, + offset: int, + ) -> tuple[list[Automation], int]: + """Return a page of automations and the total count.""" + await self._authorize(search_space_id, Permission.AUTOMATIONS_READ.value) + + base = select(Automation).where(Automation.search_space_id == search_space_id) + total = await self.session.scalar( + select(func.count()).select_from(base.subquery()) + ) + + rows = ( + await self.session.execute( + base.order_by(Automation.created_at.desc()).limit(limit).offset(offset) + ) + ).scalars().all() + return list(rows), int(total or 0) + + async def get(self, automation_id: int) -> Automation: + """Get an automation with its triggers loaded.""" + automation = await self._get_with_triggers_or_raise(automation_id) + await self._authorize(automation.search_space_id, Permission.AUTOMATIONS_READ.value) + return automation + + async def update(self, automation_id: int, patch: AutomationUpdate) -> Automation: + """Patch fields. Bumps ``version`` when ``definition`` changes.""" + automation = await self._get_with_triggers_or_raise(automation_id) + await self._authorize(automation.search_space_id, Permission.AUTOMATIONS_UPDATE.value) + + data = patch.model_dump(exclude_unset=True) + + if "name" in data: + automation.name = data["name"] + if "description" in data: + automation.description = data["description"] + if "status" in data: + automation.status = data["status"] + if "definition" in data: + automation.definition = patch.definition.model_dump(mode="json", by_alias=True) + automation.version += 1 + + await self.session.commit() + return await self._get_with_triggers_or_raise(automation_id) + + async def delete(self, automation_id: int) -> None: + """Delete an automation; FK cascades remove triggers and runs.""" + automation = await self._get_or_raise(automation_id) + await self._authorize(automation.search_space_id, Permission.AUTOMATIONS_DELETE.value) + await self.session.delete(automation) + await self.session.commit() + + async def _get_or_raise(self, automation_id: int) -> Automation: + automation = await self.session.get(Automation, automation_id) + if automation is None: + raise HTTPException( + status_code=404, detail=f"automation {automation_id} not found" + ) + return automation + + async def _get_with_triggers_or_raise(self, automation_id: int) -> Automation: + stmt = ( + select(Automation) + .where(Automation.id == automation_id) + .options(selectinload(Automation.triggers)) + ) + automation = (await self.session.execute(stmt)).scalar_one_or_none() + if automation is None: + raise HTTPException( + status_code=404, detail=f"automation {automation_id} not found" + ) + return automation + + async def _authorize(self, search_space_id: int, permission: str) -> None: + await check_permission( + self.session, + self.user, + search_space_id, + permission, + f"You don't have permission to {permission.split(':')[1]} automations in this search space", + ) + + +def _build_trigger(spec: TriggerCreate) -> AutomationTrigger: + """Validate trigger params via its registered Pydantic model and build the ORM row.""" + definition = get_trigger(spec.type.value) + if definition is None: + raise HTTPException(status_code=422, detail=f"unknown trigger type {spec.type.value!r}") + + try: + validated = definition.params_model.model_validate(spec.params) + except ValidationError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + + params = validated.model_dump(mode="json") + + next_fire_at = None + if spec.type == TriggerType.SCHEDULE and spec.enabled: + next_fire_at = compute_next_fire_at( + params["cron"], params["timezone"], after=datetime.now(UTC) + ) + + return AutomationTrigger( + type=spec.type, + params=params, + static_inputs=spec.static_inputs, + enabled=spec.enabled, + next_fire_at=next_fire_at, + ) + + +def get_automation_service( + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +) -> AutomationService: + return AutomationService(session=session, user=user) diff --git a/surfsense_backend/app/automations/services/run.py b/surfsense_backend/app/automations/services/run.py new file mode 100644 index 000000000..ac9970241 --- /dev/null +++ b/surfsense_backend/app/automations/services/run.py @@ -0,0 +1,72 @@ +"""``RunService`` — read-only access to automation run history.""" + +from __future__ import annotations + +from fastapi import Depends, HTTPException +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.automations.persistence.models.automation import Automation +from app.automations.persistence.models.run import AutomationRun +from app.db import Permission, User, get_async_session +from app.users import current_active_user +from app.utils.rbac import check_permission + + +class RunService: + """Read-only access to ``AutomationRun`` history.""" + + def __init__(self, *, session: AsyncSession, user: User) -> None: + self.session = session + self.user = user + + async def list( + self, + *, + automation_id: int, + limit: int, + offset: int, + ) -> tuple[list[AutomationRun], int]: + """Return a page of runs for an automation, newest first.""" + await self._authorize(automation_id, Permission.AUTOMATIONS_READ.value) + + base = select(AutomationRun).where(AutomationRun.automation_id == automation_id) + total = await self.session.scalar( + select(func.count()).select_from(base.subquery()) + ) + + rows = ( + await self.session.execute( + base.order_by(AutomationRun.created_at.desc()).limit(limit).offset(offset) + ) + ).scalars().all() + return list(rows), int(total or 0) + + async def get(self, *, automation_id: int, run_id: int) -> AutomationRun: + await self._authorize(automation_id, Permission.AUTOMATIONS_READ.value) + run = await self.session.get(AutomationRun, run_id) + if run is None or run.automation_id != automation_id: + raise HTTPException(status_code=404, detail=f"run {run_id} not found") + return run + + async def _authorize(self, automation_id: int, permission: str) -> Automation: + automation = await self.session.get(Automation, automation_id) + if automation is None: + raise HTTPException( + status_code=404, detail=f"automation {automation_id} not found" + ) + await check_permission( + self.session, + self.user, + automation.search_space_id, + permission, + f"You don't have permission to {permission.split(':')[1]} automations in this search space", + ) + return automation + + +def get_run_service( + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +) -> RunService: + return RunService(session=session, user=user) diff --git a/surfsense_backend/app/automations/services/trigger.py b/surfsense_backend/app/automations/services/trigger.py new file mode 100644 index 000000000..c76cc0740 --- /dev/null +++ b/surfsense_backend/app/automations/services/trigger.py @@ -0,0 +1,143 @@ +"""``TriggerService`` — lifecycle of triggers attached to an automation.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from fastapi import Depends, HTTPException +from pydantic import ValidationError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.automations.schemas.api import TriggerCreate, TriggerUpdate +from app.automations.persistence.enums.trigger_type import TriggerType +from app.automations.persistence.models.automation import Automation +from app.automations.persistence.models.trigger import AutomationTrigger +from app.automations.triggers import get_trigger +from app.automations.triggers.schedule import compute_next_fire_at +from app.db import Permission, User, get_async_session +from app.users import current_active_user +from app.utils.rbac import check_permission + + +class TriggerService: + """Lifecycle of the ``AutomationTrigger`` sub-resource.""" + + def __init__(self, *, session: AsyncSession, user: User) -> None: + self.session = session + self.user = user + + async def add( + self, *, automation_id: int, payload: TriggerCreate + ) -> AutomationTrigger: + automation = await self._authorize_automation( + automation_id, Permission.AUTOMATIONS_UPDATE.value + ) + + validated_params = _validate_params(payload.type, payload.params) + trigger = AutomationTrigger( + automation_id=automation.id, + type=payload.type, + params=validated_params, + static_inputs=payload.static_inputs, + enabled=payload.enabled, + next_fire_at=_initial_next_fire(payload.type, validated_params, payload.enabled), + ) + self.session.add(trigger) + await self.session.commit() + await self.session.refresh(trigger) + return trigger + + async def update( + self, + *, + automation_id: int, + trigger_id: int, + patch: TriggerUpdate, + ) -> AutomationTrigger: + await self._authorize_automation(automation_id, Permission.AUTOMATIONS_UPDATE.value) + trigger = await self._get_trigger_or_raise(automation_id, trigger_id) + + data = patch.model_dump(exclude_unset=True) + + if "params" in data: + trigger.params = _validate_params(trigger.type, data["params"]) + + if "static_inputs" in data: + trigger.static_inputs = data["static_inputs"] + + if "enabled" in data: + trigger.enabled = data["enabled"] + + # Recompute next_fire_at when schedule timing changed or the trigger was + # toggled back on. + if trigger.type == TriggerType.SCHEDULE: + trigger.next_fire_at = _initial_next_fire( + trigger.type, trigger.params, trigger.enabled + ) + + await self.session.commit() + await self.session.refresh(trigger) + return trigger + + async def remove(self, *, automation_id: int, trigger_id: int) -> None: + await self._authorize_automation(automation_id, Permission.AUTOMATIONS_UPDATE.value) + trigger = await self._get_trigger_or_raise(automation_id, trigger_id) + await self.session.delete(trigger) + await self.session.commit() + + async def _authorize_automation( + self, automation_id: int, permission: str + ) -> Automation: + automation = await self.session.get(Automation, automation_id) + if automation is None: + raise HTTPException( + status_code=404, detail=f"automation {automation_id} not found" + ) + await check_permission( + self.session, + self.user, + automation.search_space_id, + permission, + f"You don't have permission to {permission.split(':')[1]} automations in this search space", + ) + return automation + + async def _get_trigger_or_raise( + self, automation_id: int, trigger_id: int + ) -> AutomationTrigger: + trigger = await self.session.get(AutomationTrigger, trigger_id) + if trigger is None or trigger.automation_id != automation_id: + raise HTTPException( + status_code=404, detail=f"trigger {trigger_id} not found" + ) + return trigger + + +def _validate_params(trigger_type: TriggerType, raw: dict) -> dict: + definition = get_trigger(trigger_type.value) + if definition is None: + raise HTTPException( + status_code=422, detail=f"unknown trigger type {trigger_type.value!r}" + ) + try: + validated = definition.params_model.model_validate(raw) + except ValidationError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + return validated.model_dump(mode="json") + + +def _initial_next_fire( + trigger_type: TriggerType, params: dict, enabled: bool +) -> datetime | None: + if trigger_type != TriggerType.SCHEDULE or not enabled: + return None + return compute_next_fire_at( + params["cron"], params["timezone"], after=datetime.now(UTC) + ) + + +def get_trigger_service( + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +) -> TriggerService: + return TriggerService(session=session, user=user) diff --git a/surfsense_backend/app/automations/tasks/__init__.py b/surfsense_backend/app/automations/tasks/__init__.py new file mode 100644 index 000000000..6fe0d62e8 --- /dev/null +++ b/surfsense_backend/app/automations/tasks/__init__.py @@ -0,0 +1,3 @@ +"""Celery task wrappers for the automation runtime.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/automations/tasks/execute_run.py b/surfsense_backend/app/automations/tasks/execute_run.py new file mode 100644 index 000000000..5fc84698b --- /dev/null +++ b/surfsense_backend/app/automations/tasks/execute_run.py @@ -0,0 +1,33 @@ +"""Celery task that runs one automation. Thin wrapper over ``runtime.executor``.""" + +from __future__ import annotations + +import logging + +from app.automations.runtime import execute_run +from app.celery_app import celery_app +from app.tasks.celery_tasks import ( + get_celery_session_maker, + run_async_celery_task, +) + +logger = logging.getLogger(__name__) + +TASK_NAME = "automation_run_execute" + + +@celery_app.task(name=TASK_NAME, bind=True) +def automation_run_execute(self, run_id: int) -> None: # noqa: ARG001 — Celery bind + """Execute one ``AutomationRun``. Idempotent: terminal runs no-op.""" + return run_async_celery_task(lambda: _impl(run_id)) + + +async def _impl(run_id: int) -> None: + session_maker = get_celery_session_maker() + async with session_maker() as session: + try: + await execute_run(session, run_id) + except Exception: + logger.exception("automation_run %d failed unexpectedly", run_id) + await session.rollback() + raise diff --git a/surfsense_backend/app/automations/tasks/schedule_tick.py b/surfsense_backend/app/automations/tasks/schedule_tick.py new file mode 100644 index 000000000..385bd7242 --- /dev/null +++ b/surfsense_backend/app/automations/tasks/schedule_tick.py @@ -0,0 +1,187 @@ +"""Celery Beat tick that fires due ``schedule`` triggers. + +Runs every minute. Each tick performs two passes: + +1. **Self-heal**: enabled schedule triggers with NULL ``next_fire_at`` get + it computed from their ``cron`` + ``timezone`` (e.g. fresh inserts or + rows restored from backup). +2. **Claim & fire**: due rows are locked with ``FOR UPDATE SKIP LOCKED``, + their ``next_fire_at`` is advanced and ``last_fired_at`` is set, and + ``dispatch_schedule_run`` is invoked for each. Dispatch errors are + logged; a missed fire stays missed (matches K8s CronJob / Airflow + ``catchup=False`` semantics). +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from datetime import UTC, datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.automations.persistence.enums.trigger_type import TriggerType +from app.automations.persistence.models.trigger import AutomationTrigger +from app.automations.triggers.schedule import ( + InvalidCronError, + compute_next_fire_at, + dispatch_schedule_run, +) +from app.celery_app import celery_app +from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task + +logger = logging.getLogger(__name__) + +TASK_NAME = "automation_schedule_tick" + +# Cap rows touched per tick so a backlog of due triggers can't starve the +# worker; remaining rows fire on the next tick. +_TICK_BATCH = 200 + + +@dataclass(frozen=True, slots=True) +class _Claim: + """Per-trigger fire context captured before row state is mutated.""" + + trigger_id: int + scheduled_for: datetime + previous_last_fired_at: datetime | None + + +@celery_app.task(name=TASK_NAME) +def automation_schedule_tick() -> None: + """Tick once: self-heal NULL next_fire_at, claim due rows, fire each.""" + return run_async_celery_task(_tick) + + +async def _tick() -> None: + session_maker = get_celery_session_maker() + async with session_maker() as session: + now = datetime.now(UTC) + + await _self_heal_null_next_fire(session, now=now) + + claims = await _claim_due_triggers(session, now=now) + if not claims: + return + + for claim in claims: + await _fire_one(session, claim=claim, fired_at=now) + + +async def _self_heal_null_next_fire(session: AsyncSession, *, now: datetime) -> None: + """Backfill ``next_fire_at`` for enabled schedule triggers missing it.""" + stmt = ( + select(AutomationTrigger) + .where( + AutomationTrigger.type == TriggerType.SCHEDULE, + AutomationTrigger.enabled.is_(True), + AutomationTrigger.next_fire_at.is_(None), + ) + .limit(_TICK_BATCH) + ) + triggers = (await session.execute(stmt)).scalars().all() + if not triggers: + return + + for trigger in triggers: + try: + trigger.next_fire_at = compute_next_fire_at( + trigger.params["cron"], + trigger.params["timezone"], + after=now, + ) + except (InvalidCronError, KeyError, TypeError) as exc: + logger.warning( + "automation_trigger %d has invalid schedule params, disabling: %s", + trigger.id, + exc, + ) + trigger.enabled = False + + await session.commit() + + +async def _claim_due_triggers( + session: AsyncSession, *, now: datetime +) -> list[_Claim]: + """Lock and advance due rows; return per-trigger fire context.""" + stmt = ( + select(AutomationTrigger) + .where( + AutomationTrigger.type == TriggerType.SCHEDULE, + AutomationTrigger.enabled.is_(True), + AutomationTrigger.next_fire_at.isnot(None), + AutomationTrigger.next_fire_at <= now, + ) + .order_by(AutomationTrigger.next_fire_at) + .limit(_TICK_BATCH) + .with_for_update(skip_locked=True) + ) + triggers = (await session.execute(stmt)).scalars().all() + if not triggers: + return [] + + claims: list[_Claim] = [] + for trigger in triggers: + # Snapshot fire-context BEFORE we advance the row. + scheduled_for = trigger.next_fire_at + previous_last_fired_at = trigger.last_fired_at + + try: + trigger.next_fire_at = compute_next_fire_at( + trigger.params["cron"], + trigger.params["timezone"], + after=now, + ) + except (InvalidCronError, KeyError, TypeError) as exc: + logger.warning( + "automation_trigger %d has invalid schedule params, disabling: %s", + trigger.id, + exc, + ) + trigger.enabled = False + continue + + trigger.last_fired_at = now + claims.append( + _Claim( + trigger_id=trigger.id, + scheduled_for=scheduled_for, + previous_last_fired_at=previous_last_fired_at, + ) + ) + + await session.commit() + return claims + + +async def _fire_one( + session: AsyncSession, *, claim: _Claim, fired_at: datetime +) -> None: + """Reload the trigger post-commit and dispatch a run for it.""" + trigger = await session.get(AutomationTrigger, claim.trigger_id) + if trigger is None: + return + + try: + run = await dispatch_schedule_run( + session=session, + trigger=trigger, + fired_at=fired_at, + scheduled_for=claim.scheduled_for, + previous_last_fired_at=claim.previous_last_fired_at, + ) + logger.info( + "scheduled fire: trigger=%d automation=%d run=%d", + claim.trigger_id, + trigger.automation_id, + run.id, + ) + except Exception: + logger.exception( + "scheduled fire failed for trigger %d (next attempt at next match)", + claim.trigger_id, + ) + await session.rollback() diff --git a/surfsense_backend/app/automations/templating/__init__.py b/surfsense_backend/app/automations/templating/__init__.py new file mode 100644 index 000000000..1df1809c7 --- /dev/null +++ b/surfsense_backend/app/automations/templating/__init__.py @@ -0,0 +1,13 @@ +"""Sandboxed template engine for automation definitions.""" + +from __future__ import annotations + +from .context import build_run_context +from .render import evaluate_predicate, render_template, render_value + +__all__ = [ + "build_run_context", + "evaluate_predicate", + "render_template", + "render_value", +] diff --git a/surfsense_backend/app/automations/templating/allowlist.py b/surfsense_backend/app/automations/templating/allowlist.py new file mode 100644 index 000000000..ed0103c8f --- /dev/null +++ b/surfsense_backend/app/automations/templating/allowlist.py @@ -0,0 +1,31 @@ +"""Filter and test names admitted into the sandboxed environment.""" + +from __future__ import annotations + +ALLOWED_FILTERS: tuple[str, ...] = ( + "default", + "first", + "join", + "last", + "length", + "lower", + "replace", + "reverse", + "sort", + "tojson", + "trim", + "truncate", + "upper", + "date", + "slugify", +) + +ALLOWED_TESTS: tuple[str, ...] = ( + "defined", + "none", + "number", + "string", + "mapping", + "sequence", + "boolean", +) diff --git a/surfsense_backend/app/automations/templating/context.py b/surfsense_backend/app/automations/templating/context.py new file mode 100644 index 000000000..96fdb02e9 --- /dev/null +++ b/surfsense_backend/app/automations/templating/context.py @@ -0,0 +1,41 @@ +"""Builder for the ``{run, inputs, steps}`` namespace exposed to every template.""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import datetime +from typing import Any + + +def build_run_context( + *, + run_id: int, + automation_id: int, + automation_name: str | None, + automation_version: int | None, + search_space_id: int | None, + creator_id: Any, + trigger_id: int | None, + trigger_type: str | None, + started_at: datetime | None, + attempt: int, + inputs: Mapping[str, Any], + step_outputs: Mapping[str, Any], +) -> dict[str, Any]: + """Build the ``{run, inputs, steps}`` namespace exposed to every template.""" + return { + "run": { + "id": run_id, + "automation_id": automation_id, + "automation_name": automation_name, + "automation_version": automation_version, + "search_space_id": search_space_id, + "creator_id": creator_id, + "trigger_id": trigger_id, + "trigger_type": trigger_type, + "started_at": started_at, + "attempt": attempt, + }, + "inputs": dict(inputs), + "steps": dict(step_outputs), + } diff --git a/surfsense_backend/app/automations/templating/environment.py b/surfsense_backend/app/automations/templating/environment.py new file mode 100644 index 000000000..6ac5f7361 --- /dev/null +++ b/surfsense_backend/app/automations/templating/environment.py @@ -0,0 +1,43 @@ +"""SandboxedEnvironment construction with the audited filter/test allowlist.""" + +from __future__ import annotations + +import json +from datetime import datetime +from typing import Any + +from jinja2 import StrictUndefined +from jinja2.sandbox import SandboxedEnvironment + +from .allowlist import ALLOWED_FILTERS, ALLOWED_TESTS +from .filters import filter_date, filter_slugify + + +def _finalize(value: Any) -> Any: + """Stringify common non-string values at output sites.""" + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, list | dict): + return json.dumps(value, ensure_ascii=False, default=str) + return value + + +def _build_env() -> SandboxedEnvironment: + env = SandboxedEnvironment( + autoescape=False, + undefined=StrictUndefined, + finalize=_finalize, + ) + env.globals.clear() + env.filters = {k: v for k, v in env.filters.items() if k in ALLOWED_FILTERS} + env.filters["date"] = filter_date + env.filters["slugify"] = filter_slugify + env.tests = {k: v for k, v in env.tests.items() if k in ALLOWED_TESTS} + return env + + +ENV: SandboxedEnvironment = _build_env() diff --git a/surfsense_backend/app/automations/templating/filters.py b/surfsense_backend/app/automations/templating/filters.py new file mode 100644 index 000000000..65f66eb37 --- /dev/null +++ b/surfsense_backend/app/automations/templating/filters.py @@ -0,0 +1,29 @@ +"""Custom Jinja filters registered into the sandboxed environment.""" + +from __future__ import annotations + +import re +from typing import Any + + +def filter_date(value: Any, fmt: str = "%Y-%m-%d") -> str: + """Format a datetime-like value with ``strftime``. Strings pass through.""" + if value is None: + return "" + if isinstance(value, str): + return value + if hasattr(value, "strftime"): + return value.strftime(fmt) + raise ValueError(f"date filter requires datetime-like, got {type(value).__name__}") + + +_SLUG_NONALNUM = re.compile(r"[^a-z0-9]+") +_SLUG_DASHES = re.compile(r"-+") + + +def filter_slugify(value: Any) -> str: + """Lowercase, replace non-alphanumerics with hyphens, collapse and trim.""" + s = str(value).lower() + s = _SLUG_NONALNUM.sub("-", s) + s = _SLUG_DASHES.sub("-", s) + return s.strip("-") diff --git a/surfsense_backend/app/automations/templating/render.py b/surfsense_backend/app/automations/templating/render.py new file mode 100644 index 000000000..42721ddeb --- /dev/null +++ b/surfsense_backend/app/automations/templating/render.py @@ -0,0 +1,29 @@ +"""Render templates and evaluate predicates against the sandboxed environment.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from .environment import ENV + + +def render_template(template: str, context: Mapping[str, Any]) -> str: + """Render ``template`` with ``context``.""" + return ENV.from_string(template).render(**context) + + +def evaluate_predicate(expression: str, context: Mapping[str, Any]) -> bool: + """Evaluate a Jinja expression (not a template body) and coerce to bool.""" + return bool(ENV.compile_expression(expression)(**context)) + + +def render_value(value: Any, context: Mapping[str, Any]) -> Any: + """Recursively render every string in a JSON-like value structure.""" + if isinstance(value, str): + return render_template(value, context) + if isinstance(value, dict): + return {k: render_value(v, context) for k, v in value.items()} + if isinstance(value, list): + return [render_value(v, context) for v in value] + return value diff --git a/surfsense_backend/app/automations/triggers/__init__.py b/surfsense_backend/app/automations/triggers/__init__.py new file mode 100644 index 000000000..d7abb6b5d --- /dev/null +++ b/surfsense_backend/app/automations/triggers/__init__.py @@ -0,0 +1,20 @@ +"""Triggers domain: registry surface + built-in trigger packages. + +Each trigger lives in its own subpackage (``schedule/``, ...) and +self-registers at import time via its ``definition`` module. +""" + +from __future__ import annotations + +from .store import all_triggers, get_trigger, register_trigger +from .types import TriggerDefinition + +__all__ = [ + "TriggerDefinition", + "all_triggers", + "get_trigger", + "register_trigger", +] + +# Built-in triggers self-register at import time. +from . import schedule # noqa: E402, F401 diff --git a/surfsense_backend/app/automations/triggers/schedule/__init__.py b/surfsense_backend/app/automations/triggers/schedule/__init__.py new file mode 100644 index 000000000..5587692b9 --- /dev/null +++ b/surfsense_backend/app/automations/triggers/schedule/__init__.py @@ -0,0 +1,18 @@ +"""``schedule`` trigger: fired on a cron schedule in a given timezone.""" + +from __future__ import annotations + +from .cron import InvalidCronError, compute_next_fire_at, validate_cron +from .dispatch import dispatch_schedule_run +from .params import ScheduleTriggerParams + +__all__ = [ + "InvalidCronError", + "ScheduleTriggerParams", + "compute_next_fire_at", + "dispatch_schedule_run", + "validate_cron", +] + +# Side-effect: register on the triggers store. +from . import definition # noqa: E402, F401 diff --git a/surfsense_backend/app/automations/triggers/schedule/cron.py b/surfsense_backend/app/automations/triggers/schedule/cron.py new file mode 100644 index 000000000..7155bab33 --- /dev/null +++ b/surfsense_backend/app/automations/triggers/schedule/cron.py @@ -0,0 +1,37 @@ +"""Cron math for the ``schedule`` trigger: validate + advance ``next_fire_at``.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from croniter import CroniterBadCronError, croniter + + +class InvalidCronError(ValueError): + """Raised when a cron expression or timezone fails validation.""" + + +def validate_cron(cron: str, timezone: str) -> None: + """Raise ``InvalidCronError`` if cron or timezone are unusable.""" + try: + ZoneInfo(timezone) + except ZoneInfoNotFoundError as exc: + raise InvalidCronError(f"unknown timezone {timezone!r}") from exc + + try: + croniter(cron) + except (CroniterBadCronError, ValueError) as exc: + raise InvalidCronError(f"invalid cron {cron!r}: {exc}") from exc + + +def compute_next_fire_at(cron: str, timezone: str, *, after: datetime) -> datetime: + """Return the next moment matching ``cron`` in ``timezone`` strictly after ``after``. + + The result is normalized to UTC for storage. ``after`` is converted into the + given timezone before evaluation so DST and IANA rules apply correctly. + """ + tz = ZoneInfo(timezone) + base = after.astimezone(tz) if after.tzinfo else after.replace(tzinfo=UTC).astimezone(tz) + nxt: datetime = croniter(cron, base).get_next(datetime) + return nxt.astimezone(UTC) diff --git a/surfsense_backend/app/automations/triggers/schedule/definition.py b/surfsense_backend/app/automations/triggers/schedule/definition.py new file mode 100644 index 000000000..605765307 --- /dev/null +++ b/surfsense_backend/app/automations/triggers/schedule/definition.py @@ -0,0 +1,15 @@ +"""``schedule`` ``TriggerDefinition`` registration.""" + +from __future__ import annotations + +from ..store import register_trigger +from ..types import TriggerDefinition +from .params import ScheduleTriggerParams + +SCHEDULE_TRIGGER = TriggerDefinition( + type="schedule", + description="Fire on a cron schedule in a given timezone.", + params_model=ScheduleTriggerParams, +) + +register_trigger(SCHEDULE_TRIGGER) diff --git a/surfsense_backend/app/automations/triggers/schedule/dispatch.py b/surfsense_backend/app/automations/triggers/schedule/dispatch.py new file mode 100644 index 000000000..6d3d5fcb9 --- /dev/null +++ b/surfsense_backend/app/automations/triggers/schedule/dispatch.py @@ -0,0 +1,67 @@ +"""Schedule dispatch adapter: load + guard, then call generic dispatch.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.automations.dispatch import DispatchError, dispatch_run +from app.automations.persistence.enums.automation_status import AutomationStatus +from app.automations.persistence.models.automation import Automation +from app.automations.persistence.models.run import AutomationRun +from app.automations.persistence.models.trigger import AutomationTrigger + + +async def dispatch_schedule_run( + *, + session: AsyncSession, + trigger: AutomationTrigger, + fired_at: datetime, + scheduled_for: datetime, + previous_last_fired_at: datetime | None, +) -> AutomationRun: + """Fire one scheduled run for ``trigger``. + + Emits calendar context as runtime inputs: + + - ``fired_at`` — actual fire time + - ``scheduled_for`` — cron-derived target time for this fire + - ``last_fired_at`` — fire time of the previous run, or null on first fire + + The caller (the schedule tick) is responsible for selecting due triggers + and advancing ``next_fire_at`` / ``last_fired_at`` before invoking this. + """ + automation = await _load_automation(session, trigger.automation_id) + if automation is None: + raise DispatchError( + f"automation {trigger.automation_id} not found for trigger {trigger.id}" + ) + + if automation.status != AutomationStatus.ACTIVE: + raise DispatchError( + f"automation {trigger.automation_id} is {automation.status.value}, not active" + ) + + runtime_inputs = { + "fired_at": fired_at.isoformat(), + "scheduled_for": scheduled_for.isoformat(), + "last_fired_at": ( + previous_last_fired_at.isoformat() if previous_last_fired_at else None + ), + } + + return await dispatch_run( + session=session, + automation=automation, + trigger=trigger, + runtime_inputs=runtime_inputs, + ) + + +async def _load_automation( + session: AsyncSession, automation_id: int +) -> Automation | None: + stmt = select(Automation).where(Automation.id == automation_id) + return (await session.execute(stmt)).scalar_one_or_none() diff --git a/surfsense_backend/app/automations/triggers/schedule/params.py b/surfsense_backend/app/automations/triggers/schedule/params.py new file mode 100644 index 000000000..21da84f68 --- /dev/null +++ b/surfsense_backend/app/automations/triggers/schedule/params.py @@ -0,0 +1,22 @@ +"""``ScheduleTriggerParams`` — params for the ``schedule`` trigger type.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from .cron import InvalidCronError, validate_cron + + +class ScheduleTriggerParams(BaseModel): + model_config = ConfigDict(extra="forbid") + + cron: str = Field(..., description="Five-field cron expression.", examples=["0 9 * * 1-5"]) + timezone: str = Field(..., description="IANA timezone.", examples=["Africa/Kigali"]) + + @model_validator(mode="after") + def _validate(self) -> ScheduleTriggerParams: + try: + validate_cron(self.cron, self.timezone) + except InvalidCronError as exc: + raise ValueError(str(exc)) from exc + return self diff --git a/surfsense_backend/app/automations/triggers/store.py b/surfsense_backend/app/automations/triggers/store.py new file mode 100644 index 000000000..af0fafac7 --- /dev/null +++ b/surfsense_backend/app/automations/triggers/store.py @@ -0,0 +1,23 @@ +"""In-memory trigger registry. Populated once at process startup.""" + +from __future__ import annotations + +from .types import TriggerDefinition + +_REGISTRY: dict[str, TriggerDefinition] = {} + + +def register_trigger(trigger: TriggerDefinition) -> None: + """Register a trigger. Raises on duplicate type.""" + if trigger.type in _REGISTRY: + raise ValueError(f"Trigger already registered: {trigger.type!r}") + _REGISTRY[trigger.type] = trigger + + +def get_trigger(trigger_type: str) -> TriggerDefinition | None: + return _REGISTRY.get(trigger_type) + + +def all_triggers() -> dict[str, TriggerDefinition]: + """Defensive snapshot of the registry.""" + return dict(_REGISTRY) diff --git a/surfsense_backend/app/automations/triggers/types.py b/surfsense_backend/app/automations/triggers/types.py new file mode 100644 index 000000000..aa2808e4d --- /dev/null +++ b/surfsense_backend/app/automations/triggers/types.py @@ -0,0 +1,20 @@ +"""``TriggerDefinition`` dataclass. Declarative; firing is the dispatcher's job.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from pydantic import BaseModel + + +@dataclass(frozen=True, slots=True) +class TriggerDefinition: + type: str + description: str + params_model: type[BaseModel] + + @property + def params_schema(self) -> dict[str, Any]: + """JSON Schema (draft 2020-12) derived from ``params_model``.""" + return self.params_model.model_json_schema() diff --git a/surfsense_backend/app/celery_app.py b/surfsense_backend/app/celery_app.py index 5b45baca1..9169592fd 100644 --- a/surfsense_backend/app/celery_app.py +++ b/surfsense_backend/app/celery_app.py @@ -188,6 +188,8 @@ celery_app = Celery( "app.tasks.celery_tasks.document_reindex_tasks", "app.tasks.celery_tasks.stale_notification_cleanup_task", "app.tasks.celery_tasks.stripe_reconciliation_task", + "app.automations.tasks.execute_run", + "app.automations.tasks.schedule_tick", ], ) @@ -282,4 +284,14 @@ celery_app.conf.beat_schedule = { "expires": 60, }, }, + # Fire due automation schedule triggers. Ticks every minute; per-row cron + # math is precomputed (next_fire_at column) so the tick is an indexed + # lookup, not N cron evaluations. + "automation-schedule-tick": { + "task": "automation_schedule_tick", + "schedule": crontab(minute="*"), + "options": { + "expires": 50, + }, + }, } diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 9fc27fb1f..ac880ded5 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -439,6 +439,13 @@ class Permission(StrEnum): PUBLIC_SHARING_CREATE = "public_sharing:create" PUBLIC_SHARING_DELETE = "public_sharing:delete" + # Automations + AUTOMATIONS_CREATE = "automations:create" + AUTOMATIONS_READ = "automations:read" + AUTOMATIONS_UPDATE = "automations:update" + AUTOMATIONS_DELETE = "automations:delete" + AUTOMATIONS_EXECUTE = "automations:execute" + # Full access wildcard FULL_ACCESS = "*" @@ -494,6 +501,11 @@ DEFAULT_ROLE_PERMISSIONS = { # Public Sharing (can create and view, no delete) Permission.PUBLIC_SHARING_VIEW.value, Permission.PUBLIC_SHARING_CREATE.value, + # Automations (no delete) + Permission.AUTOMATIONS_CREATE.value, + Permission.AUTOMATIONS_READ.value, + Permission.AUTOMATIONS_UPDATE.value, + Permission.AUTOMATIONS_EXECUTE.value, ], "Viewer": [ # Documents (read only) @@ -525,6 +537,8 @@ DEFAULT_ROLE_PERMISSIONS = { Permission.SETTINGS_VIEW.value, # Public Sharing (view only) Permission.PUBLIC_SHARING_VIEW.value, + # Automations (read only) + Permission.AUTOMATIONS_READ.value, ], } @@ -1533,6 +1547,14 @@ class SearchSpace(BaseModel, TimestampMixin): cascade="all, delete-orphan", ) + automations = relationship( + "Automation", + back_populates="search_space", + order_by="Automation.id", + cascade="all, delete-orphan", + passive_deletes=True, + ) + # RBAC relationships roles = relationship( "SearchSpaceRole", @@ -2125,6 +2147,13 @@ if config.AUTH_TYPE == "GOOGLE": passive_deletes=True, ) + # Automations created by this user + automations = relationship( + "Automation", + back_populates="created_by", + passive_deletes=True, + ) + # Incentive tasks completed by this user incentive_tasks = relationship( "UserIncentiveTask", @@ -2257,6 +2286,13 @@ else: passive_deletes=True, ) + # Automations created by this user + automations = relationship( + "Automation", + back_populates="created_by", + passive_deletes=True, + ) + # Incentive tasks completed by this user incentive_tasks = relationship( "UserIncentiveTask", @@ -2560,6 +2596,16 @@ class RefreshToken(Base, TimestampMixin): return not self.is_expired and not self.is_revoked +# Register model packages that live outside this file so their classes +# are present in Base.metadata before configure_mappers() resolves any +# string-based relationship() references. +from app.automations.persistence import ( # noqa: E402, F401 + Automation, + AutomationRun, + AutomationTrigger, +) + + engine = create_async_engine( DATABASE_URL, pool_size=30, diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index ec4d1650f..ef1c9312a 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -7,6 +7,7 @@ from .agent_revert_route import router as agent_revert_router from .airtable_add_connector_route import ( router as airtable_add_connector_router, ) +from app.automations.api import router as automations_router from .chat_comments_routes import router as chat_comments_router from .circleback_webhook_route import router as circleback_webhook_router from .clickup_add_connector_route import router as clickup_add_connector_router @@ -119,3 +120,4 @@ router.include_router(youtube_router) # YouTube playlist resolution router.include_router(prompts_router) router.include_router(memory_router) # User personal memory (memory.md style) router.include_router(team_memory_router) # Search-space team memory +router.include_router(automations_router) # Automations CRUD + run history diff --git a/surfsense_backend/app/routes/rbac_routes.py b/surfsense_backend/app/routes/rbac_routes.py index 38ae31269..3b91e456d 100644 --- a/surfsense_backend/app/routes/rbac_routes.py +++ b/surfsense_backend/app/routes/rbac_routes.py @@ -107,6 +107,12 @@ PERMISSION_DESCRIPTIONS = { "settings:view": "View search space settings", "settings:update": "Modify search space settings", "settings:delete": "Delete the entire search space", + # Automations + "automations:create": "Create automations from chat or JSON", + "automations:read": "View automations, their triggers, and run history", + "automations:update": "Edit automations and manage their triggers", + "automations:delete": "Remove automations from the search space", + "automations:execute": "Manually fire automations", # Full access "*": "Full access to all features and settings", } diff --git a/surfsense_backend/app/tasks/chat/streaming/agent/__init__.py b/surfsense_backend/app/tasks/chat/streaming/agent/__init__.py new file mode 100644 index 000000000..260dcb3f2 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/agent/__init__.py @@ -0,0 +1,8 @@ +"""Agent construction and per-turn event-loop drivers.""" + +from __future__ import annotations + +from app.tasks.chat.streaming.agent.builder import build_main_agent_for_thread +from app.tasks.chat.streaming.agent.event_loop import stream_agent_events + +__all__ = ["build_main_agent_for_thread", "stream_agent_events"] diff --git a/surfsense_backend/app/tasks/chat/streaming/agent/builder.py b/surfsense_backend/app/tasks/chat/streaming/agent/builder.py new file mode 100644 index 000000000..0db42edbf --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/agent/builder.py @@ -0,0 +1,49 @@ +"""Single per-thread agent (re)build path. + +A graph swap mid-turn would corrupt checkpointer state for the same +``thread_id``, so both the initial build and any mid-stream 429 recovery rebuild +must funnel through this single function. +""" + +from __future__ import annotations + +from typing import Any + +from app.agents.new_chat.filesystem_selection import FilesystemSelection +from app.agents.new_chat.llm_config import AgentConfig +from app.db import ChatVisibility +from app.services.connector_service import ConnectorService + + +async def build_main_agent_for_thread( + agent_factory: Any, + *, + llm: Any, + search_space_id: int, + db_session: Any, + connector_service: ConnectorService, + checkpointer: Any, + user_id: str | None, + thread_id: int | None, + agent_config: AgentConfig | None, + firecrawl_api_key: str | None, + thread_visibility: ChatVisibility | None, + filesystem_selection: FilesystemSelection | None, + disabled_tools: list[str] | None = None, + mentioned_document_ids: list[int] | None = None, +) -> Any: + return await agent_factory( + llm=llm, + search_space_id=search_space_id, + db_session=db_session, + connector_service=connector_service, + checkpointer=checkpointer, + user_id=user_id, + thread_id=thread_id, + agent_config=agent_config, + firecrawl_api_key=firecrawl_api_key, + thread_visibility=thread_visibility, + filesystem_selection=filesystem_selection, + disabled_tools=disabled_tools, + mentioned_document_ids=mentioned_document_ids, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/agent/event_loop.py b/surfsense_backend/app/tasks/chat/streaming/agent/event_loop.py new file mode 100644 index 000000000..b77bd3890 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/agent/event_loop.py @@ -0,0 +1,175 @@ +"""Per-turn agent event-loop driver. + +Drives ``stream_output`` (graph_stream relay) for one agent turn, then runs the +post-stream agent-state inspection: safety-net commit of any staged filesystem +state (in case ``aafter_agent`` was skipped), file-operation contract scoring, +intent classification, and interrupt detection. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator +from typing import Any + +from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.new_chat.middleware.kb_persistence import ( + commit_staged_filesystem_state, +) +from app.services.new_streaming_service import VercelStreamingService +from app.tasks.chat.streaming.contract.file_contract import ( + contract_enforcement_active, + evaluate_file_contract_outcome, + log_file_contract, +) +from app.tasks.chat.streaming.graph_stream.event_stream import stream_output +from app.tasks.chat.streaming.helpers.interrupt_inspector import ( + all_interrupt_values, +) +from app.tasks.chat.streaming.shared.stream_result import StreamResult +from app.tasks.chat.streaming.shared.utils import safe_float +from app.utils.perf import get_perf_logger + +_perf_log = get_perf_logger() + + +async def stream_agent_events( + agent: Any, + config: dict[str, Any], + input_data: Any, + streaming_service: VercelStreamingService, + result: StreamResult, + step_prefix: str = "thinking", + initial_step_id: str | None = None, + initial_step_title: str = "", + initial_step_items: list[str] | None = None, + *, + fallback_commit_search_space_id: int | None = None, + fallback_commit_created_by_id: str | None = None, + fallback_commit_filesystem_mode: FilesystemMode = FilesystemMode.CLOUD, + fallback_commit_thread_id: int | None = None, + runtime_context: Any = None, + content_builder: Any | None = None, +) -> AsyncGenerator[str, None]: + """Stream and format ``astream_events`` from the agent. + + Yields SSE-formatted strings; after exhausting, ``result`` carries + ``accumulated_text`` and interrupt state. See ``StreamResult`` for the + side-channel surface populated by the underlying relay. + """ + async for sse in stream_output( + agent=agent, + config=config, + input_data=input_data, + streaming_service=streaming_service, + result=result, + step_prefix=step_prefix, + initial_step_id=initial_step_id, + initial_step_title=initial_step_title, + initial_step_items=initial_step_items, + content_builder=content_builder, + runtime_context=runtime_context, + ): + yield sse + + accumulated_text = result.accumulated_text + + state = await agent.aget_state(config) + state_values = getattr(state, "values", {}) or {} + + # Safety net: if astream_events was cancelled before + # KnowledgeBasePersistenceMiddleware.aafter_agent ran, any staged work + # (dirty_paths / staged_dirs / pending_moves / pending_deletes / + # pending_dir_deletes) is still in the checkpointed state. Run the SAME + # shared commit helper so the turn's writes don't get lost on client + # disconnect, then push the delta back into the graph using ``as_node=...`` + # so reducers fire as if the after_agent hook produced it. + if ( + fallback_commit_filesystem_mode == FilesystemMode.CLOUD + and fallback_commit_search_space_id is not None + and ( + (state_values.get("dirty_paths") or []) + or (state_values.get("staged_dirs") or []) + or (state_values.get("pending_moves") or []) + or (state_values.get("pending_deletes") or []) + or (state_values.get("pending_dir_deletes") or []) + ) + ): + try: + delta = await commit_staged_filesystem_state( + state_values, + search_space_id=fallback_commit_search_space_id, + created_by_id=fallback_commit_created_by_id, + filesystem_mode=fallback_commit_filesystem_mode, + thread_id=fallback_commit_thread_id, + dispatch_events=False, + ) + if delta: + await agent.aupdate_state( + config, + delta, + as_node="KnowledgeBasePersistenceMiddleware.after_agent", + ) + except Exception as exc: + _perf_log.warning("[stream_agent_events] safety-net commit failed: %s", exc) + + contract_state = state_values.get("file_operation_contract") or {} + contract_turn_id = contract_state.get("turn_id") + current_turn_id = config.get("configurable", {}).get("turn_id", "") + intent_value = contract_state.get("intent") + if ( + isinstance(intent_value, str) + and intent_value in ("chat_only", "file_write", "file_read") + and contract_turn_id == current_turn_id + ): + result.intent_detected = intent_value + if ( + isinstance(intent_value, str) + and intent_value in ("chat_only", "file_write", "file_read") + and contract_turn_id != current_turn_id + ): + # Ignore stale intent contracts from previous turns/checkpoints. + result.intent_detected = "chat_only" + result.intent_confidence = ( + safe_float(contract_state.get("confidence"), default=0.0) + if contract_turn_id == current_turn_id + else 0.0 + ) + + if result.intent_detected == "file_write": + result.commit_gate_passed, result.commit_gate_reason = ( + evaluate_file_contract_outcome(result) + ) + if not result.commit_gate_passed and contract_enforcement_active(result): + gate_notice = ( + "I could not complete the requested file write because no successful " + "write_file/edit_file operation was confirmed." + ) + gate_text_id = streaming_service.generate_text_id() + yield streaming_service.format_text_start(gate_text_id) + if content_builder is not None: + content_builder.on_text_start(gate_text_id) + yield streaming_service.format_text_delta(gate_text_id, gate_notice) + if content_builder is not None: + content_builder.on_text_delta(gate_text_id, gate_notice) + yield streaming_service.format_text_end(gate_text_id) + if content_builder is not None: + content_builder.on_text_end(gate_text_id) + yield streaming_service.format_terminal_info(gate_notice, "error") + accumulated_text = gate_notice + else: + result.commit_gate_passed = True + result.commit_gate_reason = "" + + result.accumulated_text = accumulated_text + log_file_contract("turn_outcome", result) + + pending_values = all_interrupt_values(state) + if pending_values: + result.is_interrupted = True + # One frame per paused subagent so each parallel HITL renders its own + # approval card on the wire. Order matches ``state.interrupts``, which + # the resume slicer in + # ``checkpointed_subagent_middleware.resume_routing`` consumes in the + # same order — keeping emit and resume in lock-step. + for interrupt_value in pending_values: + yield streaming_service.format_interrupt_request(interrupt_value) diff --git a/surfsense_backend/app/tasks/chat/streaming/context/__init__.py b/surfsense_backend/app/tasks/chat/streaming/context/__init__.py new file mode 100644 index 000000000..f858a6c06 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/context/__init__.py @@ -0,0 +1,15 @@ +"""Pre-agent context shaping: mentioned-doc rendering and todos extraction.""" + +from __future__ import annotations + +from app.tasks.chat.streaming.context.deepagents_todos import ( + extract_todos_from_deepagents, +) +from app.tasks.chat.streaming.context.mentioned_docs import ( + format_mentioned_surfsense_docs_as_context, +) + +__all__ = [ + "extract_todos_from_deepagents", + "format_mentioned_surfsense_docs_as_context", +] diff --git a/surfsense_backend/app/tasks/chat/streaming/context/deepagents_todos.py b/surfsense_backend/app/tasks/chat/streaming/context/deepagents_todos.py new file mode 100644 index 000000000..0bbf4f0a5 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/context/deepagents_todos.py @@ -0,0 +1,27 @@ +"""Extract todos from a deepagents ``TodoListMiddleware`` ``Command`` output.""" + +from __future__ import annotations + +from typing import Any + + +def extract_todos_from_deepagents(command_output: Any) -> dict: + """Normalize todos out of a deepagents ``Command`` or dict payload. + + deepagents returns a ``Command`` whose ``update['todos']`` is a list of + ``{'content': str, 'status': str}`` dicts. The UI expects the same shape, + so no transformation is required — only extraction. + """ + todos_data: list = [] + if hasattr(command_output, "update"): + update = command_output.update + todos_data = update.get("todos", []) + elif isinstance(command_output, dict): + if "todos" in command_output: + todos_data = command_output.get("todos", []) + elif "update" in command_output and isinstance( + command_output["update"], dict + ): + todos_data = command_output["update"].get("todos", []) + + return {"todos": todos_data} diff --git a/surfsense_backend/app/tasks/chat/streaming/context/mentioned_docs.py b/surfsense_backend/app/tasks/chat/streaming/context/mentioned_docs.py new file mode 100644 index 000000000..e02e98d34 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/context/mentioned_docs.py @@ -0,0 +1,58 @@ +"""Render user-mentioned SurfSense docs as XML context for the agent.""" + +from __future__ import annotations + +import json + +from app.db import SurfsenseDocsDocument +from app.utils.surfsense_docs import surfsense_docs_public_url + + +def format_mentioned_surfsense_docs_as_context( + documents: list[SurfsenseDocsDocument], +) -> str: + if not documents: + return "" + + context_parts = [""] + context_parts.append( + "The user has explicitly mentioned the following SurfSense documentation pages. " + "These are official documentation about how to use SurfSense and should be used to answer questions about the application. " + "Use [citation:CHUNK_ID] format for citations (e.g., [citation:doc-123])." + ) + + for doc in documents: + public_url = surfsense_docs_public_url(doc.source) + metadata_json = json.dumps( + {"source": doc.source, "public_url": public_url}, ensure_ascii=False + ) + + context_parts.append("") + context_parts.append("") + context_parts.append(f" doc-{doc.id}") + context_parts.append(" SURFSENSE_DOCS") + context_parts.append(f" <![CDATA[{doc.title}]]>") + context_parts.append(f" ") + context_parts.append( + f" " + ) + context_parts.append("") + context_parts.append("") + context_parts.append("") + + if hasattr(doc, "chunks") and doc.chunks: + for chunk in doc.chunks: + context_parts.append( + f" " + ) + else: + context_parts.append( + f" " + ) + + context_parts.append("") + context_parts.append("") + context_parts.append("") + + context_parts.append("") + return "\n".join(context_parts) diff --git a/surfsense_backend/app/tasks/chat/streaming/contract/__init__.py b/surfsense_backend/app/tasks/chat/streaming/contract/__init__.py new file mode 100644 index 000000000..4562b362c --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/contract/__init__.py @@ -0,0 +1,15 @@ +"""File-operation contract evaluation and logging.""" + +from __future__ import annotations + +from app.tasks.chat.streaming.contract.file_contract import ( + contract_enforcement_active, + evaluate_file_contract_outcome, + log_file_contract, +) + +__all__ = [ + "contract_enforcement_active", + "evaluate_file_contract_outcome", + "log_file_contract", +] diff --git a/surfsense_backend/app/tasks/chat/streaming/contract/file_contract.py b/surfsense_backend/app/tasks/chat/streaming/contract/file_contract.py new file mode 100644 index 000000000..f21f5da02 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/contract/file_contract.py @@ -0,0 +1,53 @@ +"""File-operation contract: when to enforce, how to score, how to log.""" + +from __future__ import annotations + +import json +from typing import Any + +from app.tasks.chat.streaming.shared.stream_result import StreamResult +from app.utils.perf import get_perf_logger + +_perf_log = get_perf_logger() + + +def contract_enforcement_active(result: StreamResult) -> bool: + # Enforce only in desktop local-folder mode. Kept deterministic, no + # env-driven progression modes. + return result.filesystem_mode == "desktop_local_folder" + + +def evaluate_file_contract_outcome(result: StreamResult) -> tuple[bool, str]: + if result.intent_detected != "file_write": + return True, "" + if not result.write_attempted: + return False, "no_write_attempt" + if not result.write_succeeded: + return False, "write_failed" + if not result.verification_succeeded: + return False, "verification_failed" + return True, "" + + +def log_file_contract(stage: str, result: StreamResult, **extra: Any) -> None: + payload: dict[str, Any] = { + "stage": stage, + "request_id": result.request_id or "unknown", + "turn_id": result.turn_id or "unknown", + "chat_id": ( + result.turn_id.split(":", 1)[0] if ":" in result.turn_id else "unknown" + ), + "filesystem_mode": result.filesystem_mode, + "client_platform": result.client_platform, + "intent_detected": result.intent_detected, + "intent_confidence": result.intent_confidence, + "write_attempted": result.write_attempted, + "write_succeeded": result.write_succeeded, + "verification_succeeded": result.verification_succeeded, + "commit_gate_passed": result.commit_gate_passed, + "commit_gate_reason": result.commit_gate_reason or None, + } + payload.update(extra) + _perf_log.info( + "[file_operation_contract] %s", json.dumps(payload, ensure_ascii=False) + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/__init__.py b/surfsense_backend/app/tasks/chat/streaming/flows/__init__.py new file mode 100644 index 000000000..522db2fad --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/__init__.py @@ -0,0 +1,17 @@ +"""Top-level streaming flows: ``new_chat`` and ``resume_chat`` orchestrators. + +Re-exports the public entry points so callers can write:: + + from app.tasks.chat.streaming.flows import stream_new_chat, stream_resume_chat + +The orchestrators themselves live under ``new_chat/orchestrator.py`` and +``resume_chat/orchestrator.py`` (slim composition of the per-concern modules in +each flow folder and the building blocks in ``shared/``). +""" + +from __future__ import annotations + +from app.tasks.chat.streaming.flows.new_chat import stream_new_chat +from app.tasks.chat.streaming.flows.resume_chat import stream_resume_chat + +__all__ = ["stream_new_chat", "stream_resume_chat"] diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/__init__.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/__init__.py new file mode 100644 index 000000000..566d5e0d9 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/__init__.py @@ -0,0 +1,12 @@ +"""New-chat streaming flow. + +The public entry point ``stream_new_chat`` is the slim coroutine in +``orchestrator.py`` that composes the per-concern modules in this folder and +the building blocks under ``flows/shared/``. +""" + +from __future__ import annotations + +from app.tasks.chat.streaming.flows.new_chat.orchestrator import stream_new_chat + +__all__ = ["stream_new_chat"] diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/auto_pin.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/auto_pin.py new file mode 100644 index 000000000..cb20eb011 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/auto_pin.py @@ -0,0 +1,95 @@ +"""Resolve the auto-pin for the *initial* turn config. + +Auto-pin (``selected_llm_config_id=0``) picks the best eligible LLM config for +this thread / search space / user, optionally filtered to vision-capable +configs when the turn carries images. + +Errors classified here: + + * ``MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT`` — the auto-pin pool has no + vision-capable cfg for an image-bearing turn. The same gate fires later + in ``llm_capability`` for explicit selections; mapping both to the same + code keeps the FE error UI consistent. + * ``SERVER_ERROR`` — any other ``ValueError`` from the resolver. + +This module owns *initial* pin resolution; the rate-limit recovery loop has +its own narrower auto-pin call (with ``exclude_config_ids``) in +``flows/shared/rate_limit_recovery``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.observability import otel as ot +from app.services.auto_model_pin_service import resolve_or_get_pinned_llm_config_id + + +@dataclass +class AutoPinResult: + """Outcome of ``resolve_initial_auto_pin``. + + ``llm_config_id`` is set when ``error`` is ``None``; ``error`` carries the + classified user-facing message plus error code/kind so the orchestrator can + emit one terminal-error SSE frame. + """ + + llm_config_id: int | None + error: tuple[str, str, Literal["user_error", "server_error"]] | None + + +async def resolve_initial_auto_pin( + session: AsyncSession, + *, + chat_id: int, + search_space_id: int, + user_id: str | None, + selected_llm_config_id: int, + requires_image_input: bool, + requested_llm_config_id: int, +) -> AutoPinResult: + """Run the resolver and classify any ``ValueError`` for the SSE error path.""" + try: + pinned = await resolve_or_get_pinned_llm_config_id( + session, + thread_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + selected_llm_config_id=selected_llm_config_id, + requires_image_input=requires_image_input, + ) + ot.add_event( + "model.pin.resolved", + { + "pin.requested_id": requested_llm_config_id, + "pin.resolved_id": pinned.resolved_llm_config_id, + "pin.requires_image_input": requires_image_input, + }, + ) + return AutoPinResult( + llm_config_id=pinned.resolved_llm_config_id, error=None + ) + except ValueError as pin_error: + # The "no vision-capable cfg" path raises a ValueError whose message + # we map to the friendly image-input SSE error so the user sees the + # same message regardless of whether the gate fired in the resolver or + # in ``llm_capability.assert_vision_capability_for_image_turn``. + is_vision_failure = ( + requires_image_input and "vision-capable" in str(pin_error) + ) + error_code = ( + "MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT" + if is_vision_failure + else "SERVER_ERROR" + ) + error_kind: Literal["user_error", "server_error"] = ( + "user_error" if is_vision_failure else "server_error" + ) + if is_vision_failure: + ot.add_event("quota.denied", {"quota.code": error_code}) + return AutoPinResult( + llm_config_id=None, error=(str(pin_error), error_code, error_kind) + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/initial_thinking_step.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/initial_thinking_step.py new file mode 100644 index 000000000..c860e517e --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/initial_thinking_step.py @@ -0,0 +1,95 @@ +"""Build and emit the first ``thinking-1`` step for a new-chat turn. + +The step title and "Processing X" items are derived from what the user sent +(text snippet, image count, mentioned doc titles) so the FE can render a +meaningful placeholder while the agent stream warms up. + +``thinking-1`` is the canonical id for this step — every subsequent +``thinking-N`` produced by ``stream_agent_events`` folds into the same +singleton ``data-thinking-steps`` part on the FE. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Any + +from app.db import SurfsenseDocsDocument +from app.services.new_streaming_service import VercelStreamingService + + +@dataclass +class InitialThinkingStep: + """Resolved fields passed both into the SSE frame and the builder hook. + + ``items`` is the bullet list under the step title; ``title`` is the + one-line step header. ``step_id`` is hard-coded ``thinking-1`` so the FE + Timeline can de-duplicate against the prior assistant message on resume. + """ + + step_id: str + title: str + items: list[str] + + +def build_initial_thinking_step( + *, + user_query: str, + user_image_data_urls: list[str] | None, + mentioned_surfsense_docs: list[SurfsenseDocsDocument], +) -> InitialThinkingStep: + if mentioned_surfsense_docs: + title = "Analyzing referenced content" + action_verb = "Analyzing" + else: + title = "Understanding your request" + action_verb = "Processing" + + processing_parts: list[str] = [] + if user_query.strip(): + query_text = user_query[:80] + ("..." if len(user_query) > 80 else "") + processing_parts.append(query_text) + elif user_image_data_urls: + processing_parts.append(f"[{len(user_image_data_urls)} image(s)]") + else: + processing_parts.append("(message)") + + if mentioned_surfsense_docs: + doc_names: list[str] = [] + for doc in mentioned_surfsense_docs: + t = doc.title + if len(t) > 30: + t = t[:27] + "..." + doc_names.append(t) + if len(doc_names) == 1: + processing_parts.append(f"[{doc_names[0]}]") + else: + processing_parts.append(f"[{len(doc_names)} docs]") + + items = [f"{action_verb}: {' '.join(processing_parts)}"] + return InitialThinkingStep(step_id="thinking-1", title=title, items=items) + + +def iter_initial_thinking_step_frame( + step: InitialThinkingStep, + *, + streaming_service: VercelStreamingService, + content_builder: Any | None, +) -> Iterator[str]: + """Drive both the SSE emission and the builder hook for the initial step. + + The FE folds this step into the same singleton ``data-thinking-steps`` part + as everything the agent stream emits later, so we mirror that fold + server-side by driving the builder lifecycle ourselves. + """ + if content_builder is not None: + content_builder.on_thinking_step( + step.step_id, step.title, "in_progress", step.items + ) + yield streaming_service.format_thinking_step( + step_id=step.step_id, + title=step.title, + status="in_progress", + items=step.items, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py new file mode 100644 index 000000000..fb171c244 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py @@ -0,0 +1,264 @@ +r"""Assemble the LangGraph ``input_state`` for the new-chat turn. + +Pipeline: + + 1. **History bootstrap** — only for cloned chats with no LangGraph checkpoint + yet; flips the per-thread ``needs_history_bootstrap`` flag back to False + once the rows are loaded. + 2. **Mentioned SurfSense docs** — eager-load chunks so the formatter has the + full content without a second roundtrip. + 3. **Recent reports** — top 3 by id desc with non-null content, so the LLM + can resolve ``report_id`` for versioning without spelunking history. + 4. **@-mention resolve** (cloud mode) — substitute ``@title`` tokens in the + query with canonical ``\`/documents/...\``` paths the LLM expects. + 5. **Context block render** — XML-wrap surfsense docs + reports, prepend to + the rewritten query, optionally prefix with display name for SEARCH_SPACE + visibility. + 6. **HumanMessage** — multimodal content if images are attached. + +Returns the assembled ``input_state`` dict plus side-channel data the +orchestrator needs downstream (``accepted_folder_ids`` for runtime context; +``mentioned_surfsense_docs`` for the initial thinking step). +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any + +from langchain_core.messages import HumanMessage +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select +from sqlalchemy.orm import selectinload + +from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.new_chat.mention_resolver import resolve_mentions, substitute_in_text +from app.db import ( + ChatVisibility, + NewChatThread, + Report, + SurfsenseDocsDocument, +) +from app.tasks.chat.streaming.context.mentioned_docs import ( + format_mentioned_surfsense_docs_as_context, +) +from app.utils.content_utils import bootstrap_history_from_db +from app.utils.user_message_multimodal import build_human_message_content + +logger = logging.getLogger(__name__) + + +@dataclass +class NewChatInputState: + """Everything ``build_new_chat_input_state`` produces. + + ``input_state`` is fed straight to the agent. ``accepted_folder_ids`` + feeds the runtime context (the resolver may have dropped some chips). + ``mentioned_surfsense_docs`` is consumed by the initial thinking-step + builder for the FE placeholder before the agent stream starts. + """ + + input_state: dict[str, Any] + accepted_folder_ids: list[int] + mentioned_surfsense_docs: list[SurfsenseDocsDocument] + + +async def build_new_chat_input_state( + session: AsyncSession, + *, + chat_id: int, + search_space_id: int, + user_query: str, + user_image_data_urls: list[str] | None, + mentioned_document_ids: list[int] | None, + mentioned_surfsense_doc_ids: list[int] | None, + mentioned_folder_ids: list[int] | None, + mentioned_documents: list[dict[str, Any]] | None, + needs_history_bootstrap: bool, + thread_visibility: ChatVisibility, + current_user_display_name: str | None, + filesystem_mode: str, + request_id: str | None, + turn_id: str, +) -> NewChatInputState: + langchain_messages: list[Any] = [] + + if needs_history_bootstrap: + langchain_messages = await bootstrap_history_from_db( + session, chat_id, thread_visibility=thread_visibility + ) + thread_result = await session.execute( + select(NewChatThread).filter(NewChatThread.id == chat_id) + ) + thread = thread_result.scalars().first() + if thread: + thread.needs_history_bootstrap = False + await session.commit() + + mentioned_surfsense_docs: list[SurfsenseDocsDocument] = [] + if mentioned_surfsense_doc_ids: + result = await session.execute( + select(SurfsenseDocsDocument) + .options(selectinload(SurfsenseDocsDocument.chunks)) + .filter(SurfsenseDocsDocument.id.in_(mentioned_surfsense_doc_ids)) + ) + mentioned_surfsense_docs = list(result.scalars().all()) + + # Top 3 reports keyed by id desc (newest first) with content present, + # surfaced inline so the LLM resolves ``report_id`` for versioning without + # digging through conversation history. + recent_reports_result = await session.execute( + select(Report) + .filter( + Report.thread_id == chat_id, + Report.content.isnot(None), + ) + .order_by(Report.id.desc()) + .limit(3) + ) + recent_reports = list(recent_reports_result.scalars().all()) + + agent_user_query, accepted_folder_ids = await _resolve_mentions_for_query( + session, + search_space_id=search_space_id, + user_query=user_query, + filesystem_mode=filesystem_mode, + mentioned_document_ids=mentioned_document_ids, + mentioned_surfsense_doc_ids=mentioned_surfsense_doc_ids, + mentioned_folder_ids=mentioned_folder_ids, + mentioned_documents=mentioned_documents, + ) + + final_query = _render_query_with_context( + agent_user_query=agent_user_query, + mentioned_surfsense_docs=mentioned_surfsense_docs, + recent_reports=recent_reports, + ) + + if thread_visibility == ChatVisibility.SEARCH_SPACE and current_user_display_name: + final_query = f"**[{current_user_display_name}]:** {final_query}" + + human_content = build_human_message_content( + final_query, list(user_image_data_urls or ()) + ) + langchain_messages.append(HumanMessage(content=human_content)) + + input_state = { + "messages": langchain_messages, + "search_space_id": search_space_id, + "request_id": request_id or "unknown", + "turn_id": turn_id, + } + + return NewChatInputState( + input_state=input_state, + accepted_folder_ids=accepted_folder_ids, + mentioned_surfsense_docs=mentioned_surfsense_docs, + ) + + +async def _resolve_mentions_for_query( + session: AsyncSession, + *, + search_space_id: int, + user_query: str, + filesystem_mode: str, + mentioned_document_ids: list[int] | None, + mentioned_surfsense_doc_ids: list[int] | None, + mentioned_folder_ids: list[int] | None, + mentioned_documents: list[dict[str, Any]] | None, +) -> tuple[str, list[int]]: + r"""Resolve @-mention chips and rewrite the user query to canonical paths. + + Cloud mode only: local-folder mode keeps the legacy ``@title`` text path + (mention support there is a follow-up task — the path scheme is + mount-rooted and the picker UI both need separate work). + + The substitution lands in the returned ``agent_user_query`` ONLY — the + original ``user_query`` (with ``@title`` tokens) flows untouched into + ``persist_user_turn`` so chip rendering on reload still works + (``UserTextPart`` → ``parseMentionSegments`` matches ``@title``, not + ``\`/documents/...\```). It also feeds the human-readable surfaces — SSE + "Processing X" status, auto thread title, memory seed — which all want + what the user typed. + """ + agent_user_query = user_query + accepted_folder_ids: list[int] = [] + + has_any_mention = bool( + mentioned_document_ids + or mentioned_surfsense_doc_ids + or mentioned_folder_ids + or mentioned_documents + ) + if filesystem_mode != FilesystemMode.CLOUD.value or not has_any_mention: + return agent_user_query, accepted_folder_ids + + from app.schemas.new_chat import MentionedDocumentInfo + + chip_objs: list[MentionedDocumentInfo] | None = None + if mentioned_documents: + chip_objs = [] + for raw in mentioned_documents: + if isinstance(raw, MentionedDocumentInfo): + chip_objs.append(raw) + continue + try: + chip_objs.append(MentionedDocumentInfo.model_validate(raw)) + except Exception: + logger.debug( + "stream_new_chat: dropping malformed mention chip %r", raw + ) + + resolved = await resolve_mentions( + session, + search_space_id=search_space_id, + mentioned_documents=chip_objs, + mentioned_document_ids=mentioned_document_ids, + mentioned_surfsense_doc_ids=mentioned_surfsense_doc_ids, + mentioned_folder_ids=mentioned_folder_ids, + ) + agent_user_query = substitute_in_text(user_query, resolved.token_to_path) + accepted_folder_ids = resolved.mentioned_folder_ids + return agent_user_query, accepted_folder_ids + + +def _render_query_with_context( + *, + agent_user_query: str, + mentioned_surfsense_docs: list[SurfsenseDocsDocument], + recent_reports: list[Report], +) -> str: + """Prepend surfsense-docs + recent-reports XML blocks to the user query.""" + context_parts: list[str] = [] + + if mentioned_surfsense_docs: + context_parts.append( + format_mentioned_surfsense_docs_as_context(mentioned_surfsense_docs) + ) + + if recent_reports: + report_lines: list[str] = [] + for r in recent_reports: + report_lines.append( + f' - report_id={r.id}, title="{r.title}", ' + f'style="{r.report_style or "detailed"}"' + ) + reports_listing = "\n".join(report_lines) + context_parts.append( + "\n" + "Previously generated reports in this conversation:\n" + f"{reports_listing}\n\n" + "If the user wants to MODIFY, REVISE, UPDATE, or ADD to one of " + "these reports, set parent_report_id to the relevant report_id above.\n" + "If the user wants a completely NEW report on a different topic, " + "leave parent_report_id unset.\n" + "" + ) + + if context_parts: + context = "\n\n".join(context_parts) + return f"{context}\n\n{agent_user_query}" + + return agent_user_query diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/llm_capability.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/llm_capability.py new file mode 100644 index 000000000..ff5a56eec --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/llm_capability.py @@ -0,0 +1,62 @@ +"""Vision-capability gate for image-bearing turns. + +Capability safety net for explicit (non-auto-pin) selections: a turn carrying +user-uploaded images cannot be routed to a chat config that LiteLLM's +authoritative model map *explicitly* marks as text-only (``supports_vision`` +set to False). The check is intentionally narrow — it only fires when LiteLLM +is *certain* the model can't accept image input; unknown / unmapped / +vision-capable models pass through. + +Without this guard a known-text-only model would 404 at the provider with +``"No endpoints found that support image input"``, surfacing as an opaque +``SERVER_ERROR`` SSE chunk; failing here lets us return a friendly message that +tells the user what to change. +""" + +from __future__ import annotations + +from app.agents.new_chat.llm_config import AgentConfig +from app.observability import otel as ot + + +def check_image_input_capability( + *, + user_image_data_urls: list[str] | None, + agent_config: AgentConfig | None, +) -> tuple[str, str] | None: + """Return ``(user_message, error_code)`` when the gate trips, else ``None``. + + The caller emits one terminal-error SSE frame on a non-``None`` return. + """ + if not (user_image_data_urls and agent_config is not None): + return None + + from app.services.provider_capabilities import is_known_text_only_chat_model + + agent_litellm_params = agent_config.litellm_params or {} + agent_base_model = ( + agent_litellm_params.get("base_model") + if isinstance(agent_litellm_params, dict) + else None + ) + if not is_known_text_only_chat_model( + provider=agent_config.provider, + model_name=agent_config.model_name, + base_model=agent_base_model, + custom_provider=agent_config.custom_provider, + ): + return None + + model_label = agent_config.config_name or agent_config.model_name or "model" + ot.add_event( + "quota.denied", {"quota.code": "MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT"} + ) + return ( + ( + f"The selected model ({model_label}) does not support " + "image input. Switch to a vision-capable model " + "(e.g. GPT-4o, Claude, Gemini) or remove the image " + "attachment and try again." + ), + "MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT", + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py new file mode 100644 index 000000000..bca72b5ea --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py @@ -0,0 +1,868 @@ +"""``stream_new_chat`` — public entry point for a fresh chat turn. + +Slim composition layer over the per-concern modules in this folder and the +building blocks under ``flows/shared/``. Each phase corresponds to a numbered +block in the surrounding code so the on-the-wire ordering stays explicit: + + 1. Validation / config — auto-pin, LLM bundle, capability, premium reserve. + 2. Concurrent persistence + pre-stream setup — spawn DB writes, build the + connector, fetch the checkpointer, build the agent. + 3. Input assembly — history bootstrap, mentions, surfsense docs, reports. + 4. First SSE frames — message_start, start_step, turn-info, turn-status. + 5. Persistence join + message-id frames (ghost-thread protection). + 6. Initial thinking step + title task + runtime context. + 7. Stream loop with in-stream rate-limit recovery + mid-stream title emit. + 8. Finalize — premium debit, token-usage SSE, finish frames. + 9. Exception branch — classify, emit terminal error, finish frames. + 10. Finally — premium release, session close, assistant finalize, GC, span. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import time +from collections.abc import AsyncGenerator +from functools import partial +from typing import Any, Literal + +import anyio + +from app.agents.multi_agent_chat import create_multi_agent_chat_deep_agent +from app.agents.new_chat.chat_deepagent import create_surfsense_deep_agent +from app.agents.new_chat.filesystem_selection import FilesystemMode, FilesystemSelection +from app.agents.new_chat.middleware.busy_mutex import end_turn +from app.config import config as _app_config +from app.db import ChatVisibility, async_session_maker +from app.observability import otel as ot +from app.services.new_streaming_service import VercelStreamingService +from app.tasks.chat.content_builder import AssistantContentBuilder +from app.tasks.chat.streaming.agent.builder import build_main_agent_for_thread +from app.tasks.chat.streaming.contract.file_contract import log_file_contract +from app.tasks.chat.streaming.errors.emitter import emit_stream_terminal_error +from app.tasks.chat.streaming.flows.new_chat.auto_pin import resolve_initial_auto_pin +from app.tasks.chat.streaming.flows.new_chat.initial_thinking_step import ( + build_initial_thinking_step, + iter_initial_thinking_step_frame, +) +from app.tasks.chat.streaming.flows.new_chat.input_state import ( + build_new_chat_input_state, +) +from app.tasks.chat.streaming.flows.new_chat.llm_capability import ( + check_image_input_capability, +) +from app.tasks.chat.streaming.flows.new_chat.persistence_spawn import ( + await_persist_task, + spawn_persist_assistant_shell_task, + spawn_persist_user_task, + spawn_set_ai_responding_bg, +) +from app.tasks.chat.streaming.flows.new_chat.runtime_context import ( + build_new_chat_runtime_context, +) +from app.tasks.chat.streaming.flows.new_chat.title_gen import ( + await_pending_title_update, + maybe_emit_title_update, + spawn_title_task, +) +from app.tasks.chat.streaming.flows.shared.assistant_finalize import ( + finalize_assistant_message, +) +from app.tasks.chat.streaming.flows.shared.finalize_emit import iter_token_usage_frame +from app.tasks.chat.streaming.flows.shared.finally_cleanup import ( + close_session_and_clear_ai_responding, + run_gc_pass, +) +from app.tasks.chat.streaming.flows.shared.first_frames import ( + iter_final_frames, + iter_initial_frames, +) +from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle +from app.tasks.chat.streaming.flows.shared.pre_stream_setup import ( + get_chat_checkpointer, + setup_connector_and_firecrawl, +) +from app.tasks.chat.streaming.flows.shared.premium_quota import ( + PremiumReservation, + finalize_premium, + needs_premium_quota, + release_premium, + reserve_premium, +) +from app.tasks.chat.streaming.flows.shared.rate_limit_recovery import ( + can_recover_provider_rate_limit, + log_rate_limit_recovered, + reroute_to_next_auto_pin, +) +from app.tasks.chat.streaming.flows.shared.span import ( + close_chat_request_span, + open_chat_request_span, + set_agent_mode, +) +from app.tasks.chat.streaming.flows.shared.stream_loop import run_stream_loop +from app.tasks.chat.streaming.flows.shared.terminal_error import ( + handle_terminal_exception, +) +from app.tasks.chat.streaming.shared.stream_result import StreamResult +from app.utils.perf import get_perf_logger, log_system_snapshot + +logger = logging.getLogger(__name__) +_perf_log = get_perf_logger() + +# Holds spawned background tasks (set_ai_responding, persist_user, persist_asst) +# so the GC doesn't drop them before they finish. Kept at module level so it +# survives across turns within one process. +_background_tasks: set[asyncio.Task] = set() + + +async def stream_new_chat( + user_query: str, + search_space_id: int, + chat_id: int, + user_id: str | None = None, + llm_config_id: int = -1, + mentioned_document_ids: list[int] | None = None, + mentioned_surfsense_doc_ids: list[int] | None = None, + mentioned_folder_ids: list[int] | None = None, + mentioned_documents: list[dict[str, Any]] | None = None, + checkpoint_id: str | None = None, + needs_history_bootstrap: bool = False, + thread_visibility: ChatVisibility | None = None, + current_user_display_name: str | None = None, + disabled_tools: list[str] | None = None, + filesystem_selection: FilesystemSelection | None = None, + request_id: str | None = None, + user_image_data_urls: list[str] | None = None, + flow: Literal["new", "regenerate"] = "new", +) -> AsyncGenerator[str, None]: + """Stream a new chat turn using the SurfSense deep agent. + + Uses the Vercel AI SDK Data Stream Protocol (SSE). ``chat_id`` is the + LangGraph thread id (durable conversation memory via the checkpointer). + Manages its own database session so cleanup runs even when Starlette + cancels the task on client disconnect. + """ + streaming_service = VercelStreamingService() + stream_result = StreamResult() + _t_total = time.perf_counter() + fs_mode = filesystem_selection.mode.value if filesystem_selection else "cloud" + fs_platform = ( + filesystem_selection.client_platform.value if filesystem_selection else "web" + ) + stream_result.request_id = request_id + stream_result.turn_id = f"{chat_id}:{int(time.time() * 1000)}" + stream_result.filesystem_mode = fs_mode + stream_result.client_platform = fs_platform + + chat_agent_mode = "unknown" + chat_outcome = "success" + chat_error_category: str | None = None + chat_span_cm, chat_span = open_chat_request_span( + chat_id=chat_id, + search_space_id=search_space_id, + flow=flow, + request_id=request_id, + turn_id=stream_result.turn_id, + filesystem_mode=fs_mode, + client_platform=fs_platform, + agent_mode=chat_agent_mode, + ) + log_file_contract("turn_start", stream_result) + _perf_log.info( + "[stream_new_chat] filesystem_mode=%s client_platform=%s", + fs_mode, + fs_platform, + ) + log_system_snapshot("stream_new_chat_START") + + from app.services.token_tracking_service import start_turn + + accumulator = start_turn() + + premium_reservation: PremiumReservation | None = None + busy_error_raised = False + + emit_stream_error = partial( + emit_stream_terminal_error, + streaming_service=streaming_service, + flow=flow, + request_id=request_id, + thread_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + ) + + session = async_session_maker() + # Declared at function scope so SSE-yield join points and the finally + # clause see them on every exit path. + persist_user_task: asyncio.Task[int | None] | None = None + persist_asst_task: asyncio.Task[int | None] | None = None + try: + spawn_set_ai_responding_bg( + chat_id=chat_id, user_id=user_id, background_tasks=_background_tasks + ) + + # --- Block 1: LLM config + capability --- + + requested_llm_config_id = llm_config_id + requires_image_input = bool(user_image_data_urls) + + _t0 = time.perf_counter() + pin_result = await resolve_initial_auto_pin( + session, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + selected_llm_config_id=llm_config_id, + requires_image_input=requires_image_input, + requested_llm_config_id=requested_llm_config_id, + ) + if pin_result.error is not None: + message, error_code, error_kind = pin_result.error + yield emit_stream_error( + message=message, error_kind=error_kind, error_code=error_code + ) + yield streaming_service.format_done() + return + llm_config_id = pin_result.llm_config_id # type: ignore[assignment] + + llm, agent_config, llm_load_error = await load_llm_bundle( + session, config_id=llm_config_id, search_space_id=search_space_id + ) + if llm_load_error: + yield emit_stream_error( + message=llm_load_error, + error_kind="server_error", + error_code="SERVER_ERROR", + ) + yield streaming_service.format_done() + return + _perf_log.info( + "[stream_new_chat] LLM config loaded in %.3fs (config_id=%s)", + time.perf_counter() - _t0, + llm_config_id, + ) + + capability_error = check_image_input_capability( + user_image_data_urls=user_image_data_urls, agent_config=agent_config + ) + if capability_error is not None: + message, error_code = capability_error + yield emit_stream_error( + message=message, + error_kind="user_error", + error_code=error_code, + ) + yield streaming_service.format_done() + return + + if needs_premium_quota(agent_config, user_id): + premium_reservation = await reserve_premium( + agent_config=agent_config, user_id=user_id # type: ignore[arg-type] + ) + if not premium_reservation.allowed: + ot.add_event("quota.denied", {"quota.code": "PREMIUM_QUOTA_EXHAUSTED"}) + if requested_llm_config_id == 0: + pin_fallback = await resolve_initial_auto_pin( + session, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + selected_llm_config_id=0, + requires_image_input=requires_image_input, + requested_llm_config_id=requested_llm_config_id, + ) + if pin_fallback.error is not None: + message, error_code, error_kind = pin_fallback.error + yield emit_stream_error( + message=message, + error_kind=error_kind, + error_code=error_code, + ) + yield streaming_service.format_done() + return + llm_config_id = pin_fallback.llm_config_id # type: ignore[assignment] + ot.add_event( + "model.repin", + { + "repin.reason": "premium_quota_exhausted", + "repin.to_config_id": llm_config_id, + }, + ) + llm, agent_config, llm_load_error = await load_llm_bundle( + session, + config_id=llm_config_id, + search_space_id=search_space_id, + ) + if llm_load_error: + yield emit_stream_error( + message=llm_load_error, + error_kind="server_error", + error_code="SERVER_ERROR", + ) + yield streaming_service.format_done() + return + premium_reservation = None + # Re-route to free fallback logged via the structured + # stream-error logger so cost/analytics see the auto-switch. + from app.tasks.chat.streaming.errors.classifier import ( + log_chat_stream_error, + ) + + log_chat_stream_error( + flow=flow, + error_kind="premium_quota_exhausted", + error_code="PREMIUM_QUOTA_EXHAUSTED", + severity="info", + is_expected=True, + request_id=request_id, + thread_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + message=( + "Premium quota exhausted on pinned model; " + "auto-fallback switched to a free model" + ), + extra={ + "fallback_config_id": llm_config_id, + "auto_fallback": True, + }, + ) + else: + yield emit_stream_error( + message=( + "Buy more tokens to continue with this model, or " + "switch to a free model" + ), + error_kind="premium_quota_exhausted", + error_code="PREMIUM_QUOTA_EXHAUSTED", + severity="info", + is_expected=True, + extra={ + "resolved_config_id": llm_config_id, + "auto_fallback": False, + }, + ) + yield streaming_service.format_done() + return + + if not llm: + yield emit_stream_error( + message="Failed to create LLM instance", + error_kind="server_error", + error_code="SERVER_ERROR", + ) + yield streaming_service.format_done() + return + + # --- Block 2: Spawn concurrent persistence; build pre-stream setup --- + + persist_user_task = spawn_persist_user_task( + chat_id=chat_id, + user_id=user_id, + turn_id=stream_result.turn_id, + user_query=user_query, + user_image_data_urls=user_image_data_urls, + mentioned_documents=mentioned_documents, + background_tasks=_background_tasks, + ) + persist_asst_task = spawn_persist_assistant_shell_task( + chat_id=chat_id, + user_id=user_id, + turn_id=stream_result.turn_id, + background_tasks=_background_tasks, + ) + + _t0 = time.perf_counter() + connector_service, firecrawl_api_key = await setup_connector_and_firecrawl( + session, search_space_id=search_space_id + ) + _perf_log.info( + "[stream_new_chat] Connector service + firecrawl key in %.3fs", + time.perf_counter() - _t0, + ) + + _t0 = time.perf_counter() + checkpointer = await get_chat_checkpointer() + _perf_log.info( + "[stream_new_chat] Checkpointer ready in %.3fs", time.perf_counter() - _t0 + ) + + visibility = thread_visibility or ChatVisibility.PRIVATE + use_multi_agent = bool(_app_config.MULTI_AGENT_CHAT_ENABLED) + chat_agent_mode = "multi" if use_multi_agent else "single" + set_agent_mode(chat_span, chat_agent_mode) + + _t0 = time.perf_counter() + agent_factory = ( + create_multi_agent_chat_deep_agent + if use_multi_agent + else create_surfsense_deep_agent + ) + # Build the agent inline. Provider 429s surface through the in-stream + # recovery loop below, which repins the thread to an eligible + # alternative config and rebuilds the agent before the user sees any + # output. + agent = await build_main_agent_for_thread( + agent_factory, + llm=llm, + search_space_id=search_space_id, + db_session=session, + connector_service=connector_service, + checkpointer=checkpointer, + user_id=user_id, + thread_id=chat_id, + agent_config=agent_config, + firecrawl_api_key=firecrawl_api_key, + thread_visibility=visibility, + filesystem_selection=filesystem_selection, + disabled_tools=disabled_tools, + mentioned_document_ids=mentioned_document_ids, + ) + _perf_log.info( + "[stream_new_chat] Agent created in %.3fs", time.perf_counter() - _t0 + ) + + # --- Block 3: Input assembly --- + + _t0 = time.perf_counter() + assembled = await build_new_chat_input_state( + session, + chat_id=chat_id, + search_space_id=search_space_id, + user_query=user_query, + user_image_data_urls=user_image_data_urls, + mentioned_document_ids=mentioned_document_ids, + mentioned_surfsense_doc_ids=mentioned_surfsense_doc_ids, + mentioned_folder_ids=mentioned_folder_ids, + mentioned_documents=mentioned_documents, + needs_history_bootstrap=needs_history_bootstrap, + thread_visibility=visibility, + current_user_display_name=current_user_display_name, + filesystem_mode=fs_mode, + request_id=request_id, + turn_id=stream_result.turn_id, + ) + input_state = assembled.input_state + accepted_folder_ids = assembled.accepted_folder_ids + mentioned_surfsense_docs = assembled.mentioned_surfsense_docs + _perf_log.info( + "[stream_new_chat] History bootstrap + doc/report queries in %.3fs", + time.perf_counter() - _t0, + ) + + # All pre-streaming DB reads done. Commit to release the transaction + # and its ACCESS SHARE locks so we don't block DDL (e.g. migrations) + # for the entire LLM streaming duration. Tools that need DB access + # during streaming start their own short-lived transactions (or use + # isolated sessions). + await session.commit() + # Detach heavy ORM objects (documents with chunks, reports, etc.) + # from the session identity map now that we've extracted what we + # need. Without this they accumulate in memory for the entire + # streaming duration (which can be several minutes). + session.expunge_all() + + _perf_log.info( + "[stream_new_chat] Total pre-stream setup in %.3fs (chat_id=%s)", + time.perf_counter() - _t_total, + chat_id, + ) + + configurable: dict[str, Any] = { + "thread_id": str(chat_id), + "request_id": request_id or "unknown", + "turn_id": stream_result.turn_id, + } + if checkpoint_id: + configurable["checkpoint_id"] = checkpoint_id + + config = { + "configurable": configurable, + # Effectively uncapped, matching the agent-level ``with_config`` + # default in ``chat_deepagent.create_agent`` and the unbounded + # ``while(true)`` in OpenCode's ``session/processor.ts``. Real + # circuit-breakers live in middleware (``DoomLoopMiddleware``, + # plus ``enable_tool_call_limit`` / ``enable_model_call_limit``). + # The original 25 (and our previous 80 bump) hit users on + # legitimate multi-tool plans. + "recursion_limit": 10_000, + } + + # --- Block 4: First SSE frames --- + + for sse in iter_initial_frames(streaming_service, turn_id=stream_result.turn_id): + yield sse + + # --- Block 5: Persistence join + message-id frames --- + + user_message_id = await await_persist_task( + persist_user_task, + chat_id=chat_id, + turn_id=stream_result.turn_id, + log_label="persist_user_task", + ) + if user_message_id is None: + yield emit_stream_error( + message="We couldn't save your message. Please try again in a moment.", + error_kind="server_error", + error_code="MESSAGE_PERSIST_FAILED", + ) + for sse in iter_final_frames(streaming_service): + yield sse + return + + # Emit canonical user message id BEFORE any LLM streaming so the FE + # can rename its optimistic ``msg-user-XXX`` placeholder to + # ``msg-{user_message_id}`` and unlock features gated on a real DB id + # (comments, edit-from-this-message). See B4 in the + # ``sse-based_message_id_handshake`` plan. + yield streaming_service.format_data( + "user-message-id", + {"message_id": user_message_id, "turn_id": stream_result.turn_id}, + ) + + assistant_message_id = await await_persist_task( + persist_asst_task, + chat_id=chat_id, + turn_id=stream_result.turn_id, + log_label="persist_asst_task", + ) + if assistant_message_id is None: + # Genuine DB failure — abort the turn rather than stream into a + # void. The user row is already persisted so the legacy + # ghost-thread gate isn't reopened. + yield emit_stream_error( + message=( + "We couldn't initialize the assistant message. Please try again." + ), + error_kind="server_error", + error_code="MESSAGE_PERSIST_FAILED", + ) + for sse in iter_final_frames(streaming_service): + yield sse + return + + yield streaming_service.format_data( + "assistant-message-id", + {"message_id": assistant_message_id, "turn_id": stream_result.turn_id}, + ) + + stream_result.assistant_message_id = assistant_message_id + stream_result.content_builder = AssistantContentBuilder() + + # --- Block 6: Initial thinking step + title task + runtime context --- + + initial_step = build_initial_thinking_step( + user_query=user_query, + user_image_data_urls=user_image_data_urls, + mentioned_surfsense_docs=mentioned_surfsense_docs, + ) + for sse in iter_initial_thinking_step_frame( + initial_step, + streaming_service=streaming_service, + content_builder=stream_result.content_builder, + ): + yield sse + + initial_step_id = initial_step.step_id + initial_step_title = initial_step.title + initial_step_items = initial_step.items + # Drop the heavy ORM objects + the container that holds them so they + # aren't retained for the entire streaming duration. ``input_state`` + # already carries the langchain_messages list independently. + del assembled, mentioned_surfsense_docs + + title_task = spawn_title_task( + chat_id=chat_id, + user_query=user_query, + user_image_data_urls=user_image_data_urls, + assistant_message_id=assistant_message_id, + llm=llm, + agent_config=agent_config, + ) + title_emitted = False + + runtime_context = build_new_chat_runtime_context( + search_space_id=search_space_id, + mentioned_document_ids=mentioned_document_ids, + accepted_folder_ids=accepted_folder_ids, + mentioned_folder_ids=mentioned_folder_ids, + request_id=request_id, + turn_id=stream_result.turn_id, + ) + + # --- Block 7: Stream loop --- + + _t_stream_start = time.perf_counter() + runtime_rate_limit_recovered = False + + def _on_first_event() -> None: + _perf_log.info( + "[stream_new_chat] First agent event in %.3fs (time since stream start), " + "%.3fs (total since request start) (chat_id=%s)", + time.perf_counter() - _t_stream_start, + time.perf_counter() - _t_total, + chat_id, + ) + + async def _recover(exc: BaseException, first_event_seen: bool): + nonlocal llm_config_id, llm, agent_config, runtime_rate_limit_recovered + nonlocal title_task + if not can_recover_provider_rate_limit( + exc, + first_event_seen=first_event_seen, + runtime_rate_limit_recovered=runtime_rate_limit_recovered, + requested_llm_config_id=requested_llm_config_id, + current_llm_config_id=llm_config_id, + ): + return None + runtime_rate_limit_recovered = True + previous_config_id = llm_config_id + llm_config_id = await reroute_to_next_auto_pin( + session, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + current_llm_config_id=llm_config_id, + requires_image_input=requires_image_input, + ) + new_llm, new_agent_config, llm_load_err = await load_llm_bundle( + session, config_id=llm_config_id, search_space_id=search_space_id + ) + if llm_load_err: + # Re-raise the original so the terminal-error path classifies + # it correctly (don't swallow as "config load error"). + return None + llm = new_llm + agent_config = new_agent_config + + # Title gen used the initial llm object. After a runtime repin we + # keep the stream focused on response recovery and skip title gen + # for this turn. + if title_task is not None and not title_task.done(): + title_task.cancel() + title_task = None + + _t_rebuild = time.perf_counter() + new_agent = await build_main_agent_for_thread( + agent_factory, + llm=llm, + search_space_id=search_space_id, + db_session=session, + connector_service=connector_service, + checkpointer=checkpointer, + user_id=user_id, + thread_id=chat_id, + agent_config=agent_config, + firecrawl_api_key=firecrawl_api_key, + thread_visibility=visibility, + filesystem_selection=filesystem_selection, + disabled_tools=disabled_tools, + mentioned_document_ids=mentioned_document_ids, + ) + _perf_log.info( + "[stream_new_chat] Runtime rate-limit recovery repinned " + "config_id=%s -> %s and rebuilt agent in %.3fs", + previous_config_id, + llm_config_id, + time.perf_counter() - _t_rebuild, + ) + log_rate_limit_recovered( + flow=flow, + request_id=request_id, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + previous_config_id=previous_config_id, + new_config_id=llm_config_id, + ) + return new_agent + + async for sse in run_stream_loop( + agent=agent, + streaming_service=streaming_service, + config=config, + input_data=input_state, + stream_result=stream_result, + step_prefix="thinking", + initial_step_id=initial_step_id, + initial_step_title=initial_step_title, + initial_step_items=initial_step_items, + fallback_commit_search_space_id=search_space_id, + fallback_commit_created_by_id=user_id, + fallback_commit_filesystem_mode=( + filesystem_selection.mode if filesystem_selection else FilesystemMode.CLOUD + ), + fallback_commit_thread_id=chat_id, + runtime_context=runtime_context, + content_builder=stream_result.content_builder, + recover=_recover, + on_first_event=_on_first_event, + ): + yield sse + # Inject the title update mid-stream as soon as the background + # task finishes; gated so we emit at most once. + async for title_sse in maybe_emit_title_update( + title_task=title_task, + title_emitted=title_emitted, + chat_id=chat_id, + accumulator=accumulator, + streaming_service=streaming_service, + ): + yield title_sse + title_emitted = True + # Account for the case where the task completed but produced no + # title — flip the flag anyway so we don't keep checking it. + if ( + title_task is not None + and title_task.done() + and not title_emitted + ): + title_emitted = True + + _perf_log.info( + "[stream_new_chat] Agent stream completed in %.3fs (chat_id=%s)", + time.perf_counter() - _t_stream_start, + chat_id, + ) + log_system_snapshot("stream_new_chat_END") + + # --- Block 8: Finalize --- + + if stream_result.is_interrupted: + ot.add_event("chat.interrupted", {"chat.flow": flow}) + if title_task is not None and not title_task.done(): + title_task.cancel() + for sse in iter_token_usage_frame( + streaming_service, + accumulator=accumulator, + log_label="interrupted new_chat", + ): + yield sse + yield streaming_service.format_finish_step() + yield streaming_service.format_finish() + yield streaming_service.format_done() + return + + async for title_sse in await_pending_title_update( + title_task=title_task, + title_emitted=title_emitted, + chat_id=chat_id, + accumulator=accumulator, + streaming_service=streaming_service, + ): + yield title_sse + + # Finalize premium credit debit with the actual provider cost reported + # by LiteLLM, summed across every call in the turn. Mirrors the + # pre-cost behaviour of "premium turn → all calls count" so free + # sub-agent calls during a premium turn still contribute to the bill + # (they're $0 in practice anyway). + if premium_reservation is not None and user_id: + await finalize_premium( + reservation=premium_reservation, + user_id=user_id, + accumulator=accumulator, + ) + premium_reservation = None + + for sse in iter_token_usage_frame( + streaming_service, accumulator=accumulator, log_label="normal new_chat" + ): + yield sse + + for sse in iter_final_frames(streaming_service): + yield sse + + except Exception as exc: + frames, summary = handle_terminal_exception( + exc, + flow=flow, + flow_label="chat", + log_prefix="stream_new_chat", + streaming_service=streaming_service, + request_id=request_id, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + chat_span=chat_span, + ) + if summary["busy_error_raised"]: + busy_error_raised = True + chat_outcome = summary["chat_outcome"] + chat_error_category = summary["chat_error_category"] + for sse in frames: + yield sse + + finally: + # Shield the ENTIRE async cleanup from anyio cancel-scope cancellation. + # Starlette's BaseHTTPMiddleware uses anyio task groups; on client + # disconnect, it cancels the scope with level-triggered cancellation + # — every unshielded ``await`` would raise CancelledError immediately. + # Without this the very first ``await`` (session.rollback) would + # raise, ``except Exception`` wouldn't catch it (CancelledError is a + # BaseException), and the rest of cleanup — including session.close() + # — would never run. + with anyio.CancelScope(shield=True): + # Authoritative fallback cleanup for lock/cancel state. Middleware + # teardown can be skipped on some client-abort paths. + end_turn(str(chat_id)) + + if premium_reservation is not None and user_id: + await release_premium( + reservation=premium_reservation, user_id=user_id + ) + + await close_session_and_clear_ai_responding(session, chat_id) + + await finalize_assistant_message( + stream_result=stream_result, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + accumulator=accumulator, + log_prefix="stream_new_chat", + ) + + # Persist any sandbox-produced files to local storage so they remain + # downloadable after the Daytona sandbox auto-deletes. + if stream_result and stream_result.sandbox_files: + with contextlib.suppress(Exception): + from app.agents.new_chat.sandbox import ( + is_sandbox_enabled, + persist_and_delete_sandbox, + ) + + if is_sandbox_enabled(): + with anyio.CancelScope(shield=True): + await persist_and_delete_sandbox( + chat_id, stream_result.sandbox_files + ) + + # ``aafter_agent`` doesn't fire on ``interrupt()`` or early bailout. + # Skip on ``BusyError`` (caller never acquired the lock). + if not busy_error_raised: + with contextlib.suppress(Exception): + end_turn(str(chat_id)) + _perf_log.info( + "[stream_new_chat] end_turn cleanup (chat_id=%s)", chat_id + ) + + # Break circular refs held by the agent graph, tools, and LLM + # wrappers so the GC can reclaim them in a single pass. + agent = llm = connector_service = None # noqa: F841 + input_state = stream_result = None # noqa: F841 + session = None # noqa: F841 + + run_gc_pass(log_prefix="stream_new_chat", chat_id=chat_id) + close_chat_request_span( + span_cm=chat_span_cm, + span=chat_span, + chat_outcome=chat_outcome, + chat_agent_mode=chat_agent_mode, + flow=flow, + chat_error_category=chat_error_category, + duration_seconds=time.perf_counter() - _t_total, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/persistence_spawn.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/persistence_spawn.py new file mode 100644 index 000000000..9ea5d2ad6 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/persistence_spawn.py @@ -0,0 +1,129 @@ +"""Concurrent persistence tasks spawned right after the initial validation gate. + +These run *during* the rest of the pre-stream setup so we don't serialize +their latency against agent construction. Awaiting them at the SSE message-id +yield sites preserves the ghost-thread protection (the user-row INSERT must +succeed before any LLM streaming begins). + +The ``set_ai_responding`` flag flip runs fully fire-and-forget on its own +shielded session — failures only delay the "AI is responding…" UI flag, not +the response itself. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any +from uuid import UUID + +from app.db import shielded_async_session +from app.services.chat_session_state_service import set_ai_responding +from app.tasks.chat.persistence import ( + persist_assistant_shell, + persist_user_turn, +) + +logger = logging.getLogger(__name__) + + +def spawn_set_ai_responding_bg( + *, + chat_id: int, + user_id: str | None, + background_tasks: set[asyncio.Task[Any]], +) -> None: + """Fire-and-forget: flip the per-thread AI-responding flag on its own session. + + Errors are swallowed and logged — the worst case is a stale UI flag, which + is preferable to delaying the SSE stream behind a flag write. + """ + if not user_id: + return + + async def _bg_set_ai_responding() -> None: + try: + async with shielded_async_session() as s: + await set_ai_responding(s, chat_id, UUID(user_id)) + except Exception: + logger.warning( + "set_ai_responding failed (chat_id=%s)", + chat_id, + exc_info=True, + ) + + t = asyncio.create_task(_bg_set_ai_responding()) + background_tasks.add(t) + t.add_done_callback(background_tasks.discard) + + +def spawn_persist_user_task( + *, + chat_id: int, + user_id: str | None, + turn_id: str, + user_query: str, + user_image_data_urls: list[str] | None, + mentioned_documents: list[dict[str, Any]] | None, + background_tasks: set[asyncio.Task[Any]], +) -> asyncio.Task[int | None]: + """Spawn the user-row INSERT; await at the user-message-id yield site.""" + task = asyncio.create_task( + persist_user_turn( + chat_id=chat_id, + user_id=user_id, + turn_id=turn_id, + user_query=user_query, + user_image_data_urls=user_image_data_urls, + mentioned_documents=mentioned_documents, + ) + ) + background_tasks.add(task) + task.add_done_callback(background_tasks.discard) + return task + + +def spawn_persist_assistant_shell_task( + *, + chat_id: int, + user_id: str | None, + turn_id: str, + background_tasks: set[asyncio.Task[Any]], +) -> asyncio.Task[int | None]: + """Spawn the assistant-shell INSERT; await at the assistant-message-id yield site.""" + task = asyncio.create_task( + persist_assistant_shell( + chat_id=chat_id, + user_id=user_id, + turn_id=turn_id, + ) + ) + background_tasks.add(task) + task.add_done_callback(background_tasks.discard) + return task + + +async def await_persist_task( + task: asyncio.Task[int | None] | None, + *, + chat_id: int, + turn_id: str, + log_label: str, +) -> int | None: + """Join a spawned persistence task with ``shield`` + uniform error handling. + + ``shield`` keeps the DB write alive if the SSE generator is cancelled by + client disconnect mid-await. Returns ``None`` on failure; the caller + abort-paths the turn with a friendly error SSE. + """ + if task is None: + return None + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + raise + except Exception: + logger.exception( + "%s failed (chat_id=%s, turn_id=%s)", log_label, chat_id, turn_id + ) + return None diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py new file mode 100644 index 000000000..1f11be1fe --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py @@ -0,0 +1,38 @@ +"""Build the per-invocation ``SurfSenseContextSchema`` for a new-chat turn. + +Carries the per-turn read inputs that middlewares read via +``runtime.context.*`` instead of from their ``__init__`` closures, so the same +compiled-agent instance can serve multiple turns with different +mention lists / request ids / turn ids without rebuilding the graph. +""" + +from __future__ import annotations + +from app.agents.new_chat.context import SurfSenseContextSchema + + +def build_new_chat_runtime_context( + *, + search_space_id: int, + mentioned_document_ids: list[int] | None, + accepted_folder_ids: list[int], + mentioned_folder_ids: list[int] | None, + request_id: str | None, + turn_id: str, +) -> SurfSenseContextSchema: + """``mentioned_document_ids`` is consumed by ``KnowledgePriorityMiddleware``. + + ``accepted_folder_ids`` (post-resolve) wins over the raw + ``mentioned_folder_ids`` from the request: the resolver drops chips that + pointed at deleted folders or folders the caller can't see, so middlewares + only get authorized ids. + """ + return SurfSenseContextSchema( + search_space_id=search_space_id, + mentioned_document_ids=list(mentioned_document_ids or []), + mentioned_folder_ids=list( + accepted_folder_ids or mentioned_folder_ids or [] + ), + request_id=request_id, + turn_id=turn_id, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/title_gen.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/title_gen.py new file mode 100644 index 000000000..11312110f --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/title_gen.py @@ -0,0 +1,237 @@ +"""Background thread-title generation (first-response only). + +The first assistant response in a thread gets a short auto-generated title +inserted into ``new_chat_threads.title``. We: + + 1. Spawn the generation as an ``asyncio.Task`` so it runs in parallel with + the agent stream (no extra TTFT). + 2. Probe inside the task (on its own shielded session) whether this is + actually the first response — newer turns short-circuit to ``None``. + 3. Inject the resulting ``thread-title-update`` SSE frame on the first agent + event after the task completes (mid-stream interlock), or right before + the finish frames (post-stream join) if the task hadn't finished yet. + +Usage tokens come directly off the response (LiteLLM's async callback fires +via fire-and-forget ``create_task``, so the ``TokenTrackingCallback`` would +run too late). We also blank the per-task accumulator so the late callback +doesn't double-count. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING, Any + +from sqlalchemy.future import select + +from app.db import NewChatMessage, NewChatThread, shielded_async_session +from app.prompts import TITLE_GENERATION_PROMPT +from app.services.new_streaming_service import VercelStreamingService + +if TYPE_CHECKING: + from app.agents.new_chat.llm_config import AgentConfig + from app.services.token_tracking_service import TokenAccumulator + + +logger = logging.getLogger(__name__) + + +def spawn_title_task( + *, + chat_id: int, + user_query: str, + user_image_data_urls: list[str] | None, + assistant_message_id: int | None, + llm: Any, + agent_config: AgentConfig | None, +) -> asyncio.Task[tuple[str | None, dict | None]] | None: + """Spawn ``_generate_title``; returns ``None`` when prerequisites aren't met. + + Title gen is gated on a real ``assistant_message_id`` so a stream that + aborts before persistence can never leave a thread with a title and no + anchoring rows. + """ + if assistant_message_id is None: + return None + return asyncio.create_task( + _generate_title( + chat_id=chat_id, + user_query=user_query, + user_image_data_urls=user_image_data_urls, + assistant_message_id=assistant_message_id, + llm=llm, + agent_config=agent_config, + ) + ) + + +async def _generate_title( + *, + chat_id: int, + user_query: str, + user_image_data_urls: list[str] | None, + assistant_message_id: int, + llm: Any, + agent_config: AgentConfig | None, +) -> tuple[str | None, dict | None]: + """Probe is-first-response, then call ``acompletion``. Returns ``(title, usage)``.""" + try: + from litellm import acompletion + + from app.services.llm_router_service import LLMRouterService + from app.services.provider_api_base import resolve_api_base + from app.services.token_tracking_service import _turn_accumulator + + # Excludes this turn's own assistant row (pre-written by + # ``persist_assistant_shell``) — without the ``!=`` filter the gate + # would false-negative on every turn after the first. + try: + async with shielded_async_session() as probe_session: + probe_result = await probe_session.execute( + select(NewChatMessage.id) + .filter( + NewChatMessage.thread_id == chat_id, + NewChatMessage.role == "assistant", + NewChatMessage.id != assistant_message_id, + ) + .limit(1) + ) + is_first_response = probe_result.scalars().first() is None + except Exception: + logger.warning( + "[TitleGen] first-response probe failed (chat_id=%s)", + chat_id, + exc_info=True, + ) + return None, None + + if not is_first_response: + return None, None + + _turn_accumulator.set(None) + + title_seed = user_query.strip() or ( + f"[{len(user_image_data_urls or [])} image(s)]" + if user_image_data_urls + else "" + ) + prompt = TITLE_GENERATION_PROMPT.replace( + "{user_query}", title_seed[:500] or "(message)" + ) + messages = [{"role": "user", "content": prompt}] + + if getattr(llm, "model", None) == "auto": + router = LLMRouterService.get_router() + response = await router.acompletion(model="auto", messages=messages) + else: + # Apply the same ``api_base`` cascade chat / vision / image-gen + # call sites use so we never inherit ``litellm.api_base`` + # (commonly set by ``AZURE_OPENAI_ENDPOINT``) when the chat + # config itself ships an empty ``api_base``. Without this the + # title-gen on an OpenRouter chat config would 404 against the + # inherited Azure endpoint — see ``provider_api_base`` for the + # same bug repro on the image-gen / vision paths. + raw_model = getattr(llm, "model", "") or "" + provider_prefix = ( + raw_model.split("/", 1)[0] if "/" in raw_model else None + ) + provider_value = ( + agent_config.provider if agent_config is not None else None + ) + title_api_base = resolve_api_base( + provider=provider_value, + provider_prefix=provider_prefix, + config_api_base=getattr(llm, "api_base", None), + ) + response = await acompletion( + model=raw_model, + messages=messages, + api_key=getattr(llm, "api_key", None), + api_base=title_api_base, + ) + + usage_info = None + usage = getattr(response, "usage", None) + if usage: + raw_model = getattr(llm, "model", "") or "" + model_name = ( + raw_model.split("/", 1)[-1] + if "/" in raw_model + else (raw_model or response.model or "unknown") + ) + usage_info = { + "model": model_name, + "prompt_tokens": getattr(usage, "prompt_tokens", 0) or 0, + "completion_tokens": getattr(usage, "completion_tokens", 0) or 0, + "total_tokens": getattr(usage, "total_tokens", 0) or 0, + } + + raw_title = response.choices[0].message.content.strip() + if raw_title and len(raw_title) <= 100: + return raw_title.strip("\"'"), usage_info + return None, usage_info + except Exception: + logger.exception("[TitleGen] _generate_title failed") + return None, None + + +async def maybe_emit_title_update( + *, + title_task: asyncio.Task[tuple[str | None, dict | None]] | None, + title_emitted: bool, + chat_id: int, + accumulator: TokenAccumulator, + streaming_service: VercelStreamingService, +): + """Inject one ``thread-title-update`` SSE if the task completed. + + Yields the SSE frame (when applicable). Returns nothing; the orchestrator + flips ``title_emitted`` itself after iterating so we don't fight Python's + nonlocal-in-generator semantics. + """ + if title_task is None or title_emitted or not title_task.done(): + return + generated_title, title_usage = title_task.result() + if title_usage: + accumulator.add(**title_usage) + if generated_title: + async with shielded_async_session() as title_session: + title_thread_result = await title_session.execute( + select(NewChatThread).filter(NewChatThread.id == chat_id) + ) + title_thread = title_thread_result.scalars().first() + if title_thread: + title_thread.title = generated_title + await title_session.commit() + yield streaming_service.format_thread_title_update(chat_id, generated_title) + + +async def await_pending_title_update( + *, + title_task: asyncio.Task[tuple[str | None, dict | None]] | None, + title_emitted: bool, + chat_id: int, + accumulator: TokenAccumulator, + streaming_service: VercelStreamingService, +): + """If the task hadn't completed during the stream, await it now and emit. + + Used right before the finish frames in the success path. Mirror of + ``maybe_emit_title_update`` but unconditionally awaits. + """ + if title_task is None or title_emitted: + return + generated_title, title_usage = await title_task + if title_usage: + accumulator.add(**title_usage) + if generated_title: + async with shielded_async_session() as title_session: + title_thread_result = await title_session.execute( + select(NewChatThread).filter(NewChatThread.id == chat_id) + ) + title_thread = title_thread_result.scalars().first() + if title_thread: + title_thread.title = generated_title + await title_session.commit() + yield streaming_service.format_thread_title_update(chat_id, generated_title) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/__init__.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/__init__.py new file mode 100644 index 000000000..ed0683e19 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/__init__.py @@ -0,0 +1,12 @@ +"""Resume-chat streaming flow. + +Public entry point ``stream_resume_chat`` is the slim coroutine in +``orchestrator.py`` that composes the per-concern modules in this folder and +the building blocks under ``flows/shared/``. +""" + +from __future__ import annotations + +from app.tasks.chat.streaming.flows.resume_chat.orchestrator import stream_resume_chat + +__all__ = ["stream_resume_chat"] diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/assistant_shell.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/assistant_shell.py new file mode 100644 index 000000000..2f34387f8 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/assistant_shell.py @@ -0,0 +1,31 @@ +"""Pre-write a fresh assistant row for this resume turn. + +The original (interrupted) ``stream_new_chat`` invocation already persisted +its own assistant row anchored to a different ``turn_id``; resume allocates a +new ``turn_id`` (per-request, see ``orchestrator``) so we need a separate row +keyed on the same ``(thread_id, turn_id, ASSISTANT)`` invariant. + +Idempotent against migration 141's partial unique index — recovers the +existing id on retry. + +Resume does NOT emit ``data-user-message-id``: the user row is from the +original interrupted turn (different ``turn_id``) and is never re-persisted +here. See B5 in the ``sse-based_message_id_handshake`` plan. +""" + +from __future__ import annotations + +from app.tasks.chat.persistence import persist_assistant_shell + + +async def persist_resume_assistant_shell( + *, + chat_id: int, + user_id: str | None, + turn_id: str, +) -> int | None: + return await persist_assistant_shell( + chat_id=chat_id, + user_id=user_id, + turn_id=turn_id, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py new file mode 100644 index 000000000..b67ac987e --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py @@ -0,0 +1,629 @@ +"""``stream_resume_chat`` — public entry point for a HITL resume turn. + +Slim composition layer over the per-concern modules in this folder and the +building blocks under ``flows/shared/``. Mirrors ``stream_new_chat`` but: + + * No user-message persistence (the original turn already wrote it). + * No mentions / surfsense-doc / report context assembly (seeded by original). + * No title generation (only fires on first-response). + * Synchronous ``persist_assistant_shell`` call (we have no other in-flight + pre-stream work to overlap it with). + * ``input_data`` is a ``Command(resume=lg_resume_map)`` instead of a + LangChain message list. +""" + +from __future__ import annotations + +import contextlib +import gc +import logging +import sys +import time +import uuid as _uuid +from collections.abc import AsyncGenerator +from functools import partial +from typing import Any +from uuid import UUID + +import anyio + +from app.agents.multi_agent_chat import create_multi_agent_chat_deep_agent +from app.agents.new_chat.chat_deepagent import create_surfsense_deep_agent +from app.agents.new_chat.filesystem_selection import FilesystemMode, FilesystemSelection +from app.agents.new_chat.middleware.busy_mutex import end_turn +from app.config import config as _app_config +from app.db import ChatVisibility, async_session_maker, shielded_async_session +from app.observability import otel as ot +from app.services.chat_session_state_service import set_ai_responding +from app.services.new_streaming_service import VercelStreamingService +from app.tasks.chat.content_builder import AssistantContentBuilder +from app.tasks.chat.streaming.agent.builder import build_main_agent_for_thread +from app.tasks.chat.streaming.contract.file_contract import log_file_contract +from app.tasks.chat.streaming.errors.emitter import emit_stream_terminal_error +from app.tasks.chat.streaming.flows.resume_chat.assistant_shell import ( + persist_resume_assistant_shell, +) +from app.tasks.chat.streaming.flows.resume_chat.resume_routing import ( + build_resume_routing, +) +from app.tasks.chat.streaming.flows.resume_chat.runtime_context import ( + build_resume_chat_runtime_context, +) +from app.tasks.chat.streaming.flows.shared.assistant_finalize import ( + finalize_assistant_message, +) +from app.tasks.chat.streaming.flows.shared.finalize_emit import iter_token_usage_frame +from app.tasks.chat.streaming.flows.shared.finally_cleanup import ( + close_session_and_clear_ai_responding, + run_gc_pass, +) +from app.tasks.chat.streaming.flows.shared.first_frames import ( + iter_final_frames, + iter_initial_frames, +) +from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle +from app.tasks.chat.streaming.flows.shared.pre_stream_setup import ( + get_chat_checkpointer, + setup_connector_and_firecrawl, +) +from app.tasks.chat.streaming.flows.shared.premium_quota import ( + PremiumReservation, + finalize_premium, + needs_premium_quota, + release_premium, + reserve_premium, +) +from app.tasks.chat.streaming.flows.shared.rate_limit_recovery import ( + can_recover_provider_rate_limit, + log_rate_limit_recovered, + reroute_to_next_auto_pin, +) +from app.tasks.chat.streaming.flows.shared.span import ( + close_chat_request_span, + open_chat_request_span, + set_agent_mode, +) +from app.tasks.chat.streaming.flows.shared.stream_loop import run_stream_loop +from app.tasks.chat.streaming.flows.shared.terminal_error import ( + handle_terminal_exception, +) +from app.tasks.chat.streaming.shared.stream_result import StreamResult +from app.tasks.chat.streaming.shared.utils import resume_step_prefix +from app.utils.perf import get_perf_logger, log_system_snapshot + +logger = logging.getLogger(__name__) +_perf_log = get_perf_logger() + + +async def stream_resume_chat( + chat_id: int, + search_space_id: int, + decisions: list[dict], + user_id: str | None = None, + llm_config_id: int = -1, + thread_visibility: ChatVisibility | None = None, + filesystem_selection: FilesystemSelection | None = None, + request_id: str | None = None, + disabled_tools: list[str] | None = None, +) -> AsyncGenerator[str, None]: + """Resume a paused HITL turn with the user's decisions. + + Mirrors ``stream_new_chat`` except for the resume-specific routing of + ``decisions`` to per-``tool_call_id`` slices (``build_resume_routing``). + """ + streaming_service = VercelStreamingService() + stream_result = StreamResult() + _t_total = time.perf_counter() + fs_mode = filesystem_selection.mode.value if filesystem_selection else "cloud" + fs_platform = ( + filesystem_selection.client_platform.value if filesystem_selection else "web" + ) + stream_result.request_id = request_id + stream_result.turn_id = f"{chat_id}:{int(time.time() * 1000)}" + stream_result.filesystem_mode = fs_mode + stream_result.client_platform = fs_platform + + chat_agent_mode = "unknown" + chat_outcome = "success" + chat_error_category: str | None = None + chat_span_cm, chat_span = open_chat_request_span( + chat_id=chat_id, + search_space_id=search_space_id, + flow="resume", + request_id=request_id, + turn_id=stream_result.turn_id, + filesystem_mode=fs_mode, + client_platform=fs_platform, + agent_mode=chat_agent_mode, + ) + log_file_contract("turn_start", stream_result) + _perf_log.info( + "[stream_resume] filesystem_mode=%s client_platform=%s", + fs_mode, + fs_platform, + ) + + from app.services.token_tracking_service import start_turn + + accumulator = start_turn() + + premium_reservation: PremiumReservation | None = None + busy_error_raised = False + + emit_stream_error = partial( + emit_stream_terminal_error, + streaming_service=streaming_service, + flow="resume", + request_id=request_id, + thread_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + ) + + session = async_session_maker() + try: + if user_id: + await set_ai_responding(session, chat_id, UUID(user_id)) + + requested_llm_config_id = llm_config_id + + # --- LLM config --- + + _t0 = time.perf_counter() + try: + from app.services.auto_model_pin_service import ( + resolve_or_get_pinned_llm_config_id, + ) + + pinned = await resolve_or_get_pinned_llm_config_id( + session, + thread_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + selected_llm_config_id=llm_config_id, + ) + llm_config_id = pinned.resolved_llm_config_id + ot.add_event( + "model.pin.resolved", + { + "pin.requested_id": requested_llm_config_id, + "pin.resolved_id": llm_config_id, + "pin.requires_image_input": False, + }, + ) + except ValueError as pin_error: + yield emit_stream_error( + message=str(pin_error), + error_kind="server_error", + error_code="SERVER_ERROR", + ) + yield streaming_service.format_done() + return + + llm, agent_config, llm_load_error = await load_llm_bundle( + session, config_id=llm_config_id, search_space_id=search_space_id + ) + if llm_load_error: + yield emit_stream_error( + message=llm_load_error, + error_kind="server_error", + error_code="SERVER_ERROR", + ) + yield streaming_service.format_done() + return + _perf_log.info( + "[stream_resume] LLM config loaded in %.3fs", time.perf_counter() - _t0 + ) + + if needs_premium_quota(agent_config, user_id): + premium_reservation = await reserve_premium( + agent_config=agent_config, user_id=user_id # type: ignore[arg-type] + ) + if not premium_reservation.allowed: + ot.add_event( + "quota.denied", {"quota.code": "PREMIUM_QUOTA_EXHAUSTED"} + ) + if requested_llm_config_id == 0: + try: + pinned_fb = await resolve_or_get_pinned_llm_config_id( + session, + thread_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + selected_llm_config_id=0, + force_repin_free=True, + ) + llm_config_id = pinned_fb.resolved_llm_config_id + ot.add_event( + "model.repin", + { + "repin.reason": "premium_quota_exhausted", + "repin.to_config_id": llm_config_id, + }, + ) + except ValueError as pin_error: + yield emit_stream_error( + message=str(pin_error), + error_kind="server_error", + error_code="SERVER_ERROR", + ) + yield streaming_service.format_done() + return + llm, agent_config, llm_load_error = await load_llm_bundle( + session, + config_id=llm_config_id, + search_space_id=search_space_id, + ) + if llm_load_error: + yield emit_stream_error( + message=llm_load_error, + error_kind="server_error", + error_code="SERVER_ERROR", + ) + yield streaming_service.format_done() + return + premium_reservation = None + from app.tasks.chat.streaming.errors.classifier import ( + log_chat_stream_error, + ) + + log_chat_stream_error( + flow="resume", + error_kind="premium_quota_exhausted", + error_code="PREMIUM_QUOTA_EXHAUSTED", + severity="info", + is_expected=True, + request_id=request_id, + thread_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + message=( + "Premium quota exhausted on pinned model; " + "auto-fallback switched to a free model" + ), + extra={ + "fallback_config_id": llm_config_id, + "auto_fallback": True, + }, + ) + else: + yield emit_stream_error( + message=( + "Buy more tokens to continue with this model, or " + "switch to a free model" + ), + error_kind="premium_quota_exhausted", + error_code="PREMIUM_QUOTA_EXHAUSTED", + severity="info", + is_expected=True, + extra={ + "resolved_config_id": llm_config_id, + "auto_fallback": False, + }, + ) + yield streaming_service.format_done() + return + + if not llm: + yield emit_stream_error( + message="Failed to create LLM instance", + error_kind="server_error", + error_code="SERVER_ERROR", + ) + yield streaming_service.format_done() + return + + # --- Pre-stream setup --- + + _t0 = time.perf_counter() + connector_service, firecrawl_api_key = await setup_connector_and_firecrawl( + session, search_space_id=search_space_id + ) + _perf_log.info( + "[stream_resume] Connector service + firecrawl key in %.3fs", + time.perf_counter() - _t0, + ) + + _t0 = time.perf_counter() + checkpointer = await get_chat_checkpointer() + _perf_log.info( + "[stream_resume] Checkpointer ready in %.3fs", time.perf_counter() - _t0 + ) + + visibility = thread_visibility or ChatVisibility.PRIVATE + use_multi_agent = bool(_app_config.MULTI_AGENT_CHAT_ENABLED) + chat_agent_mode = "multi" if use_multi_agent else "single" + set_agent_mode(chat_span, chat_agent_mode) + + _t0 = time.perf_counter() + agent_factory = ( + create_multi_agent_chat_deep_agent + if use_multi_agent + else create_surfsense_deep_agent + ) + agent = await build_main_agent_for_thread( + agent_factory, + llm=llm, + search_space_id=search_space_id, + db_session=session, + connector_service=connector_service, + checkpointer=checkpointer, + user_id=user_id, + thread_id=chat_id, + agent_config=agent_config, + firecrawl_api_key=firecrawl_api_key, + thread_visibility=visibility, + filesystem_selection=filesystem_selection, + disabled_tools=disabled_tools, + ) + _perf_log.info( + "[stream_resume] Agent created in %.3fs", time.perf_counter() - _t0 + ) + + # Release the transaction before streaming (same rationale as stream_new_chat). + await session.commit() + session.expunge_all() + + _perf_log.info( + "[stream_resume] Total pre-stream setup in %.3fs (chat_id=%s)", + time.perf_counter() - _t_total, + chat_id, + ) + + # --- Resume routing --- + + from langgraph.types import Command + + routing = await build_resume_routing( + agent, chat_id=chat_id, decisions=decisions + ) + + config = { + "configurable": { + "thread_id": str(chat_id), + "request_id": request_id or "unknown", + "turn_id": stream_result.turn_id, + # Per-``tool_call_id`` resume slices read by + # ``SurfSenseCheckpointedSubAgentMiddleware``. Parallel + # siblings each pop their own entry, so they never race. + "surfsense_resume_value": routing.routed_resume_value, + }, + # Same rationale as ``stream_new_chat``: effectively uncapped to + # mirror the agent default and OpenCode's session loop. Doom-loop + # / call-limit middleware enforce the real ceiling. + "recursion_limit": 10_000, + } + + # --- First SSE frames --- + + for sse in iter_initial_frames(streaming_service, turn_id=stream_result.turn_id): + yield sse + + # --- Assistant-shell persistence + id frame --- + + assistant_message_id = await persist_resume_assistant_shell( + chat_id=chat_id, + user_id=user_id, + turn_id=stream_result.turn_id, + ) + if assistant_message_id is None: + yield emit_stream_error( + message=( + "We couldn't initialize the assistant message. Please try again." + ), + error_kind="server_error", + error_code="MESSAGE_PERSIST_FAILED", + ) + for sse in iter_final_frames(streaming_service): + yield sse + return + + yield streaming_service.format_data( + "assistant-message-id", + {"message_id": assistant_message_id, "turn_id": stream_result.turn_id}, + ) + + stream_result.assistant_message_id = assistant_message_id + stream_result.content_builder = AssistantContentBuilder() + + runtime_context = build_resume_chat_runtime_context( + search_space_id=search_space_id, + request_id=request_id, + turn_id=stream_result.turn_id, + ) + + # --- Stream loop --- + + _t_stream_start = time.perf_counter() + runtime_rate_limit_recovered = False + + def _on_first_event() -> None: + _perf_log.info( + "[stream_resume] First agent event in %.3fs (stream), %.3fs (total) (chat_id=%s)", + time.perf_counter() - _t_stream_start, + time.perf_counter() - _t_total, + chat_id, + ) + + async def _recover(exc: BaseException, first_event_seen: bool): + nonlocal llm_config_id, llm, agent_config, runtime_rate_limit_recovered + if not can_recover_provider_rate_limit( + exc, + first_event_seen=first_event_seen, + runtime_rate_limit_recovered=runtime_rate_limit_recovered, + requested_llm_config_id=requested_llm_config_id, + current_llm_config_id=llm_config_id, + ): + return None + runtime_rate_limit_recovered = True + previous_config_id = llm_config_id + llm_config_id = await reroute_to_next_auto_pin( + session, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + current_llm_config_id=llm_config_id, + requires_image_input=False, + ) + new_llm, new_agent_config, llm_load_err = await load_llm_bundle( + session, config_id=llm_config_id, search_space_id=search_space_id + ) + if llm_load_err: + return None + llm = new_llm + agent_config = new_agent_config + + _t_rebuild = time.perf_counter() + new_agent = await build_main_agent_for_thread( + agent_factory, + llm=llm, + search_space_id=search_space_id, + db_session=session, + connector_service=connector_service, + checkpointer=checkpointer, + user_id=user_id, + thread_id=chat_id, + agent_config=agent_config, + firecrawl_api_key=firecrawl_api_key, + thread_visibility=visibility, + filesystem_selection=filesystem_selection, + disabled_tools=disabled_tools, + ) + _perf_log.info( + "[stream_resume] Runtime rate-limit recovery repinned " + "config_id=%s -> %s and rebuilt agent in %.3fs", + previous_config_id, + llm_config_id, + time.perf_counter() - _t_rebuild, + ) + log_rate_limit_recovered( + flow="resume", + request_id=request_id, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + previous_config_id=previous_config_id, + new_config_id=llm_config_id, + ) + return new_agent + + async for sse in run_stream_loop( + agent=agent, + streaming_service=streaming_service, + config=config, + input_data=Command(resume=routing.lg_resume_map), + stream_result=stream_result, + step_prefix=resume_step_prefix(stream_result.turn_id), + fallback_commit_search_space_id=search_space_id, + fallback_commit_created_by_id=user_id, + fallback_commit_filesystem_mode=( + filesystem_selection.mode if filesystem_selection else FilesystemMode.CLOUD + ), + fallback_commit_thread_id=chat_id, + runtime_context=runtime_context, + content_builder=stream_result.content_builder, + recover=_recover, + on_first_event=_on_first_event, + ): + yield sse + + _perf_log.info( + "[stream_resume] Agent stream completed in %.3fs (chat_id=%s)", + time.perf_counter() - _t_stream_start, + chat_id, + ) + + # --- Finalize --- + + if stream_result.is_interrupted: + ot.add_event("chat.interrupted", {"chat.flow": "resume"}) + for sse in iter_token_usage_frame( + streaming_service, + accumulator=accumulator, + log_label="interrupted resume_chat", + ): + yield sse + yield streaming_service.format_finish_step() + yield streaming_service.format_finish() + yield streaming_service.format_done() + return + + if premium_reservation is not None and user_id: + await finalize_premium( + reservation=premium_reservation, + user_id=user_id, + accumulator=accumulator, + ) + premium_reservation = None + + for sse in iter_token_usage_frame( + streaming_service, accumulator=accumulator, log_label="normal resume_chat" + ): + yield sse + + for sse in iter_final_frames(streaming_service): + yield sse + + except Exception as exc: + frames, summary = handle_terminal_exception( + exc, + flow="resume", + flow_label="resume", + log_prefix="stream_resume_chat", + streaming_service=streaming_service, + request_id=request_id, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + chat_span=chat_span, + ) + if summary["busy_error_raised"]: + busy_error_raised = True + chat_outcome = summary["chat_outcome"] + chat_error_category = summary["chat_error_category"] + for sse in frames: + yield sse + + finally: + with anyio.CancelScope(shield=True): + end_turn(str(chat_id)) + + if premium_reservation is not None and user_id: + await release_premium( + reservation=premium_reservation, user_id=user_id + ) + + await close_session_and_clear_ai_responding(session, chat_id) + + await finalize_assistant_message( + stream_result=stream_result, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + accumulator=accumulator, + log_prefix="stream_resume", + ) + + # Release the lock from the original interrupted turn or any + # re-interrupt/bailout. Skip on ``BusyError`` (lock not held here). + if not busy_error_raised: + with contextlib.suppress(Exception): + end_turn(str(chat_id)) + _perf_log.info( + "[stream_resume] end_turn cleanup (chat_id=%s)", chat_id + ) + + agent = llm = connector_service = None # noqa: F841 + stream_result = None # noqa: F841 + session = None # noqa: F841 + + run_gc_pass(log_prefix="stream_resume", chat_id=chat_id) + close_chat_request_span( + span_cm=chat_span_cm, + span=chat_span, + chat_outcome=chat_outcome, + chat_agent_mode=chat_agent_mode, + flow="resume", + chat_error_category=chat_error_category, + duration_seconds=time.perf_counter() - _t_total, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/resume_routing.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/resume_routing.py new file mode 100644 index 000000000..300fbc9bd --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/resume_routing.py @@ -0,0 +1,65 @@ +"""Route a flat ``decisions`` list back to the right paused subagent. + +Each pending interrupt is stamped with its originating ``tool_call_id`` (see +``checkpointed_subagent_middleware.propagation``) so the resume slicer can +re-target each ``HumanReview`` decision at the right ``tool_call_id``. + +LangGraph rejects scalar ``Command(resume=...)`` when multiple interrupts are +pending (parallel HITL); the mapped form works for the single-pause case too, +so we always use it. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any + +from app.utils.perf import get_perf_logger + +_perf_log = get_perf_logger() +logger = logging.getLogger(__name__) + + +@dataclass +class ResumeRoutingPayload: + """Resolved per-``tool_call_id`` resume slices + the lg-shaped resume map.""" + + routed_resume_value: dict[str, Any] + lg_resume_map: dict[str, Any] + + +async def build_resume_routing( + agent: Any, + *, + chat_id: int, + decisions: list[dict], +) -> ResumeRoutingPayload: + """Read parent_state, collect pending tool-calls, slice decisions, build map. + + The middleware reads its per-``tool_call_id`` resume slice from the + ``surfsense_resume_value`` configurable; parallel siblings each pop their + own entry so they never race. + """ + from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume_routing import ( + build_lg_resume_map, + collect_pending_tool_calls, + slice_decisions_by_tool_call, + ) + + parent_state = await agent.aget_state( + {"configurable": {"thread_id": str(chat_id)}} + ) + pending = collect_pending_tool_calls(parent_state) + _perf_log.info( + "[hitl_route] resume_entry chat_id=%s decisions=%d pending_subagents=%d", + chat_id, + len(decisions), + len(pending), + ) + routed_resume_value = slice_decisions_by_tool_call(decisions, pending) + lg_resume_map = build_lg_resume_map(parent_state, routed_resume_value) + return ResumeRoutingPayload( + routed_resume_value=routed_resume_value, + lg_resume_map=lg_resume_map, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/runtime_context.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/runtime_context.py new file mode 100644 index 000000000..59d5d8ca7 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/runtime_context.py @@ -0,0 +1,23 @@ +"""Build the per-invocation ``SurfSenseContextSchema`` for a resume turn. + +Resume doesn't carry new ``mentioned_document_ids`` (those are seeded by the +original turn). We still build the context so future middleware extensions +can rely on ``runtime.context`` always being populated. +""" + +from __future__ import annotations + +from app.agents.new_chat.context import SurfSenseContextSchema + + +def build_resume_chat_runtime_context( + *, + search_space_id: int, + request_id: str | None, + turn_id: str, +) -> SurfSenseContextSchema: + return SurfSenseContextSchema( + search_space_id=search_space_id, + request_id=request_id, + turn_id=turn_id, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/__init__.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/__init__.py new file mode 100644 index 000000000..b65acc43c --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/__init__.py @@ -0,0 +1,3 @@ +"""Building blocks shared by ``new_chat`` and ``resume_chat`` orchestrators.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/assistant_finalize.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/assistant_finalize.py new file mode 100644 index 000000000..d16f81ac7 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/assistant_finalize.py @@ -0,0 +1,109 @@ +"""Server-side assistant-message + token_usage finalization. + +Runs inside the streaming flow's ``finally`` block, after the main session has +been closed (uses its own shielded session, so we don't fight the same DB +connection). + +Idempotent against the legacy frontend ``appendMessage`` recovery branch: + + * the assistant row was already INSERTed by ``persist_assistant_shell`` + earlier in the turn, so this just UPDATEs it with the rich + ``ContentPart[]`` projection from the builder. + * ``token_usage`` uses ``INSERT ... ON CONFLICT DO NOTHING`` against the + partial unique index from migration 142, so a racing append_message + recovery branch can never double-write. + +``mark_interrupted`` closes any open text/reasoning blocks and flips running +tool-calls (no result) to ``state=aborted`` so the persisted JSONB reflects a +coherent end-state even on client disconnect. + +Never raises (best-effort, logs only). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from app.tasks.chat.streaming.shared.stream_result import StreamResult +from app.utils.perf import get_perf_logger + +if TYPE_CHECKING: + from app.services.token_tracking_service import TokenAccumulator + +_perf_log = get_perf_logger() + + +async def finalize_assistant_message( + *, + stream_result: StreamResult | None, + chat_id: int, + search_space_id: int, + user_id: str | None, + accumulator: TokenAccumulator, + log_prefix: str, +) -> None: + """Snapshot the content builder and persist the final assistant payload. + + No-op when ``stream_result`` was never populated, the turn never reached + ``persist_assistant_shell`` (no ``assistant_message_id``), or the turn id + was never assigned. + """ + if not ( + stream_result + and stream_result.turn_id + and stream_result.assistant_message_id + ): + return + + from app.tasks.chat.persistence import finalize_assistant_turn + + builder_stats: dict[str, int] | None = None + if stream_result.content_builder is not None: + stream_result.content_builder.mark_interrupted() + # Snapshot stats BEFORE ``snapshot()`` deepcopies so the perf log + # records the actual finalised payload (post-mark_interrupted), not + # the live-mutating builder state. + builder_stats = stream_result.content_builder.stats() + content_payload = stream_result.content_builder.snapshot() + else: + # Defensive fallback — we always set the builder alongside + # ``assistant_message_id`` in the orchestrator, so this branch only + # fires if a future refactor ever decouples them. Persist whatever + # accumulated text we captured so the row at least renders. + content_payload = [ + { + "type": "text", + "text": stream_result.accumulated_text or "", + } + ] + + if builder_stats is not None: + _perf_log.info( + "[%s] finalize_payload chat_id=%s " + "message_id=%s parts=%d bytes=%d text=%d " + "reasoning=%d tool_calls=%d " + "tool_calls_completed=%d tool_calls_aborted=%d " + "thinking_step_parts=%d step_separators=%d", + log_prefix, + chat_id, + stream_result.assistant_message_id, + builder_stats["parts"], + builder_stats["bytes"], + builder_stats["text"], + builder_stats["reasoning"], + builder_stats["tool_calls"], + builder_stats["tool_calls_completed"], + builder_stats["tool_calls_aborted"], + builder_stats["thinking_step_parts"], + builder_stats["step_separators"], + ) + + await finalize_assistant_turn( + message_id=stream_result.assistant_message_id, + chat_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + turn_id=stream_result.turn_id, + content=content_payload, + accumulator=accumulator, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/finalize_emit.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/finalize_emit.py new file mode 100644 index 000000000..e5de3f6a4 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/finalize_emit.py @@ -0,0 +1,54 @@ +"""Emit the per-turn token-usage SSE frame from the accumulator. + +``per_message_summary()`` returns ``None`` when the turn made no chargeable +LLM calls (e.g. interrupt-on-input). In that case we skip the frame; the +frontend has no usage to render. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from app.services.new_streaming_service import VercelStreamingService +from app.utils.perf import get_perf_logger + +if TYPE_CHECKING: + from app.services.token_tracking_service import TokenAccumulator + +_perf_log = get_perf_logger() +logger = logging.getLogger(__name__) + + +def iter_token_usage_frame( + streaming_service: VercelStreamingService, + *, + accumulator: TokenAccumulator, + log_label: str, +): + """Yield zero or one ``data: token-usage`` SSE frame. + + Side effect: logs a one-line ``[token_usage] {log_label}: ...`` summary so + cost analysis can grep call/total/cost across all flows. + """ + usage_summary = accumulator.per_message_summary() + _perf_log.info( + "[token_usage] %s: calls=%d total=%d cost_micros=%d summary=%s", + log_label, + len(accumulator.calls), + accumulator.grand_total, + accumulator.total_cost_micros, + usage_summary, + ) + if usage_summary: + yield streaming_service.format_data( + "token-usage", + { + "usage": usage_summary, + "prompt_tokens": accumulator.total_prompt_tokens, + "completion_tokens": accumulator.total_completion_tokens, + "total_tokens": accumulator.grand_total, + "cost_micros": accumulator.total_cost_micros, + "call_details": accumulator.serialized_calls(), + }, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/finally_cleanup.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/finally_cleanup.py new file mode 100644 index 000000000..8d425402f --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/finally_cleanup.py @@ -0,0 +1,69 @@ +"""Shared finally-block helpers: session close, GC pass, native-heap trim. + +These are called from inside an ``anyio.CancelScope(shield=True)`` block in +each flow's ``finally`` (Starlette's BaseHTTPMiddleware cancels the scope on +client disconnect; without the shield the very first ``await`` would raise +``CancelledError`` and the rest of cleanup — including ``session.close()`` — +would never run). +""" + +from __future__ import annotations + +import contextlib +import gc +import logging + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import shielded_async_session +from app.services.chat_session_state_service import clear_ai_responding +from app.utils.perf import get_perf_logger, log_system_snapshot, trim_native_heap + +_perf_log = get_perf_logger() +logger = logging.getLogger(__name__) + + +async def close_session_and_clear_ai_responding( + session: AsyncSession, chat_id: int +) -> None: + """Rollback + clear AI-responding flag + expunge_all + close. + + On rollback failure we fall back to a fresh shielded session for the flag + clear so a UI is never stuck on "AI is responding…" after a crash. + """ + try: + await session.rollback() + await clear_ai_responding(session, chat_id) + except Exception: + try: + async with shielded_async_session() as fresh_session: + await clear_ai_responding(fresh_session, chat_id) + except Exception: + logger.warning( + "Failed to clear AI responding state for thread %s", chat_id + ) + + with contextlib.suppress(Exception): + session.expunge_all() + + with contextlib.suppress(Exception): + await session.close() + + +def run_gc_pass(*, log_prefix: str, chat_id: int) -> None: + """One full gen0/1/2 pass + native-heap trim + END system snapshot. + + Breaking circular refs held by the agent graph, tools, and LLM wrappers + needs to happen in the caller (set the locals to ``None``) — this just + runs the collector and logs how many objects came back. + """ + collected = gc.collect(0) + gc.collect(1) + gc.collect(2) + if collected: + _perf_log.info( + "[%s] gc.collect() reclaimed %d objects (chat_id=%s)", + log_prefix, + collected, + chat_id, + ) + trim_native_heap() + log_system_snapshot(f"{log_prefix}_END") diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/first_frames.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/first_frames.py new file mode 100644 index 000000000..5e568b1e8 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/first_frames.py @@ -0,0 +1,40 @@ +"""Initial SSE frames every flow emits right after pre-stream setup. + +Order matters: ``message_start`` opens the assistant message, ``start_step`` +opens the first thinking step, ``turn-info`` lets the frontend stamp the +correlation id onto the in-flight message, and ``turn-status: busy`` flips the +UI into the streaming state. +""" + +from __future__ import annotations + +from collections.abc import Iterator + +from app.services.new_streaming_service import VercelStreamingService + + +def iter_initial_frames( + streaming_service: VercelStreamingService, + *, + turn_id: str, +) -> Iterator[str]: + """Yield the four canonical opening frames in order. + + ``turn-info`` carries ``chat_turn_id`` so even pure-text turns (which + never produce a tool / action-log event) still teach the frontend the + turn correlation id used for ``appendMessage`` durable storage. + """ + yield streaming_service.format_message_start() + yield streaming_service.format_start_step() + yield streaming_service.format_data("turn-info", {"chat_turn_id": turn_id}) + yield streaming_service.format_data("turn-status", {"status": "busy"}) + + +def iter_final_frames( + streaming_service: VercelStreamingService, +) -> Iterator[str]: + """Yield ``turn-status: idle`` plus the finish/done trailer in order.""" + yield streaming_service.format_data("turn-status", {"status": "idle"}) + yield streaming_service.format_finish_step() + yield streaming_service.format_finish() + yield streaming_service.format_done() diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py new file mode 100644 index 000000000..2f334114c --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py @@ -0,0 +1,57 @@ +"""Load an LLM + AgentConfig bundle for a given config id. + +Handles both code paths uniformly: +- ``config_id >= 0`` → database-backed ``NewLLMConfig`` row (per-user/per-space). +- ``config_id < 0`` → YAML-defined global LLM config (built-in defaults). + +Returns ``(llm, agent_config, error_message)``; on success ``error_message`` is +``None``. The caller emits the friendly SSE error frame. +""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.agents.new_chat.llm_config import ( + AgentConfig, + create_chat_litellm_from_agent_config, + create_chat_litellm_from_config, + load_agent_config, + load_global_llm_config_by_id, +) + + +async def load_llm_bundle( + session: AsyncSession, + *, + config_id: int, + search_space_id: int, +) -> tuple[Any, AgentConfig | None, str | None]: + if config_id >= 0: + loaded_agent_config = await load_agent_config( + session=session, + config_id=config_id, + search_space_id=search_space_id, + ) + if not loaded_agent_config: + return ( + None, + None, + f"Failed to load NewLLMConfig with id {config_id}", + ) + return ( + create_chat_litellm_from_agent_config(loaded_agent_config), + loaded_agent_config, + None, + ) + + loaded_llm_config = load_global_llm_config_by_id(config_id) + if not loaded_llm_config: + return None, None, f"Failed to load LLM config with id {config_id}" + return ( + create_chat_litellm_from_config(loaded_llm_config), + AgentConfig.from_yaml_config(loaded_llm_config), + None, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py new file mode 100644 index 000000000..ec92306dd --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py @@ -0,0 +1,40 @@ +"""Pre-stream setup: connector service, firecrawl key, checkpointer.""" + +from __future__ import annotations + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.agents.new_chat.checkpointer import get_checkpointer +from app.db import SearchSourceConnectorType +from app.services.connector_service import ConnectorService + + +async def setup_connector_and_firecrawl( + session: AsyncSession, + *, + search_space_id: int, +) -> tuple[ConnectorService, str | None]: + """Build the per-turn connector service and pull the firecrawl API key. + + Returns ``(connector_service, firecrawl_api_key)``. ``firecrawl_api_key`` is + ``None`` when no web-crawler connector is configured (the agent simply + skips firecrawl-backed tools in that case). + """ + connector_service = ConnectorService(session, search_space_id=search_space_id) + firecrawl_api_key: str | None = None + webcrawler_connector = await connector_service.get_connector_by_type( + SearchSourceConnectorType.WEBCRAWLER_CONNECTOR, search_space_id + ) + if webcrawler_connector and webcrawler_connector.config: + firecrawl_api_key = webcrawler_connector.config.get("FIRECRAWL_API_KEY") + return connector_service, firecrawl_api_key + + +async def get_chat_checkpointer(): + """Resolve the PostgreSQL checkpointer for persistent conversation memory. + + Thin wrapper around ``app.agents.new_chat.checkpointer.get_checkpointer`` so + flow orchestrators can rely on a streaming-local symbol and we have a hook + point if the checkpointer source ever needs to vary per flow. + """ + return await get_checkpointer() diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py new file mode 100644 index 000000000..0ec40d275 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py @@ -0,0 +1,132 @@ +"""Premium credit (USD micro-units) reserve / finalize / release lifecycle. + +Both ``stream_new_chat`` and ``stream_resume_chat`` reserve premium credits up +front (so a single LLM call can't run away with the budget), then finalize the +actual provider cost reported by LiteLLM when the turn completes successfully, +or release the reservation on the cancellation / interrupted-without-finalize +paths. + +State is held by the orchestrator as a simple ``PremiumReservation`` tuple +so reservation, fallback-on-denied, finalize, and release can all be reasoned +about from one place. +""" + +from __future__ import annotations + +import logging +import uuid as _uuid +from dataclasses import dataclass +from typing import TYPE_CHECKING +from uuid import UUID + +from app.agents.new_chat.llm_config import AgentConfig +from app.db import shielded_async_session + +if TYPE_CHECKING: + from app.services.token_tracking_service import TokenAccumulator + + +@dataclass +class PremiumReservation: + """Active premium-credit reservation for one turn. + + ``request_id`` is the per-reservation idempotency key (also passed to + ``finalize``/``release`` so racing branches resolve to the same row). + ``reserved_micros`` is the up-front estimate; ``finalize`` debits the + actual cost, ``release`` returns it untouched. + """ + + request_id: str + reserved_micros: int + allowed: bool + + +def needs_premium_quota( + agent_config: AgentConfig | None, user_id: str | None +) -> bool: + return bool(agent_config is not None and user_id and agent_config.is_premium) + + +async def reserve_premium( + *, + agent_config: AgentConfig, + user_id: str, +) -> PremiumReservation: + """Reserve estimated micros up front; returns the reservation handle.""" + from app.services.token_quota_service import ( + TokenQuotaService, + estimate_call_reserve_micros, + ) + + request_id = _uuid.uuid4().hex[:16] + litellm_params = agent_config.litellm_params or {} + base_model = ( + litellm_params.get("base_model") if isinstance(litellm_params, dict) else None + ) or agent_config.model_name or "" + reserve_amount_micros = estimate_call_reserve_micros( + base_model=base_model, + quota_reserve_tokens=agent_config.quota_reserve_tokens, + ) + async with shielded_async_session() as quota_session: + quota_result = await TokenQuotaService.premium_reserve( + db_session=quota_session, + user_id=UUID(user_id), + request_id=request_id, + reserve_micros=reserve_amount_micros, + ) + return PremiumReservation( + request_id=request_id, + reserved_micros=reserve_amount_micros, + allowed=quota_result.allowed, + ) + + +async def finalize_premium( + *, + reservation: PremiumReservation, + user_id: str, + accumulator: TokenAccumulator, +) -> None: + """Finalize debit using the actual provider cost reported by LiteLLM. + + Best-effort: failures here must not bubble up to the SSE stream — the user + has already received their tokens; we log and move on. + """ + try: + from app.services.token_quota_service import TokenQuotaService + + async with shielded_async_session() as quota_session: + await TokenQuotaService.premium_finalize( + db_session=quota_session, + user_id=UUID(user_id), + request_id=reservation.request_id, + actual_micros=accumulator.total_cost_micros, + reserved_micros=reservation.reserved_micros, + ) + except Exception: + logging.getLogger(__name__).warning( + "Failed to finalize premium quota for user %s", + user_id, + exc_info=True, + ) + + +async def release_premium( + *, + reservation: PremiumReservation, + user_id: str, +) -> None: + """Release the reservation on cancellation paths; never raises.""" + try: + from app.services.token_quota_service import TokenQuotaService + + async with shielded_async_session() as quota_session: + await TokenQuotaService.premium_release( + db_session=quota_session, + user_id=UUID(user_id), + reserved_micros=reservation.reserved_micros, + ) + except Exception: + logging.getLogger(__name__).warning( + "Failed to release premium quota for user %s", user_id + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/rate_limit_recovery.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/rate_limit_recovery.py new file mode 100644 index 000000000..6b3857594 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/rate_limit_recovery.py @@ -0,0 +1,129 @@ +"""Shared steps for the in-stream provider rate-limit recovery loop. + +Both flows wrap ``run_stream_loop`` with a flow-specific ``recover`` closure; +the *guard*, the *auto-pin reroute*, and the *post-recovery telemetry* are the +same on both sides and live here so behaviour can't drift. + +The orchestrator owns the parts that genuinely diverge: + + * cancelling the title task (new_chat only), + * passing ``mentioned_document_ids`` to ``build_main_agent_for_thread``, + * the log prefix (``stream_new_chat`` vs ``stream_resume``). +""" + +from __future__ import annotations + +from typing import Literal + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.agents.new_chat.middleware.busy_mutex import end_turn +from app.observability import otel as ot +from app.services.auto_model_pin_service import ( + mark_runtime_cooldown, + resolve_or_get_pinned_llm_config_id, +) +from app.tasks.chat.streaming.errors.classifier import ( + is_provider_rate_limited, + log_chat_stream_error, +) + + +def can_recover_provider_rate_limit( + exc: BaseException, + *, + first_event_seen: bool, + runtime_rate_limit_recovered: bool, + requested_llm_config_id: int, + current_llm_config_id: int, +) -> bool: + """Guard: only the first auto-pin → provider-rate-limited failure recovers. + + All conditions must hold: + + * ``runtime_rate_limit_recovered is False`` — at most one recovery per turn. + * ``requested_llm_config_id == 0`` — caller opted into auto-pin (id=0). + * ``current_llm_config_id < 0`` — currently on a YAML config (the only + kind the auto-pin pool draws from). + * ``first_event_seen is False`` — we haven't sent any SSE to the user yet, + so a silent rebuild + retry is invisible. + * The exception is provider-side rate-limited (HTTP 429 or known shape). + """ + return ( + not runtime_rate_limit_recovered + and requested_llm_config_id == 0 + and current_llm_config_id < 0 + and not first_event_seen + and is_provider_rate_limited(exc) + ) + + +async def reroute_to_next_auto_pin( + session: AsyncSession, + *, + chat_id: int, + search_space_id: int, + user_id: str | None, + current_llm_config_id: int, + requires_image_input: bool, +) -> int: + """Release lock, cool down the failing config, pick a new auto-pin id. + + Returns the new ``llm_config_id``. ``end_turn`` is called because the failed + attempt may still hold the per-thread busy mutex (middleware teardown can + lag behind raised provider errors) — the same-request retry would otherwise + bounce on ``BusyError``. + """ + end_turn(str(chat_id)) + mark_runtime_cooldown(current_llm_config_id, reason="provider_rate_limited") + pinned = await resolve_or_get_pinned_llm_config_id( + session, + thread_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + selected_llm_config_id=0, + exclude_config_ids={current_llm_config_id}, + requires_image_input=requires_image_input, + ) + return pinned.resolved_llm_config_id + + +def log_rate_limit_recovered( + *, + flow: Literal["new", "regenerate", "resume"], + request_id: str | None, + chat_id: int, + search_space_id: int, + user_id: str | None, + previous_config_id: int, + new_config_id: int, +) -> None: + """Emit the OTEL event + structured ``[chat_stream_error]`` log line.""" + ot.add_event( + "chat.rate_limit.recovered", + { + "recovery.reason": "provider_rate_limited", + "recovery.previous_config_id": previous_config_id, + "recovery.fallback_config_id": new_config_id, + }, + ) + log_chat_stream_error( + flow=flow, + error_kind="rate_limited", + error_code="RATE_LIMITED", + severity="info", + is_expected=True, + request_id=request_id, + thread_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + message=( + "Auto-pinned model hit runtime rate limit; switched to " + "another eligible model and retried." + ), + extra={ + "auto_runtime_recover": True, + "previous_config_id": previous_config_id, + "fallback_config_id": new_config_id, + }, + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/span.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/span.py new file mode 100644 index 000000000..1e5169af1 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/span.py @@ -0,0 +1,80 @@ +"""OpenTelemetry chat-request span wrapper for streaming flows.""" + +from __future__ import annotations + +import contextlib +import sys +from typing import Any, Literal + +from app.observability import metrics as ot_metrics +from app.observability import otel as ot + + +def open_chat_request_span( + *, + chat_id: int, + search_space_id: int, + flow: Literal["new", "regenerate", "resume"], + request_id: str | None, + turn_id: str, + filesystem_mode: str, + client_platform: str, + agent_mode: str, +) -> tuple[Any, Any]: + """Open the per-request span; returns ``(span_cm, span)`` for finally-close.""" + span_cm = ot.chat_request_span( + chat_id=chat_id, + search_space_id=search_space_id, + flow=flow, + request_id=request_id, + turn_id=turn_id, + filesystem_mode=filesystem_mode, + client_platform=client_platform, + agent_mode=agent_mode, + ) + span = span_cm.__enter__() + return span_cm, span + + +def set_agent_mode(span: Any, agent_mode: str) -> None: + """Tag the span with the resolved agent mode (single / multi).""" + with contextlib.suppress(Exception): + span.set_attribute("agent.mode", agent_mode) + + +def close_chat_request_span( + *, + span_cm: Any, + span: Any, + chat_outcome: str, + chat_agent_mode: str, + flow: Literal["new", "regenerate", "resume"], + chat_error_category: str | None, + duration_seconds: float, +) -> None: + """Record metrics + close the span. Swallows errors (finally-block context).""" + with contextlib.suppress(Exception): + span.set_attribute("chat.outcome", chat_outcome) + ot_metrics.record_chat_request_duration( + duration_seconds * 1000, + flow=flow, + outcome=chat_outcome, + agent_mode=chat_agent_mode, + ) + ot_metrics.record_chat_request_outcome( + flow=flow, + outcome=chat_outcome, + agent_mode=chat_agent_mode, + error_category=chat_error_category, + ) + span_cm.__exit__(*sys.exc_info()) + + +def record_outcome_attrs( + span: Any, *, chat_outcome: str, chat_error_category: str | None +) -> None: + """Stamp outcome + error.category on the span (used in the except branch).""" + with contextlib.suppress(Exception): + span.set_attribute("chat.outcome", chat_outcome) + if chat_error_category is not None: + span.set_attribute("error.category", chat_error_category) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/stream_loop.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/stream_loop.py new file mode 100644 index 000000000..6cf0df855 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/stream_loop.py @@ -0,0 +1,85 @@ +"""Drive ``stream_agent_events`` with in-stream rate-limit recovery. + +Both ``stream_new_chat`` and ``stream_resume_chat`` wrap the agent event loop +in a ``while True`` that catches the *first* provider rate-limit error +(``can_runtime_recover``) before any SSE event reaches the user, rebuilds the +agent on an alternative auto-pin, and retries the turn. + +The recovery callback is flow-specific (different ``mentioned_document_ids`` +contract, different logging label, etc.) — this module owns the loop shape, +the caller owns the rebuild. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator, Awaitable, Callable +from typing import Any + +from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.services.new_streaming_service import VercelStreamingService +from app.tasks.chat.streaming.agent.event_loop import stream_agent_events +from app.tasks.chat.streaming.shared.stream_result import StreamResult + +# Returns the rebuilt agent on a successful recovery, or ``None`` to re-raise +# the original exception (and let the orchestrator's terminal-error path +# handle it). +RecoverFn = Callable[[BaseException, bool], Awaitable[Any | None]] + + +async def run_stream_loop( + *, + agent: Any, + streaming_service: VercelStreamingService, + config: dict[str, Any], + input_data: Any, + stream_result: StreamResult, + step_prefix: str = "thinking", + initial_step_id: str | None = None, + initial_step_title: str = "", + initial_step_items: list[str] | None = None, + fallback_commit_search_space_id: int | None, + fallback_commit_created_by_id: str | None, + fallback_commit_filesystem_mode: FilesystemMode, + fallback_commit_thread_id: int | None, + runtime_context: Any, + content_builder: Any | None, + recover: RecoverFn, + on_first_event: Callable[[], None] | None = None, +) -> AsyncGenerator[str, None]: + """Yield SSE frames; rebuild and retry once on a pre-first-event rate limit. + + ``on_first_event`` fires after the first frame is observed (used by both + flows to write a one-time ``First agent event in N.NNNs`` perf line). + """ + first_event_logged = False + while True: + try: + async for sse in stream_agent_events( + agent=agent, + config=config, + input_data=input_data, + streaming_service=streaming_service, + result=stream_result, + step_prefix=step_prefix, + initial_step_id=initial_step_id, + initial_step_title=initial_step_title, + initial_step_items=initial_step_items, + fallback_commit_search_space_id=fallback_commit_search_space_id, + fallback_commit_created_by_id=fallback_commit_created_by_id, + fallback_commit_filesystem_mode=fallback_commit_filesystem_mode, + fallback_commit_thread_id=fallback_commit_thread_id, + runtime_context=runtime_context, + content_builder=content_builder, + ): + if not first_event_logged: + if on_first_event is not None: + on_first_event() + first_event_logged = True + yield sse + return + except Exception as exc: + new_agent = await recover(exc, first_event_logged) + if new_agent is None: + raise + agent = new_agent + continue diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/terminal_error.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/terminal_error.py new file mode 100644 index 000000000..c9db2caf2 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/terminal_error.py @@ -0,0 +1,120 @@ +"""Handle the ``except Exception`` branch of a streaming flow. + +Classifies the exception, records OpenTelemetry attributes, emits one terminal +error SSE frame and the trailing ``turn-status: idle`` + finish/done frames. + +Used by both ``stream_new_chat`` and ``stream_resume_chat``; flow-specific bits +(label, span, BusyError tracking) are passed by the caller. +""" + +from __future__ import annotations + +import logging +import traceback +from collections.abc import Iterator +from typing import Any, Literal + +from app.agents.new_chat.errors import BusyError +from app.observability import metrics as ot_metrics +from app.observability import otel as ot +from app.services.new_streaming_service import VercelStreamingService +from app.tasks.chat.streaming.errors.classifier import classify_stream_exception +from app.tasks.chat.streaming.errors.emitter import emit_stream_terminal_error +from app.tasks.chat.streaming.flows.shared.first_frames import iter_final_frames +from app.tasks.chat.streaming.flows.shared.span import record_outcome_attrs + +logger = logging.getLogger(__name__) + + +def handle_terminal_exception( + exc: Exception, + *, + flow: Literal["new", "regenerate", "resume"], + flow_label: str, + log_prefix: str, + streaming_service: VercelStreamingService, + request_id: str | None, + chat_id: int, + search_space_id: int, + user_id: str | None, + chat_span: Any, +) -> tuple[Iterator[str], dict[str, Any]]: + """Classify, log, and produce the SSE frames for a terminal exception. + + Returns ``(frame_iterator, summary)``. ``summary`` carries:: + + - ``busy_error_raised``: bool — caller must skip the lock-release path + when True (caller never acquired the busy mutex). + - ``chat_outcome``: str — span outcome attribute. + - ``chat_error_category``: str — categorized error label for metrics. + """ + busy_error_raised = isinstance(exc, BusyError) + + ( + error_kind, + error_code, + severity, + is_expected, + user_message, + error_extra, + ) = classify_stream_exception(exc, flow_label=flow_label) + chat_outcome = error_code or error_kind or "error" + chat_error_category = ot_metrics.categorize_exception(exc) + record_outcome_attrs( + chat_span, + chat_outcome=chat_outcome, + chat_error_category=chat_error_category, + ) + with __suppress(): + ot.record_error(chat_span, exc) + error_message = f"Error during {flow_label}: {exc!s}" + # Match the original behavior: log full traceback via ``print`` so it lands + # in stderr regardless of the logger config. + print(f"[{log_prefix}] {error_message}") + print(f"[{log_prefix}] Exception type: {type(exc).__name__}") + print(f"[{log_prefix}] Traceback:\n{traceback.format_exc()}") + + def _iter_frames() -> Iterator[str]: + if error_code == "TURN_CANCELLING": + status_payload: dict[str, Any] = {"status": "cancelling"} + if error_extra: + status_payload.update(error_extra) + yield streaming_service.format_data("turn-status", status_payload) + else: + yield streaming_service.format_data("turn-status", {"status": "busy"}) + + yield emit_stream_terminal_error( + streaming_service=streaming_service, + flow=flow, + request_id=request_id, + thread_id=chat_id, + search_space_id=search_space_id, + user_id=user_id, + message=user_message, + error_kind=error_kind, + error_code=error_code, + severity=severity, + is_expected=is_expected, + extra=error_extra, + ) + yield from iter_final_frames(streaming_service) + + return ( + _iter_frames(), + { + "busy_error_raised": busy_error_raised, + "chat_outcome": chat_outcome, + "chat_error_category": chat_error_category, + }, + ) + + +def __suppress(): + """Local single-use ``contextlib.suppress(Exception)`` factory. + + Inlined here so callers don't import ``contextlib`` just for the + ``record_error`` call site. + """ + import contextlib + + return contextlib.suppress(Exception) diff --git a/surfsense_backend/app/tasks/chat/streaming/shared/__init__.py b/surfsense_backend/app/tasks/chat/streaming/shared/__init__.py new file mode 100644 index 000000000..6c9f1f6b5 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/shared/__init__.py @@ -0,0 +1,15 @@ +"""Shared building blocks used across every streaming flow.""" + +from __future__ import annotations + +from app.tasks.chat.streaming.shared.stream_result import StreamResult +from app.tasks.chat.streaming.shared.utils import ( + resume_step_prefix, + safe_float, +) + +__all__ = [ + "StreamResult", + "resume_step_prefix", + "safe_float", +] diff --git a/surfsense_backend/app/tasks/chat/streaming/shared/stream_result.py b/surfsense_backend/app/tasks/chat/streaming/shared/stream_result.py new file mode 100644 index 000000000..a940e8a9f --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/shared/stream_result.py @@ -0,0 +1,37 @@ +"""Per-turn streaming state shared between the orchestrator and event loop.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class StreamResult: + accumulated_text: str = "" + is_interrupted: bool = False + sandbox_files: list[str] = field(default_factory=list) + request_id: str | None = None + turn_id: str = "" + filesystem_mode: str = "cloud" + client_platform: str = "web" + intent_detected: str = "chat_only" + intent_confidence: float = 0.0 + write_attempted: bool = False + write_succeeded: bool = False + verification_succeeded: bool = False + commit_gate_passed: bool = True + commit_gate_reason: str = "" + # Pre-allocated assistant ``new_chat_messages.id`` for this turn, captured by + # ``persist_assistant_shell`` right after the user row is persisted. ``None`` + # for the legacy/anonymous code paths that don't opt in to server-side + # ``ContentPart[]`` projection. + assistant_message_id: int | None = None + # In-memory mirror of the FE's assistant-ui ``ContentPartsState``, populated + # by the lifecycle methods called from the streaming event loop at each + # ``streaming_service.format_*`` yield site. Snapshot in the streaming + # ``finally`` to produce the rich JSONB persisted by + # ``finalize_assistant_turn``. ``repr=False`` keeps the log-on-error path + # (``StreamResult`` is logged in some error branches) from dumping a + # potentially-large parts list. + content_builder: Any | None = field(default=None, repr=False) diff --git a/surfsense_backend/app/tasks/chat/streaming/shared/utils.py b/surfsense_backend/app/tasks/chat/streaming/shared/utils.py new file mode 100644 index 000000000..fe6901543 --- /dev/null +++ b/surfsense_backend/app/tasks/chat/streaming/shared/utils.py @@ -0,0 +1,27 @@ +"""Small utilities used by streaming orchestrators and phases.""" + +from __future__ import annotations + +from typing import Any + + +def resume_step_prefix(turn_id: str) -> str: + """Per-turn ``step_prefix`` for resume invocations. + + Each ``stream_agent_events`` call constructs a fresh + ``AgentEventRelayState`` with ``thinking_step_counter=0``, so two consecutive + resume turns would otherwise both emit ``thinking-resume-1``, ``-2`` etc. + The frontend rehydrates ``currentThinkingSteps`` from the immediate prior + assistant message at the start of every resume — if the new stream's IDs + collide with the seeded ones, React renders sibling Timeline rows with the + same key. Salting with ``turn_id`` guarantees disjoint IDs across resumes + within one thread. + """ + return f"thinking-resume-{turn_id}" + + +def safe_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 71c53caae..2ed0acca4 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -87,6 +87,7 @@ dependencies = [ "opentelemetry-instrumentation-httpx>=0.61b0", "opentelemetry-instrumentation-celery>=0.61b0", "opentelemetry-instrumentation-logging>=0.61b0", + "croniter>=2.0.0", ] [dependency-groups] diff --git a/surfsense_backend/tests/unit/automations/__init__.py b/surfsense_backend/tests/unit/automations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/actions/__init__.py b/surfsense_backend/tests/unit/automations/actions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/actions/agent_task/__init__.py b/surfsense_backend/tests/unit/automations/actions/agent_task/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/actions/agent_task/test_auto_decide.py b/surfsense_backend/tests/unit/automations/actions/agent_task/test_auto_decide.py new file mode 100644 index 000000000..439f32e41 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/actions/agent_task/test_auto_decide.py @@ -0,0 +1,73 @@ +"""Lock ``build_auto_decisions`` — the HITL auto-approve/reject wire mapper. + +``build_auto_decisions`` walks ``state.interrupts`` (duck-typed) and produces +two parallel resume maps: one keyed by LangGraph ``Interrupt.id`` and one +keyed by ``tool_call_id`` for the subagent middleware bridge. Both carry +the same decision payload. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest + +from app.automations.actions.agent_task.auto_decide import build_auto_decisions + +pytestmark = pytest.mark.unit + + +def _state(interrupts: list[Any]) -> SimpleNamespace: + """Build a duck-typed LangGraph state stub carrying ``interrupts``.""" + return SimpleNamespace(interrupts=interrupts) + + +def _interrupt(*, id_: str, value: Any) -> SimpleNamespace: + """Build a duck-typed interrupt with the canonical ``(id, value)`` shape.""" + return SimpleNamespace(id=id_, value=value) + + +def test_build_auto_decisions_produces_one_decision_per_action_request() -> None: + """An interrupt carrying N ``action_requests`` produces N decisions of + the requested type in both maps. This is the canonical batched-HITL + wire shape — losing a decision would leave a pending action stuck.""" + interrupt = _interrupt( + id_="lg-1", + value={ + "tool_call_id": "tc-1", + "action_requests": [{"id": "a"}, {"id": "b"}], + }, + ) + + lg_map, routed = build_auto_decisions(_state([interrupt]), "approve") + + assert lg_map == {"lg-1": {"decisions": [{"type": "approve"}, {"type": "approve"}]}} + assert routed == {"tc-1": {"decisions": [{"type": "approve"}, {"type": "approve"}]}} + + +def test_build_auto_decisions_defaults_to_one_decision_for_scalar_interrupt() -> None: + """When an interrupt's value has no ``action_requests`` list, the + function defaults to a single decision. Locks compatibility with + older single-action interrupt shapes still emitted by some tools.""" + interrupt = _interrupt(id_="lg-2", value={"tool_call_id": "tc-2"}) + + lg_map, routed = build_auto_decisions(_state([interrupt]), "reject") + + assert lg_map == {"lg-2": {"decisions": [{"type": "reject"}]}} + assert routed == {"tc-2": {"decisions": [{"type": "reject"}]}} + + +def test_build_auto_decisions_skips_interrupts_with_invalid_shape() -> None: + """Interrupts missing the canonical ``(str id, dict value)`` shape are + skipped silently rather than crashing the resume loop. Locks the + resilience contract — a malformed interrupt from a misbehaving tool + shouldn't take down the whole agent_task step.""" + good = _interrupt(id_="lg-good", value={"tool_call_id": "tc-good"}) + bad_value = _interrupt(id_="lg-bad-value", value="not a dict") + bad_id = _interrupt(id_=None, value={"tool_call_id": "tc-bad-id"}) # type: ignore[arg-type] + + lg_map, routed = build_auto_decisions(_state([good, bad_value, bad_id]), "approve") + + assert lg_map == {"lg-good": {"decisions": [{"type": "approve"}]}} + assert routed == {"tc-good": {"decisions": [{"type": "approve"}]}} diff --git a/surfsense_backend/tests/unit/automations/actions/agent_task/test_finalize.py b/surfsense_backend/tests/unit/automations/actions/agent_task/test_finalize.py new file mode 100644 index 000000000..bd49d764c --- /dev/null +++ b/surfsense_backend/tests/unit/automations/actions/agent_task/test_finalize.py @@ -0,0 +1,80 @@ +"""Lock ``extract_final_assistant_message`` — what surfaces in run output. + +Each scenario is one shape the agent runtime is observed to produce. +Locking these means we can refactor the extractor without losing +backwards compatibility with already-stored ``run.output`` payloads. +""" + +from __future__ import annotations + +import pytest +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage + +from app.automations.actions.agent_task.finalize import extract_final_assistant_message + +pytestmark = pytest.mark.unit + + +def test_extract_returns_last_ai_message_string_content() -> None: + """The canonical shape: the agent's final ``AIMessage`` carries a + plain string. That string is returned verbatim, trimmed.""" + result = { + "messages": [ + HumanMessage(content="ask"), + AIMessage(content="the answer"), + ] + } + + assert extract_final_assistant_message(result) == "the answer" + + +def test_extract_concatenates_text_parts_and_skips_non_text_parts() -> None: + """Multi-part AIMessage content (Anthropic / OpenAI list shape) joins + its ``text`` parts in order; non-text parts (tool_use, images, ...) + are skipped. Locks the wire shape used when the model emits tool + calls alongside narrative text in the same turn.""" + result = { + "messages": [ + AIMessage( + content=[ + {"type": "text", "text": "Hello "}, + {"type": "tool_use", "name": "search", "input": {}}, + {"type": "text", "text": "world"}, + ] + ) + ] + } + + assert extract_final_assistant_message(result) == "Hello world" + + +def test_extract_returns_last_ai_message_skipping_tool_messages() -> None: + """When the transcript ends with tool calls and tool results, the + extractor still walks back to the **last** ``AIMessage`` (the agent's + final narrative answer). Locks resilience against trailing + ``ToolMessage`` payloads in the transcript.""" + result = { + "messages": [ + HumanMessage(content="ask"), + AIMessage(content="thinking..."), + ToolMessage(content="tool output", tool_call_id="tc-1"), + AIMessage(content="final answer"), + ToolMessage(content="trailing tool noise", tool_call_id="tc-2"), + ] + } + + assert extract_final_assistant_message(result) == "final answer" + + +def test_extract_returns_none_when_no_assistant_text_is_present() -> None: + """No ``AIMessage`` with extractable text → ``None`` rather than the + empty string. Lets callers branch on "did the agent actually say + anything?" rather than guess whether ``""`` means silence or empty + output. Empty-string contents are normalized to ``None`` too.""" + no_ai = {"messages": [HumanMessage(content="just a question")]} + only_tools = {"messages": [AIMessage(content=[{"type": "tool_use", "name": "x", "input": {}}])]} + empty_string = {"messages": [AIMessage(content=" ")]} + + assert extract_final_assistant_message(no_ai) is None + assert extract_final_assistant_message(only_tools) is None + assert extract_final_assistant_message(empty_string) is None diff --git a/surfsense_backend/tests/unit/automations/conftest.py b/surfsense_backend/tests/unit/automations/conftest.py new file mode 100644 index 000000000..0fbf03234 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/conftest.py @@ -0,0 +1,39 @@ +"""Shared fixtures for the ``app.automations`` unit-test tree. + +Provides registry isolation: the built-in ``schedule`` trigger and +``agent_task`` action self-register at import time. Tests that register +additional triggers/actions (or assert on the registry contents) must +not leak that state to other tests. These fixtures snapshot and restore +the module-level registry dicts. +""" + +from __future__ import annotations + +from collections.abc import Iterator + +import pytest + +from app.automations.actions import store as action_store +from app.automations.triggers import store as trigger_store + + +@pytest.fixture +def isolated_action_registry() -> Iterator[None]: + """Snapshot and restore the action registry around a test.""" + snapshot = dict(action_store._REGISTRY) + try: + yield + finally: + action_store._REGISTRY.clear() + action_store._REGISTRY.update(snapshot) + + +@pytest.fixture +def isolated_trigger_registry() -> Iterator[None]: + """Snapshot and restore the trigger registry around a test.""" + snapshot = dict(trigger_store._REGISTRY) + try: + yield + finally: + trigger_store._REGISTRY.clear() + trigger_store._REGISTRY.update(snapshot) diff --git a/surfsense_backend/tests/unit/automations/dispatch/__init__.py b/surfsense_backend/tests/unit/automations/dispatch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/dispatch/test_errors.py b/surfsense_backend/tests/unit/automations/dispatch/test_errors.py new file mode 100644 index 000000000..89c1bede9 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/dispatch/test_errors.py @@ -0,0 +1,28 @@ +"""Lock the ``DispatchError`` exception contract. + +``DispatchError`` is the uniform exception type the dispatch layer raises +for any "cannot turn this fire request into a run" condition. Other +modules (templates of error envelopes, run records) compare on +``isinstance(exc, DispatchError)``, so the inheritance is the contract. +""" + +from __future__ import annotations + +import pytest + +from app.automations.dispatch.errors import DispatchError + +pytestmark = pytest.mark.unit + + +def test_dispatch_error_is_exception_subclass_and_carries_message() -> None: + """Lifting a string into ``DispatchError`` preserves the message and + behaves as a regular ``Exception`` for ``isinstance`` / ``raise`` / + ``except`` consumers.""" + error = DispatchError("missing trigger") + + assert isinstance(error, Exception) + assert str(error) == "missing trigger" + + with pytest.raises(DispatchError): + raise error diff --git a/surfsense_backend/tests/unit/automations/dispatch/test_validate_inputs.py b/surfsense_backend/tests/unit/automations/dispatch/test_validate_inputs.py new file mode 100644 index 000000000..ec99e51c2 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/dispatch/test_validate_inputs.py @@ -0,0 +1,77 @@ +"""Lock the input-validation contract used by ``dispatch_run``. + +``_validate_inputs`` is module-internal by convention (underscore), but it +encodes a real behavior contract the rest of the system depends on, and the +public alternative (``dispatch_run``) requires a real DB session. Tests +target the pure function directly; the contract — not the symbol — is what's +locked. +""" + +from __future__ import annotations + +import pytest + +from app.automations.dispatch.errors import DispatchError +from app.automations.dispatch.run import _validate_inputs +from app.automations.schemas.definition.envelope import AutomationDefinition +from app.automations.schemas.definition.inputs import Inputs +from app.automations.schemas.definition.plan_step import PlanStep + +pytestmark = pytest.mark.unit + + +def _minimal_definition(*, inputs: Inputs | None = None) -> AutomationDefinition: + """One-step definition with an optional declared input schema.""" + return AutomationDefinition( + name="test", + inputs=inputs, + plan=[PlanStep(step_id="s1", action="agent_task")], + ) + + +def test_validate_inputs_passes_through_when_no_schema_is_declared() -> None: + """When the definition declares no input schema, runtime inputs reach + the template context **unchanged**. Regression site: previously this + branch returned ``{}``, which stripped runtime keys like ``fired_at`` + and ``last_fired_at`` and made Jinja blow up on ``{{ inputs.* }}``. + """ + definition = _minimal_definition(inputs=None) + runtime_inputs = { + "fired_at": "2026-01-01T00:00:00+00:00", + "last_fired_at": None, + "static_key": "value", + } + + assert _validate_inputs(definition, runtime_inputs) == runtime_inputs + + +def test_validate_inputs_returns_inputs_when_they_match_declared_schema() -> None: + """With a declared JSON schema, inputs that satisfy it pass through + unchanged (validation succeeds; the function does not coerce or + strip extra fields not mentioned in the schema).""" + schema = { + "type": "object", + "properties": {"topic": {"type": "string"}}, + "required": ["topic"], + } + definition = _minimal_definition(inputs=Inputs(schema=schema)) + + inputs = {"topic": "weekly report"} + + assert _validate_inputs(definition, inputs) == inputs + + +def test_validate_inputs_raises_dispatch_error_when_inputs_violate_schema() -> None: + """Inputs that don't match the declared schema must surface as + ``DispatchError`` (not the raw ``jsonschema.ValidationError``), so the + schedule tick and any other caller can handle one dispatch-domain + exception type uniformly.""" + schema = { + "type": "object", + "properties": {"topic": {"type": "string"}}, + "required": ["topic"], + } + definition = _minimal_definition(inputs=Inputs(schema=schema)) + + with pytest.raises(DispatchError): + _validate_inputs(definition, {"topic": 42}) # type violates string diff --git a/surfsense_backend/tests/unit/automations/runtime/__init__.py b/surfsense_backend/tests/unit/automations/runtime/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/runtime/test_execute_step.py b/surfsense_backend/tests/unit/automations/runtime/test_execute_step.py new file mode 100644 index 000000000..9b203fdba --- /dev/null +++ b/surfsense_backend/tests/unit/automations/runtime/test_execute_step.py @@ -0,0 +1,272 @@ +"""Lock the ``execute_step`` orchestration contract. + +Covers the pure step-execution logic: predicate gate, params rendering, +action lookup, retry budget, error shaping. The ``ActionContext.session`` +is never touched by ``execute_step`` itself (it's only forwarded to the +handler), so unit tests pass ``None`` cast to the type. +""" + +from __future__ import annotations + +from typing import Any, cast + +import pytest +from pydantic import BaseModel +from sqlalchemy.ext.asyncio import AsyncSession + +from app.automations.actions.store import register_action +from app.automations.actions.types import ActionContext, ActionDefinition +from app.automations.runtime.step import execute_step +from app.automations.schemas.definition.plan_step import PlanStep + +pytestmark = pytest.mark.unit + + +class _AnyParams(BaseModel): + """Open params model used by test actions — they never validate.""" + + model_config = {"extra": "allow"} + + +def _action_context() -> ActionContext: + """Minimal context: session is unused by ``execute_step``, only forwarded.""" + return ActionContext( + session=cast(AsyncSession, None), + run_id=1, + step_id="s1", + search_space_id=1, + creator_user_id=None, + ) + + +async def test_execute_step_runs_registered_action_handler_and_wraps_result( + isolated_action_registry: None, +) -> None: + """A step pointing at a registered action runs its handler with the + step's params and returns a ``succeeded`` entry carrying the handler's + output plus ``attempts=1`` (one try, no retries triggered).""" + invocations: list[dict[str, Any]] = [] + + async def echo(params: dict[str, Any]) -> dict[str, Any]: + invocations.append(params) + return {"echoed": params["value"]} + + register_action( + ActionDefinition( + type="test_echo", + name="Echo", + description="Test action.", + params_model=_AnyParams, + build_handler=lambda _ctx: echo, + ) + ) + + step = PlanStep(step_id="s1", action="test_echo", params={"value": "hello"}) + + result = await execute_step( + step=step, + template_context={}, + action_context=_action_context(), + default_max_retries=0, + default_retry_backoff="none", + default_timeout_seconds=30, + ) + + assert result["status"] == "succeeded" + assert result["step_id"] == "s1" + assert result["action"] == "test_echo" + assert result["attempts"] == 1 + assert result["result"] == {"echoed": "hello"} + assert invocations == [{"value": "hello"}] + + +async def test_execute_step_skips_step_when_predicate_is_falsy( + isolated_action_registry: None, +) -> None: + """If ``step.when`` evaluates to falsy in the template context, the + handler is **not** invoked, the result entry has ``status=skipped`` + and ``attempts=0``, and no ``result`` key is present.""" + invoked = False + + async def must_not_run(_params: dict[str, Any]) -> dict[str, Any]: + nonlocal invoked + invoked = True + return {} + + register_action( + ActionDefinition( + type="test_guarded", + name="Guarded", + description="Test action that should not run.", + params_model=_AnyParams, + build_handler=lambda _ctx: must_not_run, + ) + ) + + step = PlanStep( + step_id="s1", + action="test_guarded", + when="inputs.enabled", + params={}, + ) + + result = await execute_step( + step=step, + template_context={"inputs": {"enabled": False}}, + action_context=_action_context(), + default_max_retries=0, + default_retry_backoff="none", + default_timeout_seconds=30, + ) + + assert result["status"] == "skipped" + assert result["attempts"] == 0 + assert "result" not in result + assert invoked is False + + +async def test_execute_step_fails_when_step_references_an_unknown_action( + isolated_action_registry: None, +) -> None: + """A step pointing at an action that isn't in the registry must fail + with ``ActionNotFound`` rather than crashing. Catches typos in the + plan and removed actions without the run going off the rails.""" + step = PlanStep(step_id="s1", action="no_such_action", params={}) + + result = await execute_step( + step=step, + template_context={}, + action_context=_action_context(), + default_max_retries=0, + default_retry_backoff="none", + default_timeout_seconds=30, + ) + + assert result["status"] == "failed" + assert result["attempts"] == 0 + assert result["error"]["type"] == "ActionNotFound" + assert "no_such_action" in result["error"]["message"] + + +async def test_execute_step_retries_failing_handler_up_to_default_budget( + isolated_action_registry: None, +) -> None: + """A handler that raises on every attempt consumes the retry budget + (1 initial try + ``default_max_retries`` retries) and the step ends + ``failed`` with the exception's type and message surfaced through + the error envelope.""" + calls = 0 + + async def always_fails(_params: dict[str, Any]) -> dict[str, Any]: + nonlocal calls + calls += 1 + raise RuntimeError("boom") + + register_action( + ActionDefinition( + type="test_fails", + name="Fails", + description="Always raises.", + params_model=_AnyParams, + build_handler=lambda _ctx: always_fails, + ) + ) + + step = PlanStep(step_id="s1", action="test_fails", params={}) + + result = await execute_step( + step=step, + template_context={}, + action_context=_action_context(), + default_max_retries=2, + default_retry_backoff="none", + default_timeout_seconds=30, + ) + + assert result["status"] == "failed" + assert result["attempts"] == 3 + assert calls == 3 + assert result["error"]["type"] == "RuntimeError" + assert "boom" in result["error"]["message"] + + +async def test_execute_step_succeeds_when_handler_recovers_within_retry_budget( + isolated_action_registry: None, +) -> None: + """A handler that fails the first N times and then succeeds yields a + ``succeeded`` entry with ``attempts == N + 1``. Locks that retries + can actually recover (not just exhaust).""" + calls = 0 + + async def flaky(_params: dict[str, Any]) -> dict[str, Any]: + nonlocal calls + calls += 1 + if calls < 3: + raise RuntimeError("transient") + return {"ok": True} + + register_action( + ActionDefinition( + type="test_flaky", + name="Flaky", + description="Fails twice, succeeds third time.", + params_model=_AnyParams, + build_handler=lambda _ctx: flaky, + ) + ) + + step = PlanStep(step_id="s1", action="test_flaky", params={}) + + result = await execute_step( + step=step, + template_context={}, + action_context=_action_context(), + default_max_retries=2, + default_retry_backoff="none", + default_timeout_seconds=30, + ) + + assert result["status"] == "succeeded" + assert result["attempts"] == 3 + assert result["result"] == {"ok": True} + assert calls == 3 + + +async def test_execute_step_renders_step_params_through_template_engine( + isolated_action_registry: None, +) -> None: + """Step params are rendered against the template context before the + handler is invoked. String values containing Jinja expressions get + substituted from ``inputs`` and ``steps`` in the run context.""" + received: list[dict[str, Any]] = [] + + async def capture(params: dict[str, Any]) -> dict[str, Any]: + received.append(params) + return {} + + register_action( + ActionDefinition( + type="test_capture", + name="Capture", + description="Captures the params passed in.", + params_model=_AnyParams, + build_handler=lambda _ctx: capture, + ) + ) + + step = PlanStep( + step_id="s1", + action="test_capture", + params={"message": "Hello {{ inputs.name }}"}, + ) + + await execute_step( + step=step, + template_context={"inputs": {"name": "World"}, "steps": {}}, + action_context=_action_context(), + default_max_retries=0, + default_retry_backoff="none", + default_timeout_seconds=30, + ) + + assert received == [{"message": "Hello World"}] diff --git a/surfsense_backend/tests/unit/automations/runtime/test_retries.py b/surfsense_backend/tests/unit/automations/runtime/test_retries.py new file mode 100644 index 000000000..f0f12ca59 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/runtime/test_retries.py @@ -0,0 +1,72 @@ +"""Lock the ``with_retries`` policy: budget, recovery, exhaustion, timeout, backoff. + +Tests with ``backoff="none"`` to keep wall-clock time zero. Backoff sleep +values themselves are observed by monkeypatching ``asyncio.sleep`` so we +don't introduce flakiness via real timing. +""" + +from __future__ import annotations + +import pytest + +from app.automations.runtime.retries import with_retries + +pytestmark = pytest.mark.unit + + +async def test_with_retries_returns_result_and_attempts_one_on_first_success() -> None: + """A coroutine that succeeds on the first call returns its result + paired with ``attempts=1`` — no retry consumed.""" + calls = 0 + + async def succeed() -> str: + nonlocal calls + calls += 1 + return "ok" + + result, attempts = await with_retries( + succeed, max_retries=2, backoff="none", timeout=None + ) + + assert result == "ok" + assert attempts == 1 + assert calls == 1 + + +async def test_with_retries_returns_attempt_count_when_succeeding_after_failures() -> None: + """A coroutine that fails twice then succeeds returns ``attempts=3`` + (the actual attempt that produced the result). Locks the contract + that the caller can distinguish first-try success from a recovery.""" + calls = 0 + + async def flaky() -> str: + nonlocal calls + calls += 1 + if calls < 3: + raise RuntimeError("transient") + return "ok" + + result, attempts = await with_retries( + flaky, max_retries=5, backoff="none", timeout=None + ) + + assert result == "ok" + assert attempts == 3 + assert calls == 3 + + +async def test_with_retries_reraises_after_exhausting_the_budget() -> None: + """When the coroutine raises on every attempt within + ``1 + max_retries`` tries, the last exception propagates and the + handler is called exactly ``1 + max_retries`` times.""" + calls = 0 + + async def always_fails() -> str: + nonlocal calls + calls += 1 + raise RuntimeError(f"boom-{calls}") + + with pytest.raises(RuntimeError, match="boom-3"): + await with_retries(always_fails, max_retries=2, backoff="none", timeout=None) + + assert calls == 3 # 1 initial + 2 retries diff --git a/surfsense_backend/tests/unit/automations/schemas/__init__.py b/surfsense_backend/tests/unit/automations/schemas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/schemas/api/__init__.py b/surfsense_backend/tests/unit/automations/schemas/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/schemas/api/test_api_automation.py b/surfsense_backend/tests/unit/automations/schemas/api/test_api_automation.py new file mode 100644 index 000000000..6ae3ce794 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/schemas/api/test_api_automation.py @@ -0,0 +1,82 @@ +"""Lock the request-side automation API schemas — the public validation gate.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.automations.schemas.api.automation import AutomationCreate, AutomationUpdate + +pytestmark = pytest.mark.unit + + +_VALID_DEFINITION = { + "name": "Test", + "plan": [{"step_id": "s1", "action": "agent_task"}], +} + + +def test_automation_create_accepts_valid_minimal_payload() -> None: + """Happy path: just search_space_id, name, and a valid definition. + Triggers default to ``[]`` so users can attach them later.""" + payload = AutomationCreate.model_validate( + { + "search_space_id": 1, + "name": "Daily digest", + "definition": _VALID_DEFINITION, + } + ) + + assert payload.name == "Daily digest" + assert payload.description is None + assert payload.triggers == [] + + +def test_automation_create_cascades_validation_into_nested_definition() -> None: + """A bad ``definition`` (e.g. empty plan) fails at the API boundary, + not at the DB layer. Locks the cascade so corrupt definitions can't + sneak through a misshapen wire payload.""" + with pytest.raises(ValidationError): + AutomationCreate.model_validate( + { + "search_space_id": 1, + "name": "Bad", + "definition": {"name": "X", "plan": []}, # empty plan + } + ) + + +def test_automation_create_rejects_unknown_top_level_field() -> None: + """``extra='forbid'`` catches typos in API payloads at the boundary.""" + with pytest.raises(ValidationError): + AutomationCreate.model_validate( + { + "search_space_id": 1, + "name": "X", + "definition": _VALID_DEFINITION, + "owner": "tg", # not allowed + } + ) + + +def test_automation_create_rejects_empty_name() -> None: + """Name is required and constrained to 1..200 chars.""" + with pytest.raises(ValidationError): + AutomationCreate.model_validate( + { + "search_space_id": 1, + "name": "", + "definition": _VALID_DEFINITION, + } + ) + + +def test_automation_update_accepts_partial_payload_with_no_fields() -> None: + """All fields on ``AutomationUpdate`` are optional. An empty body is + a valid no-op update (the service layer decides what to do with it).""" + update = AutomationUpdate.model_validate({}) + + assert update.name is None + assert update.description is None + assert update.status is None + assert update.definition is None diff --git a/surfsense_backend/tests/unit/automations/schemas/api/test_api_trigger.py b/surfsense_backend/tests/unit/automations/schemas/api/test_api_trigger.py new file mode 100644 index 000000000..cabfc41af --- /dev/null +++ b/surfsense_backend/tests/unit/automations/schemas/api/test_api_trigger.py @@ -0,0 +1,47 @@ +"""Lock the request-side trigger API schemas.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.automations.persistence.enums.trigger_type import TriggerType +from app.automations.schemas.api.trigger import TriggerCreate, TriggerUpdate + +pytestmark = pytest.mark.unit + + +def test_trigger_create_uses_safe_defaults_for_optional_fields() -> None: + """Defaults: empty ``params`` and ``static_inputs``, ``enabled=True``. + These let callers create a trigger with just ``type`` + the params + the trigger requires.""" + trigger = TriggerCreate(type=TriggerType.SCHEDULE) # type: ignore[arg-type] + + assert trigger.type is TriggerType.SCHEDULE + assert trigger.params == {} + assert trigger.static_inputs == {} + assert trigger.enabled is True + + +def test_trigger_create_rejects_unknown_trigger_type_string() -> None: + """``type`` is a ``TriggerType`` enum, so any string outside the + enum's known values fails validation at the boundary.""" + with pytest.raises(ValidationError): + TriggerCreate.model_validate({"type": "webhook"}) # not in TriggerType + + +def test_trigger_create_rejects_unknown_field() -> None: + """``extra='forbid'`` catches typos in trigger payloads.""" + with pytest.raises(ValidationError): + TriggerCreate.model_validate( + {"type": "schedule", "param": {}} # typo: param vs params + ) + + +def test_trigger_update_accepts_partial_payload_with_no_fields() -> None: + """``TriggerUpdate`` is fully optional — empty body is valid (no-op).""" + update = TriggerUpdate() + + assert update.enabled is None + assert update.params is None + assert update.static_inputs is None diff --git a/surfsense_backend/tests/unit/automations/schemas/definition/__init__.py b/surfsense_backend/tests/unit/automations/schemas/definition/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/schemas/definition/test_envelope.py b/surfsense_backend/tests/unit/automations/schemas/definition/test_envelope.py new file mode 100644 index 000000000..c625b0ec9 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/schemas/definition/test_envelope.py @@ -0,0 +1,57 @@ +"""Lock the ``AutomationDefinition`` envelope contract.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.automations.schemas.definition.envelope import AutomationDefinition +from app.automations.schemas.definition.plan_step import PlanStep + +pytestmark = pytest.mark.unit + + +def test_automation_definition_accepts_minimal_valid_input_with_sensible_defaults() -> None: + """A definition with just ``name`` + a one-step ``plan`` is valid and + fills in the rest with safe defaults so users don't have to write + out every section to get started.""" + definition = AutomationDefinition( + name="Daily digest", + plan=[PlanStep(step_id="s1", action="agent_task")], + ) + + assert definition.name == "Daily digest" + assert definition.schema_version == "1.0" + assert definition.goal is None + assert definition.inputs is None + assert definition.triggers == [] + + +def test_automation_definition_rejects_unknown_top_level_field() -> None: + """``extra='forbid'`` catches typos at validation time (e.g. ``pln`` + instead of ``plan``) before the bad definition reaches storage.""" + with pytest.raises(ValidationError): + AutomationDefinition.model_validate( + { + "name": "X", + "plan": [{"step_id": "s1", "action": "agent_task"}], + "extra_field": "unexpected", + } + ) + + +def test_automation_definition_rejects_empty_plan() -> None: + """An automation with no plan steps has nothing to execute and must + be rejected at validation time.""" + with pytest.raises(ValidationError): + AutomationDefinition(name="X", plan=[]) + + +def test_automation_definition_rejects_empty_name() -> None: + """Name is required and must be non-empty so list views and audit + logs have something meaningful to display.""" + with pytest.raises(ValidationError): + AutomationDefinition( + name="", + plan=[PlanStep(step_id="s1", action="agent_task")], + ) diff --git a/surfsense_backend/tests/unit/automations/schemas/definition/test_execution.py b/surfsense_backend/tests/unit/automations/schemas/definition/test_execution.py new file mode 100644 index 000000000..15adefab0 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/schemas/definition/test_execution.py @@ -0,0 +1,49 @@ +"""Lock the ``Execution`` defaults + literal-constraint contract. + +These defaults control production behavior of every automation that +doesn't override them; the defaults *are* the contract. +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.automations.schemas.definition.execution import Execution + +pytestmark = pytest.mark.unit + + +def test_execution_uses_production_defaults_when_no_overrides_provided() -> None: + """The defaults shipped to prod: 10-minute wall clock, 2 retries + per step, exponential backoff, drop overlapping runs. Changing any + of these is a behavioral release-note change.""" + execution = Execution() + + assert execution.timeout_seconds == 600 + assert execution.max_retries == 2 + assert execution.retry_backoff == "exponential" + assert execution.concurrency == "drop_if_running" + assert execution.on_failure == [] + + +def test_execution_rejects_unknown_retry_backoff_strategy() -> None: + """``retry_backoff`` is constrained to a closed set — typos like + ``"expontential"`` must fail validation, not silently coerce.""" + with pytest.raises(ValidationError): + Execution(retry_backoff="expontential") # type: ignore[arg-type] + + +def test_execution_rejects_unknown_concurrency_strategy() -> None: + """Same closed-set constraint on ``concurrency``.""" + with pytest.raises(ValidationError): + Execution(concurrency="parallel") # type: ignore[arg-type] + + +def test_execution_rejects_invalid_numeric_bounds() -> None: + """``timeout_seconds > 0`` and ``max_retries >= 0``. Zero or negative + values would produce nonsensical run behavior.""" + with pytest.raises(ValidationError): + Execution(timeout_seconds=0) + with pytest.raises(ValidationError): + Execution(max_retries=-1) diff --git a/surfsense_backend/tests/unit/automations/schemas/definition/test_inputs.py b/surfsense_backend/tests/unit/automations/schemas/definition/test_inputs.py new file mode 100644 index 000000000..5dc24463f --- /dev/null +++ b/surfsense_backend/tests/unit/automations/schemas/definition/test_inputs.py @@ -0,0 +1,39 @@ +"""Lock the ``Inputs`` JSON ``schema``-alias roundtrip. + +The field is ``schema_`` in Python (``schema`` shadows a Pydantic builtin) +but is wire-named ``schema``. Locking the roundtrip means JSON definitions +authored anywhere (UI raw editor, NL drafter, CLI export) speak the same +wire shape. +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.automations.schemas.definition.inputs import Inputs + +pytestmark = pytest.mark.unit + + +def test_inputs_parses_wire_field_named_schema_into_schema_attribute() -> None: + """JSON payloads use ``schema`` (the convention). The model stores it + on the Python attribute ``schema_`` without shadowing the builtin.""" + parsed = Inputs.model_validate({"schema": {"type": "object"}}) + + assert parsed.schema_ == {"type": "object"} + + +def test_inputs_serializes_schema_attribute_back_to_wire_field_named_schema() -> None: + """Round-trip: serializing emits ``schema`` (alias), not ``schema_``. + Locks the consumer-visible JSON shape regardless of the Python name.""" + inputs = Inputs(schema={"type": "object"}) # type: ignore[call-arg] + + assert inputs.model_dump() == {"schema": {"type": "object"}} + + +def test_inputs_rejects_unknown_field() -> None: + """``extra='forbid'`` catches typos like ``shema`` so bad definitions + don't silently lose their input declaration.""" + with pytest.raises(ValidationError): + Inputs.model_validate({"schema": {}, "extra": "x"}) diff --git a/surfsense_backend/tests/unit/automations/schemas/definition/test_metadata.py b/surfsense_backend/tests/unit/automations/schemas/definition/test_metadata.py new file mode 100644 index 000000000..9ac90bb3f --- /dev/null +++ b/surfsense_backend/tests/unit/automations/schemas/definition/test_metadata.py @@ -0,0 +1,37 @@ +"""Lock the ``Metadata`` ``extra='allow'`` contract — the only schema +that does. Free-form annotations on definitions (e.g. ``owner``, +``project``, ``created_by_ai``) need to round-trip through the envelope +without being rejected. +""" + +from __future__ import annotations + +import pytest + +from app.automations.schemas.definition.metadata import Metadata + +pytestmark = pytest.mark.unit + + +def test_metadata_preserves_unknown_keys() -> None: + """Unlike every other definition sub-schema, ``Metadata`` allows + extra keys and round-trips them — that's its purpose.""" + metadata = Metadata.model_validate( + { + "tags": ["weekly", "report"], + "owner": "tg", + "created_by_ai": True, + } + ) + + dumped = metadata.model_dump() + + assert dumped["tags"] == ["weekly", "report"] + assert dumped["owner"] == "tg" + assert dumped["created_by_ai"] is True + + +def test_metadata_defaults_tags_to_empty_list() -> None: + """No tags is the common case; the default is the empty list so + callers can append without a None check.""" + assert Metadata().tags == [] diff --git a/surfsense_backend/tests/unit/automations/schemas/definition/test_plan_step.py b/surfsense_backend/tests/unit/automations/schemas/definition/test_plan_step.py new file mode 100644 index 000000000..6896a7f5a --- /dev/null +++ b/surfsense_backend/tests/unit/automations/schemas/definition/test_plan_step.py @@ -0,0 +1,52 @@ +"""Lock the ``PlanStep`` validation contract.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.automations.schemas.definition.plan_step import PlanStep + +pytestmark = pytest.mark.unit + + +def test_plan_step_accepts_minimal_input_with_safe_defaults() -> None: + """A step with just ``step_id`` + ``action`` is valid. Defaults + (no when, empty params, no output_as override, no retry/timeout + override) let the run inherit automation-wide defaults.""" + step = PlanStep(step_id="s1", action="agent_task") + + assert step.step_id == "s1" + assert step.action == "agent_task" + assert step.when is None + assert step.params == {} + assert step.output_as is None + assert step.max_retries is None + assert step.timeout_seconds is None + + +def test_plan_step_rejects_empty_step_id_and_action() -> None: + """``step_id`` and ``action`` are addressing primitives — empty + strings would silently break runtime lookups.""" + with pytest.raises(ValidationError): + PlanStep(step_id="", action="agent_task") + with pytest.raises(ValidationError): + PlanStep(step_id="s1", action="") + + +def test_plan_step_rejects_negative_max_retries_and_non_positive_timeout() -> None: + """Numeric constraints: ``max_retries >= 0`` and ``timeout_seconds > 0``. + Negative budgets or zero timeouts produce nonsensical run behavior.""" + with pytest.raises(ValidationError): + PlanStep(step_id="s1", action="agent_task", max_retries=-1) + with pytest.raises(ValidationError): + PlanStep(step_id="s1", action="agent_task", timeout_seconds=0) + + +def test_plan_step_rejects_unknown_field() -> None: + """``extra='forbid'`` catches typos like ``actoin`` (instead of + ``action``) before the bad step reaches storage.""" + with pytest.raises(ValidationError): + PlanStep.model_validate( + {"step_id": "s1", "action": "agent_task", "actoin": "agent_task"} + ) diff --git a/surfsense_backend/tests/unit/automations/schemas/definition/test_trigger_spec.py b/surfsense_backend/tests/unit/automations/schemas/definition/test_trigger_spec.py new file mode 100644 index 000000000..cf1a52466 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/schemas/definition/test_trigger_spec.py @@ -0,0 +1,33 @@ +"""Lock the ``TriggerSpec`` validation contract — the entry shape used +inside an automation's ``triggers[]`` array. +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.automations.schemas.definition.trigger_spec import TriggerSpec + +pytestmark = pytest.mark.unit + + +def test_trigger_spec_accepts_type_with_default_empty_params() -> None: + """``type`` is required; ``params`` defaults to ``{}`` so triggers + that take no params don't need an explicit body.""" + spec = TriggerSpec(type="schedule") + + assert spec.type == "schedule" + assert spec.params == {} + + +def test_trigger_spec_rejects_empty_type() -> None: + """``type`` is the registry lookup key — empty would silently miss.""" + with pytest.raises(ValidationError): + TriggerSpec(type="") + + +def test_trigger_spec_rejects_unknown_field() -> None: + """``extra='forbid'`` catches typos at definition-validation time.""" + with pytest.raises(ValidationError): + TriggerSpec.model_validate({"type": "schedule", "paramz": {}}) diff --git a/surfsense_backend/tests/unit/automations/templating/__init__.py b/surfsense_backend/tests/unit/automations/templating/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/templating/test_context.py b/surfsense_backend/tests/unit/automations/templating/test_context.py new file mode 100644 index 000000000..54f372e77 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/templating/test_context.py @@ -0,0 +1,53 @@ +"""Lock the ``{run, inputs, steps}`` namespace exposed to every template.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from uuid import UUID + +import pytest + +from app.automations.templating.context import build_run_context + +pytestmark = pytest.mark.unit + + +def test_build_run_context_exposes_run_inputs_and_steps_namespaces() -> None: + """The namespace handed to templates groups run metadata under ``run``, + runtime + static inputs under ``inputs``, and step outputs (keyed by + ``output_as`` / ``step_id``) under ``steps``. Locks the contract that + every plan template body relies on.""" + creator = UUID("00000000-0000-0000-0000-000000000001") + started = datetime(2026, 5, 28, 14, 30, tzinfo=UTC) + + ctx = build_run_context( + run_id=42, + automation_id=7, + automation_name="Weekly digest", + automation_version=3, + search_space_id=1, + creator_id=creator, + trigger_id=11, + trigger_type="schedule", + started_at=started, + attempt=2, + inputs={"topic": "weekly"}, + step_outputs={"summarize": {"text": "ok"}}, + ) + + assert ctx == { + "run": { + "id": 42, + "automation_id": 7, + "automation_name": "Weekly digest", + "automation_version": 3, + "search_space_id": 1, + "creator_id": creator, + "trigger_id": 11, + "trigger_type": "schedule", + "started_at": started, + "attempt": 2, + }, + "inputs": {"topic": "weekly"}, + "steps": {"summarize": {"text": "ok"}}, + } diff --git a/surfsense_backend/tests/unit/automations/templating/test_environment.py b/surfsense_backend/tests/unit/automations/templating/test_environment.py new file mode 100644 index 000000000..ec1c0ee40 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/templating/test_environment.py @@ -0,0 +1,51 @@ +"""Lock the sandbox boundary: disallowed filters/tests reject, finalize coerces non-strings. + +These behaviors live in ``environment.py`` but are observed through the +public ``render_template`` surface — the same surface every step uses. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest +from jinja2.exceptions import TemplateError + +from app.automations.templating.render import render_template + +pytestmark = pytest.mark.unit + + +def test_environment_rejects_filters_not_in_the_allowlist() -> None: + """A template that pipes through a Jinja built-in **not** in the + allowlist (e.g. ``pprint``) must fail rather than rendering. Locks + the sandbox surface against accidental re-introduction of removed + filters.""" + with pytest.raises(TemplateError): + render_template("{{ value | pprint }}", {"value": {"k": 1}}) + + +def test_environment_finalizes_datetime_output_to_iso_string() -> None: + """A datetime that lands directly at an output site is stringified + via ``isoformat()`` rather than producing ``str(datetime)`` (which + has a space separator). Locks the wire shape templates produce + when emitting ``inputs.fired_at`` and other datetime values.""" + dt = datetime(2026, 5, 28, 14, 30, tzinfo=UTC) + + assert render_template("{{ moment }}", {"moment": dt}) == "2026-05-28T14:30:00+00:00" + + +def test_environment_finalizes_none_output_to_empty_string() -> None: + """A ``None`` at an output site becomes the empty string. Lets + templates write ``{{ inputs.last_fired_at }}`` unconditionally on + the first run without exploding on the null.""" + assert render_template("{{ missing }}", {"missing": None}) == "" + + +def test_environment_finalizes_dict_output_to_json() -> None: + """A dict at an output site is JSON-serialized. Same for lists. + Locks the wire shape so users embedding structured values into + prompts get deterministic, parseable output.""" + rendered = render_template("{{ payload }}", {"payload": {"a": 1, "b": [2, 3]}}) + + assert rendered == '{"a": 1, "b": [2, 3]}' diff --git a/surfsense_backend/tests/unit/automations/templating/test_filters.py b/surfsense_backend/tests/unit/automations/templating/test_filters.py new file mode 100644 index 000000000..cf83ee337 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/templating/test_filters.py @@ -0,0 +1,42 @@ +"""Lock the custom Jinja filters: ``date`` and ``slugify``.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from app.automations.templating.filters import filter_date, filter_slugify + +pytestmark = pytest.mark.unit + + +def test_filter_slugify_produces_url_safe_slug_from_typical_title() -> None: + """``filter_slugify`` lowercases, replaces non-alphanumerics with + hyphens, collapses repeats, and trims edge hyphens — the standard + URL-slug contract users expect when piping titles into paths.""" + assert filter_slugify("Hello, World! 2026") == "hello-world-2026" + + +def test_filter_date_formats_datetime_with_strftime_format() -> None: + """``filter_date`` calls ``strftime`` on datetime-like values with the + provided format. Default format yields ISO date (YYYY-MM-DD).""" + dt = datetime(2026, 5, 28, 14, 30, tzinfo=UTC) + + assert filter_date(dt) == "2026-05-28" + assert filter_date(dt, "%Y/%m/%d %H:%M") == "2026/05/28 14:30" + + +def test_filter_date_returns_empty_string_for_none() -> None: + """``None`` (e.g., a never-fired ``last_fired_at``) renders as the + empty string rather than the literal ``"None"`` or raising. This is + what lets templates write ``{{ inputs.last_fired_at | date }}`` + unconditionally on the first run.""" + assert filter_date(None) == "" + + +def test_filter_date_passes_strings_through_unchanged() -> None: + """Already-formatted ISO strings (the JSON-serialized shape of + runtime inputs like ``fired_at``) pass through unchanged so callers + don't have to special-case the type.""" + assert filter_date("2026-05-28T14:30:00+00:00") == "2026-05-28T14:30:00+00:00" diff --git a/surfsense_backend/tests/unit/automations/templating/test_render.py b/surfsense_backend/tests/unit/automations/templating/test_render.py new file mode 100644 index 000000000..42a7c7082 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/templating/test_render.py @@ -0,0 +1,59 @@ +"""Lock the public template-rendering surface: render, predicate, recursive.""" + +from __future__ import annotations + +import pytest +from jinja2 import UndefinedError + +from app.automations.templating.render import ( + evaluate_predicate, + render_template, + render_value, +) + +pytestmark = pytest.mark.unit + + +def test_render_template_substitutes_context_variables() -> None: + """A template referencing a context variable produces the substituted + string. Most basic contract of the template engine.""" + result = render_template("Hello {{ name }}!", {"name": "World"}) + + assert result == "Hello World!" + + +def test_render_template_raises_on_undefined_variable() -> None: + """Referencing a variable that isn't in the context raises rather than + rendering the empty string. Locks the StrictUndefined safety net so + template typos surface as run failures instead of silent corruption.""" + with pytest.raises(UndefinedError): + render_template("Hello {{ missing }}!", {}) + + +def test_evaluate_predicate_returns_truthy_outcome_of_expression() -> None: + """``evaluate_predicate`` compiles a Jinja **expression** (not template + body) and coerces the value to ``bool``. Drives ``step.when`` gating.""" + assert evaluate_predicate("inputs.count > 0", {"inputs": {"count": 3}}) is True + assert evaluate_predicate("inputs.count > 0", {"inputs": {"count": 0}}) is False + + +def test_render_value_renders_strings_recursively_through_dicts_and_lists() -> None: + """``render_value`` walks dicts and lists, renders string leaves through + the template engine, and leaves non-strings untouched. This is the + primitive ``execute_step`` uses to render step params at run time.""" + context = {"inputs": {"name": "World"}, "topic": "weekly"} + + rendered = render_value( + { + "greeting": "Hello {{ inputs.name }}", + "tags": ["{{ topic }}", "static"], + "config": {"retries": 3, "label": "{{ topic }}-{{ inputs.name }}"}, + }, + context, + ) + + assert rendered == { + "greeting": "Hello World", + "tags": ["weekly", "static"], + "config": {"retries": 3, "label": "weekly-World"}, + } diff --git a/surfsense_backend/tests/unit/automations/test_definition_types.py b/surfsense_backend/tests/unit/automations/test_definition_types.py new file mode 100644 index 000000000..231e4fa97 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/test_definition_types.py @@ -0,0 +1,56 @@ +"""Lock the ``params_schema`` derivation on action + trigger definitions. + +Both definition dataclasses expose ``params_schema`` as the JSON Schema +of their ``params_model``. This is what the registry endpoints surface +to the UI as the "what shape do these params take?" contract. +""" + +from __future__ import annotations + +import pytest +from pydantic import BaseModel + +from app.automations.actions.types import ActionDefinition +from app.automations.triggers.types import TriggerDefinition + +pytestmark = pytest.mark.unit + + +class _Topic(BaseModel): + """Model with one required string field — minimal schema fingerprint.""" + + topic: str + + +def test_action_definition_params_schema_reflects_params_model() -> None: + """``ActionDefinition.params_schema`` returns a JSON Schema derived + from the Pydantic ``params_model`` — required fields and types are + visible to clients consuming the registry endpoint.""" + definition = ActionDefinition( + type="t", + name="N", + description="D", + params_model=_Topic, + build_handler=lambda _ctx: (lambda _p: {}), # type: ignore[arg-type,return-value] + ) + + schema = definition.params_schema + + assert schema["type"] == "object" + assert schema["properties"]["topic"]["type"] == "string" + assert "topic" in schema["required"] + + +def test_trigger_definition_params_schema_reflects_params_model() -> None: + """Same JSON-Schema derivation contract on the trigger side.""" + definition = TriggerDefinition( + type="t", + description="D", + params_model=_Topic, + ) + + schema = definition.params_schema + + assert schema["type"] == "object" + assert schema["properties"]["topic"]["type"] == "string" + assert "topic" in schema["required"] diff --git a/surfsense_backend/tests/unit/automations/test_import_registrations.py b/surfsense_backend/tests/unit/automations/test_import_registrations.py new file mode 100644 index 000000000..35b1effa7 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/test_import_registrations.py @@ -0,0 +1,37 @@ +"""Lock the bundled import side-effects. + +Importing ``app.automations`` (the package) registers the v1 bundled +action (``agent_task``) and the v1 bundled trigger (``schedule``). If the +import chain breaks (e.g. someone removes ``from . import definition`` +in a sub-package ``__init__``), the system would silently launch with an +empty registry. These tests are the canary. +""" + +from __future__ import annotations + +import pytest + +import app.automations # noqa: F401 (force the package import + its side-effects) +from app.automations.actions.store import get_action +from app.automations.persistence.enums.trigger_type import TriggerType +from app.automations.triggers.store import get_trigger + +pytestmark = pytest.mark.unit + + +def test_bundled_agent_task_action_is_registered_after_package_import() -> None: + """``agent_task`` — the v1 default action — must be discoverable in + the registry after the package is imported.""" + definition = get_action("agent_task") + + assert definition is not None + assert definition.type == "agent_task" + + +def test_bundled_schedule_trigger_is_registered_after_package_import() -> None: + """``schedule`` — the only v1 trigger — must be discoverable in the + registry after the package is imported.""" + definition = get_trigger(TriggerType.SCHEDULE.value) + + assert definition is not None + assert definition.type == TriggerType.SCHEDULE.value diff --git a/surfsense_backend/tests/unit/automations/test_persistence_enums.py b/surfsense_backend/tests/unit/automations/test_persistence_enums.py new file mode 100644 index 000000000..59703dfc6 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/test_persistence_enums.py @@ -0,0 +1,45 @@ +"""Lock the persistence enum string values + members. + +These enums are mirrored by Postgres enum types, embedded in stored DB +rows, and surfaced in the JSON API. Renaming a value (or removing a +member) silently breaks production data and previously-issued API +responses, so the strings + the set of members are the contract. +""" + +from __future__ import annotations + +import pytest + +from app.automations.persistence.enums.automation_status import AutomationStatus +from app.automations.persistence.enums.run_status import RunStatus +from app.automations.persistence.enums.trigger_type import TriggerType + +pytestmark = pytest.mark.unit + + +def test_automation_status_string_values_are_stable() -> None: + """The exact strings persisted to Postgres and served in API JSON.""" + assert {member.value for member in AutomationStatus} == { + "active", + "paused", + "archived", + } + + +def test_run_status_string_values_are_stable() -> None: + """Run lifecycle states embedded in the ``automation_runs`` table.""" + assert {member.value for member in RunStatus} == { + "pending", + "running", + "succeeded", + "failed", + "cancelled", + "timed_out", + } + + +def test_trigger_type_keeps_manual_member_even_though_unregistered() -> None: + """``MANUAL`` is reserved (mirrors the Postgres enum) but the trigger + store does not register it in v1. The enum must keep both members so + existing DB rows and the schema migration plan stay valid.""" + assert {member.value for member in TriggerType} == {"schedule", "manual"} diff --git a/surfsense_backend/tests/unit/automations/test_stores.py b/surfsense_backend/tests/unit/automations/test_stores.py new file mode 100644 index 000000000..e54062d64 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/test_stores.py @@ -0,0 +1,115 @@ +"""Lock the trigger + action registry contracts. + +Both stores share the same API shape (register/get/all + duplicate-raise), +so they're tested together to keep the contract visible side-by-side. +""" + +from __future__ import annotations + +import pytest +from pydantic import BaseModel + +from app.automations.actions.store import ( + get_action, + register_action, +) +from app.automations.actions.types import ActionDefinition +from app.automations.triggers.store import ( + all_triggers, + get_trigger, + register_trigger, +) +from app.automations.triggers.types import TriggerDefinition + +pytestmark = pytest.mark.unit + + +class _Params(BaseModel): + """Empty params model used by test-only registrations.""" + + +def _trigger(type_: str = "test_trigger") -> TriggerDefinition: + return TriggerDefinition(type=type_, description="Test trigger.", params_model=_Params) + + +def _action(type_: str = "test_action") -> ActionDefinition: + return ActionDefinition( + type=type_, + name="Test", + description="Test action.", + params_model=_Params, + build_handler=lambda _ctx: (lambda _p: {}), # type: ignore[arg-type,return-value] + ) + + +def test_register_trigger_then_get_trigger_returns_the_same_definition( + isolated_trigger_registry: None, +) -> None: + """The canonical round-trip: register, look up by type, get the same + definition back. Locks the basic registry contract.""" + definition = _trigger() + register_trigger(definition) + + assert get_trigger("test_trigger") is definition + + +def test_register_action_then_get_action_returns_the_same_definition( + isolated_action_registry: None, +) -> None: + """Same round-trip contract for the action registry.""" + definition = _action() + register_action(definition) + + assert get_action("test_action") is definition + + +def test_get_trigger_returns_none_for_unknown_type( + isolated_trigger_registry: None, +) -> None: + """An unknown type returns ``None`` (not raises). Lets callers like + the dispatcher branch on "is this trigger still registered?" without + try/except.""" + assert get_trigger("never_registered") is None + + +def test_get_action_returns_none_for_unknown_type( + isolated_action_registry: None, +) -> None: + """Same ``None``-not-raise contract on the action side.""" + assert get_action("never_registered") is None + + +def test_register_trigger_rejects_duplicate_type( + isolated_trigger_registry: None, +) -> None: + """Re-registering the same ``type`` raises rather than silently + overwriting. Locks the safety net against accidental double-import + (e.g., circular imports re-running the registration block).""" + register_trigger(_trigger()) + + with pytest.raises(ValueError, match="test_trigger"): + register_trigger(_trigger()) + + +def test_register_action_rejects_duplicate_type( + isolated_action_registry: None, +) -> None: + """Same duplicate-rejection contract on the action side.""" + register_action(_action()) + + with pytest.raises(ValueError, match="test_action"): + register_action(_action()) + + +def test_all_triggers_returns_defensive_snapshot( + isolated_trigger_registry: None, +) -> None: + """``all_triggers()`` returns a copy: mutating the returned dict does + not corrupt the internal registry. Locks the snapshot contract that + UI/listing endpoints rely on.""" + register_trigger(_trigger("snapshot_test")) + + snapshot = all_triggers() + snapshot.pop("snapshot_test") + + assert get_trigger("snapshot_test") is not None \ No newline at end of file diff --git a/surfsense_backend/tests/unit/automations/triggers/__init__.py b/surfsense_backend/tests/unit/automations/triggers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/triggers/schedule/__init__.py b/surfsense_backend/tests/unit/automations/triggers/schedule/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/automations/triggers/schedule/test_cron.py b/surfsense_backend/tests/unit/automations/triggers/schedule/test_cron.py new file mode 100644 index 000000000..261e51b18 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/triggers/schedule/test_cron.py @@ -0,0 +1,82 @@ +"""Lock the cron + timezone + UTC normalization contract.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import pytest + +from app.automations.triggers.schedule.cron import ( + InvalidCronError, + compute_next_fire_at, + validate_cron, +) + +pytestmark = pytest.mark.unit + + +def test_compute_next_fire_at_returns_next_match_normalized_to_utc() -> None: + """``compute_next_fire_at`` evaluates the cron in the given IANA timezone + and returns the next strictly-later match expressed in UTC. + + Setup: ``0 9 * * 1-5`` (09:00 Monday-Friday) in ``Africa/Kigali`` + (UTC+2, no DST). With ``after`` = Tuesday 05:00 UTC (= 07:00 local), + the next fire is the same Tuesday at 09:00 local = 07:00 UTC. + """ + after = datetime(2026, 5, 26, 5, 0, tzinfo=UTC) # Tue 07:00 Kigali + + next_fire = compute_next_fire_at("0 9 * * 1-5", "Africa/Kigali", after=after) + + assert next_fire == datetime(2026, 5, 26, 7, 0, tzinfo=UTC) + + +def test_compute_next_fire_at_respects_dst_offset_change() -> None: + """A daily cron in a DST-observing tz fires at the same local hour + across the DST boundary, which produces a different UTC offset on + either side of the transition. + + Setup: ``0 9 * * *`` (09:00 every day) in ``America/New_York``. + NY is UTC-5 in winter (EST), UTC-4 in summer (EDT). Evaluating from + each side of the spring-forward in 2026 (Sun Mar 8 at 02:00 → 03:00): + + - winter: ``after`` = 2026-02-15 (EST, UTC-5) → next 09:00 EST = 14:00 UTC + - summer: ``after`` = 2026-04-15 (EDT, UTC-4) → next 09:00 EDT = 13:00 UTC + """ + winter_after = datetime(2026, 2, 15, 0, 0, tzinfo=UTC) + summer_after = datetime(2026, 4, 15, 0, 0, tzinfo=UTC) + + winter_fire = compute_next_fire_at("0 9 * * *", "America/New_York", after=winter_after) + summer_fire = compute_next_fire_at("0 9 * * *", "America/New_York", after=summer_after) + + assert winter_fire == datetime(2026, 2, 15, 14, 0, tzinfo=UTC) + assert summer_fire == datetime(2026, 4, 15, 13, 0, tzinfo=UTC) + + +def test_compute_next_fire_at_is_strictly_after_when_after_equals_a_match() -> None: + """When ``after`` lands exactly on a cron match, the result jumps to the + next match — never the same instant. Required so the schedule-tick + can pass ``next_fire_at`` itself as ``after`` to advance to the + following slot without double-firing. + + Setup: weekday 09:00 Kigali. ``after`` = Mon 09:00 Kigali = 07:00 UTC + (an exact match) → next fire must be Tue 09:00 Kigali = next day 07:00 UTC. + """ + after = datetime(2026, 5, 25, 7, 0, tzinfo=UTC) # Mon 09:00 Kigali — exact match + + next_fire = compute_next_fire_at("0 9 * * 1-5", "Africa/Kigali", after=after) + + assert next_fire == datetime(2026, 5, 26, 7, 0, tzinfo=UTC) # Tue 09:00 Kigali + + +def test_validate_cron_rejects_malformed_cron_expression() -> None: + """A syntactically invalid cron must be rejected at validation time so + bad triggers can't reach storage and explode at fire time.""" + with pytest.raises(InvalidCronError): + validate_cron("this is not cron", "UTC") + + +def test_validate_cron_rejects_unknown_timezone() -> None: + """A non-IANA timezone string must be rejected at validation time — + the same protective gate as the cron expression itself.""" + with pytest.raises(InvalidCronError): + validate_cron("0 9 * * *", "Mars/Olympus_Mons") diff --git a/surfsense_backend/tests/unit/automations/triggers/schedule/test_params.py b/surfsense_backend/tests/unit/automations/triggers/schedule/test_params.py new file mode 100644 index 000000000..be98c5be1 --- /dev/null +++ b/surfsense_backend/tests/unit/automations/triggers/schedule/test_params.py @@ -0,0 +1,34 @@ +"""Lock the ``ScheduleTriggerParams`` validation contract.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from app.automations.triggers.schedule.params import ScheduleTriggerParams + +pytestmark = pytest.mark.unit + + +def test_schedule_params_accept_valid_cron_and_iana_timezone() -> None: + """A well-formed cron + IANA timezone yields a populated model. + Locks the round-trip path users go through when creating a trigger.""" + params = ScheduleTriggerParams(cron="0 9 * * 1-5", timezone="Africa/Kigali") + + assert params.cron == "0 9 * * 1-5" + assert params.timezone == "Africa/Kigali" + + +def test_schedule_params_reject_malformed_cron_with_validation_error() -> None: + """``InvalidCronError`` from ``validate_cron`` must surface as + Pydantic ``ValidationError`` so the FastAPI layer returns 422 instead + of letting the bad value reach storage.""" + with pytest.raises(ValidationError): + ScheduleTriggerParams(cron="not cron", timezone="UTC") + + +def test_schedule_params_reject_unknown_timezone_with_validation_error() -> None: + """An unknown timezone is rejected at the API boundary — same gate + as the cron expression itself.""" + with pytest.raises(ValidationError): + ScheduleTriggerParams(cron="0 9 * * *", timezone="Mars/Olympus_Mons") diff --git a/surfsense_backend/tests/unit/tasks/chat/streaming/test_parallel_refactor_parity.py b/surfsense_backend/tests/unit/tasks/chat/streaming/test_parallel_refactor_parity.py new file mode 100644 index 000000000..eb24b4df8 --- /dev/null +++ b/surfsense_backend/tests/unit/tasks/chat/streaming/test_parallel_refactor_parity.py @@ -0,0 +1,582 @@ +"""Parity gate for the parallel refactor of ``stream_new_chat.py``. + +The new tree under ``app.tasks.chat.streaming.flows`` is built side-by-side with +the legacy monolithic ``app.tasks.chat.stream_new_chat`` so we can cut over +atomically. This file pins externally-observable behaviour at module +boundaries so a divergence between the two trees fails loudly *before* the +cutover. + +What we verify: + + 1. **Signature parity** — ``stream_new_chat`` / ``stream_resume_chat`` from + the new tree have the same call signature as the originals. + 2. **Helper extraction parity** — the SRP modules in ``flows/`` produce the + same outputs as the inline code in the legacy file for representative + inputs (initial thinking step, image-capability gate, runtime context, + SSE frame sequences, token-usage frame shape, persistence guards). + 3. **Wrapper delegation** — wrappers like ``load_llm_bundle`` / + ``can_recover_provider_rate_limit`` exist and are addressable. + +Delete this file along with ``stream_new_chat.py`` once the cutover is done +(see the parent refactor plan). +""" + +from __future__ import annotations + +import asyncio +import inspect +from dataclasses import dataclass +from typing import Any +from unittest.mock import AsyncMock, patch + +import pytest + +from app.agents.new_chat.context import SurfSenseContextSchema +from app.services.new_streaming_service import VercelStreamingService + +from app.tasks.chat.stream_new_chat import ( + stream_new_chat as old_stream_new_chat, + stream_resume_chat as old_stream_resume_chat, +) +from app.tasks.chat.streaming.flows import ( + stream_new_chat as new_stream_new_chat, + stream_resume_chat as new_stream_resume_chat, +) +from app.tasks.chat.streaming.flows.new_chat.initial_thinking_step import ( + build_initial_thinking_step, +) +from app.tasks.chat.streaming.flows.new_chat.llm_capability import ( + check_image_input_capability, +) +from app.tasks.chat.streaming.flows.new_chat.persistence_spawn import ( + await_persist_task, + spawn_persist_assistant_shell_task, + spawn_persist_user_task, + spawn_set_ai_responding_bg, +) +from app.tasks.chat.streaming.flows.new_chat.runtime_context import ( + build_new_chat_runtime_context, +) +from app.tasks.chat.streaming.flows.resume_chat.runtime_context import ( + build_resume_chat_runtime_context, +) +from app.tasks.chat.streaming.flows.shared.finalize_emit import iter_token_usage_frame +from app.tasks.chat.streaming.flows.shared.first_frames import ( + iter_final_frames, + iter_initial_frames, +) +from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle +from app.tasks.chat.streaming.flows.shared.premium_quota import ( + PremiumReservation, + needs_premium_quota, +) +from app.tasks.chat.streaming.flows.shared.rate_limit_recovery import ( + can_recover_provider_rate_limit, +) + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------- signature + + +def _normalize_annotation(ann: Any) -> str: + """Compare-friendly form for an annotation. + + The legacy ``stream_new_chat.py`` does NOT use ``from __future__ import + annotations``, so its annotations are evaluated at import time and come + back as type objects / typing generics. The new tree DOES use it, so its + annotations are PEP-563 strings. + + Both reprs describe the same types — strip the module prefixes / typing + namespace + the ```` wrapper so we compare the canonical + declared form. + """ + if ann is inspect.Signature.empty: + return "" + raw = ann if isinstance(ann, str) else repr(ann) + cleaned = ( + raw.replace("typing.", "") + .replace("collections.abc.", "") + .replace("app.db.", "") + .replace("app.agents.new_chat.filesystem_selection.", "") + .replace("app.agents.new_chat.context.", "") + ) + # Unwrap ```` → ``int`` (legacy-side type objects). + if cleaned.startswith(""): + cleaned = cleaned[len("")] + return cleaned + + +def _normalize_sig(sig: inspect.Signature) -> list[tuple[str, Any, str]]: + return [ + (p.name, p.default, _normalize_annotation(p.annotation)) + for p in sig.parameters.values() + ] + + +def test_stream_new_chat_signature_matches_legacy() -> None: + old = inspect.signature(old_stream_new_chat) + new = inspect.signature(new_stream_new_chat) + assert _normalize_sig(new) == _normalize_sig(old) + assert _normalize_annotation(new.return_annotation) == _normalize_annotation( + old.return_annotation + ) + + +def test_stream_resume_chat_signature_matches_legacy() -> None: + old = inspect.signature(old_stream_resume_chat) + new = inspect.signature(new_stream_resume_chat) + assert _normalize_sig(new) == _normalize_sig(old) + assert _normalize_annotation(new.return_annotation) == _normalize_annotation( + old.return_annotation + ) + + +def test_orchestrators_are_async_generator_functions() -> None: + assert inspect.isasyncgenfunction(new_stream_new_chat) + assert inspect.isasyncgenfunction(new_stream_resume_chat) + + +# ------------------------------------------------------------ initial thinking + + +@dataclass +class _FakeSurfsenseDoc: + """Stand-in for ``SurfsenseDocsDocument`` with just the field we read.""" + + title: str + + +@pytest.mark.parametrize( + "user_query, image_urls, docs, expected_title, expected_action", + [ + ("hello world", None, [], "Understanding your request", "Processing"), + ("", ["data:image/png;base64,AAA"], [], "Understanding your request", "Processing"), + ("", None, [], "Understanding your request", "Processing"), + ( + "doc question", + None, + [_FakeSurfsenseDoc(title="My Doc")], + "Analyzing referenced content", + "Analyzing", + ), + ], +) +def test_initial_thinking_step_branches( + user_query: str, + image_urls: list[str] | None, + docs: list[Any], + expected_title: str, + expected_action: str, +) -> None: + step = build_initial_thinking_step( + user_query=user_query, + user_image_data_urls=image_urls, + mentioned_surfsense_docs=docs, # type: ignore[arg-type] + ) + assert step.step_id == "thinking-1" + assert step.title == expected_title + assert len(step.items) == 1 + assert step.items[0].startswith(f"{expected_action}: ") + + +def test_initial_thinking_step_truncates_long_query() -> None: + long_query = "x" * 200 + step = build_initial_thinking_step( + user_query=long_query, + user_image_data_urls=None, + mentioned_surfsense_docs=[], + ) + # 80-char truncation + ellipsis, sandwiched after "Processing: ". + assert "..." in step.items[0] + item = step.items[0] + payload = item[len("Processing: ") :] + assert payload.startswith("x" * 80) and payload.endswith("...") + + +def test_initial_thinking_step_collapses_many_doc_names() -> None: + docs = [_FakeSurfsenseDoc(title=f"Doc {i}") for i in range(5)] + step = build_initial_thinking_step( + user_query="q", + user_image_data_urls=None, + mentioned_surfsense_docs=docs, # type: ignore[arg-type] + ) + assert "[5 docs]" in step.items[0] + + +# ------------------------------------------------------------ capability gate + + +def test_image_capability_passes_without_images() -> None: + assert check_image_input_capability( + user_image_data_urls=None, agent_config=None + ) is None + + +def test_image_capability_passes_when_capability_unknown() -> None: + """Unknown / unmapped models are not blocked — only models LiteLLM has + *explicitly* marked text-only trip the gate.""" + + class _AgentConfig: + provider = "openrouter" + model_name = "unknown-mystery-model" + custom_provider = None + config_name = "Unknown" + litellm_params: dict[str, Any] = {} + + with patch( + "app.services.provider_capabilities.is_known_text_only_chat_model", + return_value=False, + ): + assert ( + check_image_input_capability( + user_image_data_urls=["data:image/png;base64,AAA"], + agent_config=_AgentConfig(), # type: ignore[arg-type] + ) + is None + ) + + +def test_image_capability_blocks_known_text_only_models() -> None: + class _AgentConfig: + provider = "openai" + model_name = "gpt-3.5-turbo" + custom_provider = None + config_name = "GPT-3.5" + litellm_params: dict[str, Any] = {"base_model": "gpt-3.5-turbo"} + + with patch( + "app.services.provider_capabilities.is_known_text_only_chat_model", + return_value=True, + ): + result = check_image_input_capability( + user_image_data_urls=["data:image/png;base64,AAA"], + agent_config=_AgentConfig(), # type: ignore[arg-type] + ) + assert result is not None + message, error_code = result + assert error_code == "MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT" + assert "GPT-3.5" in message + + +# ---------------------------------------------------------------- runtime ctx + + +def test_new_chat_runtime_context_prefers_accepted_folder_ids() -> None: + ctx = build_new_chat_runtime_context( + search_space_id=7, + mentioned_document_ids=[1, 2], + accepted_folder_ids=[10], + mentioned_folder_ids=[20, 30], + request_id="req", + turn_id="t1", + ) + assert isinstance(ctx, SurfSenseContextSchema) + assert ctx.search_space_id == 7 + assert list(ctx.mentioned_document_ids) == [1, 2] + assert list(ctx.mentioned_folder_ids) == [10] + assert ctx.request_id == "req" + assert ctx.turn_id == "t1" + + +def test_new_chat_runtime_context_falls_back_to_mentioned_folder_ids() -> None: + ctx = build_new_chat_runtime_context( + search_space_id=7, + mentioned_document_ids=None, + accepted_folder_ids=[], + mentioned_folder_ids=[20, 30], + request_id=None, + turn_id="t2", + ) + assert list(ctx.mentioned_folder_ids) == [20, 30] + + +def test_resume_chat_runtime_context_empty_mention_lists() -> None: + ctx = build_resume_chat_runtime_context( + search_space_id=42, request_id="req-r", turn_id="t-r" + ) + assert ctx.search_space_id == 42 + assert ctx.request_id == "req-r" + assert ctx.turn_id == "t-r" + + +# ---------------------------------------------------------------- SSE frames + + +def test_iter_initial_frames_emits_canonical_sequence() -> None: + svc = VercelStreamingService() + frames = list(iter_initial_frames(svc, turn_id="42:1700000000000")) + # Exactly 4 frames: message_start, start_step, turn-info (turn_id), turn-status (busy). + assert len(frames) == 4 + assert "42:1700000000000" in frames[2] + assert '"status":"busy"' in frames[3] or '"status": "busy"' in frames[3] + + +def test_iter_final_frames_emits_idle_then_finish_done() -> None: + svc = VercelStreamingService() + frames = list(iter_final_frames(svc)) + assert len(frames) == 4 + assert '"status":"idle"' in frames[0] or '"status": "idle"' in frames[0] + + +# ----------------------------------------------------------- token usage frame + + +class _FakeAccumulator: + """Minimal stand-in covering only the fields ``iter_token_usage_frame`` reads.""" + + def __init__(self, summary: Any = None) -> None: + self._summary = summary + self.calls = [1, 2, 3] + self.grand_total = 100 + self.total_cost_micros = 50_000 + self.total_prompt_tokens = 60 + self.total_completion_tokens = 40 + + def per_message_summary(self) -> Any: + return self._summary + + def serialized_calls(self) -> list[Any]: + return list(self.calls) + + +def test_token_usage_frame_skipped_when_no_summary() -> None: + svc = VercelStreamingService() + frames = list( + iter_token_usage_frame( + svc, + accumulator=_FakeAccumulator(summary=None), # type: ignore[arg-type] + log_label="parity-empty", + ) + ) + assert frames == [] + + +def test_token_usage_frame_emitted_when_summary_present() -> None: + svc = VercelStreamingService() + frames = list( + iter_token_usage_frame( + svc, + accumulator=_FakeAccumulator(summary=[{"m": "x", "t": 100}]), # type: ignore[arg-type] + log_label="parity-populated", + ) + ) + assert len(frames) == 1 + # Field shape on the wire is fixed by the FE; assert each surfaces. + payload = frames[0] + for key in ( + '"prompt_tokens":60', + '"completion_tokens":40', + '"total_tokens":100', + '"cost_micros":50000', + ): + assert key in payload.replace(" ", "") + + +# ------------------------------------------------------------------ llm_bundle + + +def test_load_llm_bundle_routes_negative_id_to_yaml_loader() -> None: + async def _run() -> tuple[Any, Any, str | None]: + with ( + patch( + "app.tasks.chat.streaming.flows.shared.llm_bundle.load_global_llm_config_by_id", + return_value=None, + ), + ): + return await load_llm_bundle( + session=AsyncMock(), # type: ignore[arg-type] + config_id=-1, + search_space_id=7, + ) + + llm, agent_config, error = asyncio.run(_run()) + assert llm is None + assert agent_config is None + assert error is not None and "id -1" in error + + +def test_load_llm_bundle_routes_nonnegative_id_to_db_loader() -> None: + async def _run() -> tuple[Any, Any, str | None]: + with ( + patch( + "app.tasks.chat.streaming.flows.shared.llm_bundle.load_agent_config", + new=AsyncMock(return_value=None), + ), + ): + return await load_llm_bundle( + session=AsyncMock(), # type: ignore[arg-type] + config_id=12, + search_space_id=7, + ) + + llm, agent_config, error = asyncio.run(_run()) + assert llm is None + assert agent_config is None + assert error is not None and "id 12" in error + + +# ----------------------------------------------------------------- premium quota + + +def test_needs_premium_quota_requires_user_and_premium_flag() -> None: + class _AgentConfig: + is_premium = True + + class _NonPremium: + is_premium = False + + assert needs_premium_quota(_AgentConfig(), "user-1") is True # type: ignore[arg-type] + assert needs_premium_quota(_AgentConfig(), None) is False # type: ignore[arg-type] + assert needs_premium_quota(_NonPremium(), "user-1") is False # type: ignore[arg-type] + assert needs_premium_quota(None, "user-1") is False + + +def test_premium_reservation_dataclass_shape() -> None: + # Sanity: the dataclass exists and carries the fields the orchestrator uses. + r = PremiumReservation(request_id="abc", reserved_micros=100, allowed=True) + assert r.request_id == "abc" + assert r.reserved_micros == 100 + assert r.allowed is True + + +# ----------------------------------------------------------- rate-limit guard + + +@pytest.mark.parametrize( + "first_event_seen, recovered, requested_id, current_id, expected", + [ + (False, False, 0, -1, True), + # Already recovered: no second pass. + (False, True, 0, -1, False), + # User explicitly picked a config: don't silently switch. + (False, False, 5, -1, False), + # Already on a database-backed (positive) id. + (False, False, 0, 7, False), + # User has already seen output: silent rebuild not possible. + (True, False, 0, -1, False), + ], +) +def test_can_recover_provider_rate_limit_truth_table( + first_event_seen: bool, + recovered: bool, + requested_id: int, + current_id: int, + expected: bool, +) -> None: + # Use a known rate-limit-shaped exception so the helper's last condition + # is satisfied; the guard only short-circuits to False when one of the + # *other* preconditions fails. + exc = Exception('{"error":{"type":"rate_limit_error","message":"slow"}}') + assert ( + can_recover_provider_rate_limit( + exc, + first_event_seen=first_event_seen, + runtime_rate_limit_recovered=recovered, + requested_llm_config_id=requested_id, + current_llm_config_id=current_id, + ) + is expected + ) + + +def test_can_recover_provider_rate_limit_rejects_non_rate_limit_exception() -> None: + assert ( + can_recover_provider_rate_limit( + ValueError("not a rate limit"), + first_event_seen=False, + runtime_rate_limit_recovered=False, + requested_llm_config_id=0, + current_llm_config_id=-1, + ) + is False + ) + + +# --------------------------------------------------------- persistence spawn + + +def test_spawn_set_ai_responding_bg_noop_without_user_id() -> None: + async def _run() -> set[asyncio.Task]: + background: set[asyncio.Task] = set() + spawn_set_ai_responding_bg( + chat_id=1, user_id=None, background_tasks=background + ) + return background + + bg = asyncio.run(_run()) + assert bg == set() + + +def test_spawn_persist_user_task_registers_and_self_unregisters() -> None: + async def _run() -> tuple[int, int]: + background: set[asyncio.Task] = set() + with patch( + "app.tasks.chat.streaming.flows.new_chat.persistence_spawn.persist_user_turn", + new=AsyncMock(return_value=99), + ): + task = spawn_persist_user_task( + chat_id=1, + user_id="u", + turn_id="t", + user_query="hi", + user_image_data_urls=None, + mentioned_documents=None, + background_tasks=background, + ) + size_before_await = len(background) + result = await asyncio.shield(task) + # Give the done-callback one event-loop tick to run. + await asyncio.sleep(0) + return size_before_await, result # type: ignore[return-value] + + size_before, result = asyncio.run(_run()) + assert size_before == 1 + assert result == 99 + + +def test_spawn_persist_assistant_shell_task_registers() -> None: + async def _run() -> int | None: + background: set[asyncio.Task] = set() + with patch( + "app.tasks.chat.streaming.flows.new_chat.persistence_spawn.persist_assistant_shell", + new=AsyncMock(return_value=42), + ): + task = spawn_persist_assistant_shell_task( + chat_id=1, + user_id="u", + turn_id="t", + background_tasks=background, + ) + return await asyncio.shield(task) + + assert asyncio.run(_run()) == 42 + + +def test_await_persist_task_returns_none_on_failure() -> None: + async def _run() -> int | None: + async def _boom() -> int: + raise RuntimeError("DB down") + + task = asyncio.create_task(_boom()) + return await await_persist_task( + task, + chat_id=1, + turn_id="t", + log_label="parity-failure", + ) + + assert asyncio.run(_run()) is None + + +def test_await_persist_task_returns_none_for_none_input() -> None: + async def _run() -> int | None: + return await await_persist_task( + None, + chat_id=1, + turn_id="t", + log_label="parity-none", + ) + + assert asyncio.run(_run()) is None diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index b902363dc..ba88153c5 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -1265,6 +1265,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/ca/6a667ccbe649856dcd3458bab80b016681b274399d6211187c6ab969fc50/courlan-1.3.2-py3-none-any.whl", hash = "sha256:d0dab52cf5b5b1000ee2839fbc2837e93b2514d3cb5bb61ae158a55b7a04c6be", size = 33848, upload-time = "2024-10-29T16:40:18.325Z" }, ] +[[package]] +name = "croniter" +version = "6.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/de/5832661ed55107b8a09af3f0a2e71e0957226a59eb1dcf0a445cce6daf20/croniter-6.2.2.tar.gz", hash = "sha256:ba60832a5ec8e12e51b8691c3309a113d1cf6526bdf1a48150ce8ec7a532d0ab", size = 113762, upload-time = "2026-03-15T08:43:48.112Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/39/783980e78cb92c2d7bdb1fc7dbc86e94ccc6d58224d76a7f1f51b6c51e30/croniter-6.2.2-py3-none-any.whl", hash = "sha256:a5d17b1060974d36251ea4faf388233eca8acf0d09cbd92d35f4c4ac8f279960", size = 45422, upload-time = "2026-03-15T08:43:46.626Z" }, +] + [[package]] name = "cryptography" version = "46.0.6" @@ -8132,6 +8144,7 @@ dependencies = [ { name = "celery", extra = ["redis"] }, { name = "chonkie", extra = ["all"] }, { name = "composio" }, + { name = "croniter" }, { name = "datasets" }, { name = "daytona" }, { name = "deepagents" }, @@ -8228,6 +8241,7 @@ requires-dist = [ { name = "celery", extras = ["redis"], specifier = ">=5.5.3" }, { name = "chonkie", extras = ["all"], specifier = ">=1.5.0" }, { name = "composio", specifier = ">=0.10.9" }, + { name = "croniter", specifier = ">=2.0.0" }, { name = "datasets", specifier = ">=2.21.0" }, { name = "daytona", specifier = ">=0.146.0" }, { name = "deepagents", specifier = ">=0.4.12,<0.5" }, diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/automation-detail-content.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/automation-detail-content.tsx new file mode 100644 index 000000000..4085d47a8 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/automation-detail-content.tsx @@ -0,0 +1,91 @@ +"use client"; +import { ShieldAlert } from "lucide-react"; +import { useAutomation } from "@/hooks/use-automation"; +import { useAutomationPermissions } from "../hooks/use-automation-permissions"; +import { AutomationDefinitionSection } from "./components/automation-definition-section"; +import { AutomationDetailHeader } from "./components/automation-detail-header"; +import { AutomationDetailLoading } from "./components/automation-detail-loading"; +import { AutomationNotFound } from "./components/automation-not-found"; +import { AutomationRunsSection } from "./components/automation-runs-section"; +import { AutomationTriggersSection } from "./components/automation-triggers-section"; + +interface AutomationDetailContentProps { + searchSpaceId: number; + automationId: number; +} + +/** + * Client orchestrator for one automation's detail view. Branches: + * - permissions loading → skeleton + * - no read permission → access denied panel + * - bad id (NaN) → not-found panel + * - detail fetching → skeleton + * - detail error / null → not-found panel (we don't distinguish 404 + * from 403 in the UI) + * - detail loaded → header + definition + triggers + * + * Each child component is gated independently on the relevant permission + * so the orchestrator stays thin. + */ +export function AutomationDetailContent({ + searchSpaceId, + automationId, +}: AutomationDetailContentProps) { + const perms = useAutomationPermissions(); + const validId = Number.isInteger(automationId) && automationId > 0; + const { data: automation, isLoading, error } = useAutomation(validId ? automationId : undefined); + + if (perms.loading) { + return ; + } + + if (!perms.canRead) { + return ( +
+ +

Access denied

+

+ You don't have permission to view automations in this search space. +

+
+ ); + } + + if (!validId) { + return ; + } + + if (isLoading) { + return ; + } + + if (error || !automation) { + return ; + } + + return ( + <> + + +
+
+ + +
+
+ +
+
+ + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx new file mode 100644 index 000000000..4ff9b8b8c --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx @@ -0,0 +1,98 @@ +"use client"; +import { ListOrdered, Settings2, Tag, Target } from "lucide-react"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { AutomationDefinition } from "@/contracts/types/automation.types"; +import { ExecutionSummary } from "./execution-summary"; +import { InputsSchemaPreview } from "./inputs-schema-preview"; +import { PlanStepCard } from "./plan-step-card"; + +interface AutomationDefinitionSectionProps { + definition: AutomationDefinition; +} + +/** + * The Definition card. Read view; editing happens on the sibling /edit + * route (Edit button in the header). Layout is top-down: + * goal → tags → execution defaults → inputs schema (if any) → plan + * + * The schema_version is rendered as a small badge next to the section + * title so it's discoverable but doesn't fight for attention. + */ +export function AutomationDefinitionSection({ definition }: AutomationDefinitionSectionProps) { + const hasTags = definition.metadata.tags.length > 0; + const hasInputs = !!definition.inputs; + + return ( + + + Definition + + v{definition.schema_version} + + + + {definition.goal && ( + +

{definition.goal}

+ + )} + + {hasTags && ( + +
+ {definition.metadata.tags.map((tag) => ( + + {tag} + + ))} +
+ + )} + + + + + + {hasInputs && ( + + {definition.inputs && } + + )} + + +
+ {definition.plan.map((step, idx) => ( + + ))} +
+ + + + ); +} + +function Field({ + icon: Icon, + label, + children, +}: { + icon: typeof Target; + label: string; + children: React.ReactNode; +}) { + return ( +
+
+ + {label} +
+ {children} +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx new file mode 100644 index 000000000..0bce3fa2d --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx @@ -0,0 +1,137 @@ +"use client"; +import { useAtomValue } from "jotai"; +import { ArrowLeft, Pause, Pencil, Play, Trash2 } from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useCallback, useState } from "react"; +import { updateAutomationMutationAtom } from "@/atoms/automations/automations-mutation.atoms"; +import { Button } from "@/components/ui/button"; +import { Spinner } from "@/components/ui/spinner"; +import type { Automation } from "@/contracts/types/automation.types"; +import { AutomationStatusBadge } from "../../components/automation-status-badge"; +import { DeleteAutomationDialog } from "../../components/delete-automation-dialog"; + +interface AutomationDetailHeaderProps { + automation: Automation; + searchSpaceId: number; + canUpdate: boolean; + canDelete: boolean; +} + +/** + * Title bar for the detail page: back link, name, status badge, + * description, and the two destructive-ish primary actions (pause / + * resume + delete). Same mutation atoms as the list-row actions to + * keep caches coherent. + * + * Archived automations hide the pause/resume toggle (we don't unarchive + * here — that flow comes later if we need it). + */ +export function AutomationDetailHeader({ + automation, + searchSpaceId, + canUpdate, + canDelete, +}: AutomationDetailHeaderProps) { + const router = useRouter(); + const { mutateAsync: updateAutomation, isPending: updating } = useAtomValue( + updateAutomationMutationAtom + ); + const [deleteOpen, setDeleteOpen] = useState(false); + + const canToggle = canUpdate && automation.status !== "archived"; + const nextStatus = automation.status === "active" ? "paused" : "active"; + const pauseLabel = automation.status === "active" ? "Pause" : "Resume"; + const PauseIcon = automation.status === "active" ? Pause : Play; + + const handleDeleted = useCallback(() => { + router.push(`/dashboard/${searchSpaceId}/automations`); + }, [router, searchSpaceId]); + + async function handleTogglePause() { + await updateAutomation({ + automationId: automation.id, + patch: { status: nextStatus }, + }); + } + + return ( + <> +
+ + +
+
+
+

+ {automation.name} +

+ +
+ {automation.description && ( +

{automation.description}

+ )} +
+ +
+ {canUpdate && ( + + )} + {canToggle && ( + + )} + {canDelete && ( + + )} +
+
+
+ + {canDelete && ( + + )} + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-loading.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-loading.tsx new file mode 100644 index 000000000..0d6ba3110 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-loading.tsx @@ -0,0 +1,56 @@ +"use client"; +import { Card, CardContent, CardHeader } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; + +/** + * Skeleton for the detail page. Mirrors the loaded view's main/sidebar + * grid (Definition + Runs on the left, Triggers on the right) so layout + * doesn't reflow when data arrives. + */ +export function AutomationDetailLoading() { + return ( + <> +
+ +
+ + +
+ +
+ +
+
+ + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + +
+
+ + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-not-found.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-not-found.tsx new file mode 100644 index 000000000..1681caf25 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-not-found.tsx @@ -0,0 +1,34 @@ +"use client"; +import { ArrowLeft, FileWarning } from "lucide-react"; +import Link from "next/link"; +import { Button } from "@/components/ui/button"; + +interface AutomationNotFoundProps { + searchSpaceId: number; + error?: Error | null; +} + +/** + * Rendered when the detail fetch fails (404 / 403 / network) or the id + * is not a number. We don't distinguish "missing" from "forbidden" in the + * UI on purpose — leaking that an id exists you can't read is worse than + * a vague message. + */ +export function AutomationNotFound({ searchSpaceId, error }: AutomationNotFoundProps) { + return ( +
+ +

Automation not found

+

+ This automation doesn't exist or you don't have access to it. + {error?.message ? ` (${error.message})` : null} +

+ +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx new file mode 100644 index 000000000..d31bd696d --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx @@ -0,0 +1,67 @@ +"use client"; +import { History } from "lucide-react"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { useAutomationRuns } from "@/hooks/use-automation-runs"; +import { RunRow } from "./run-row"; +import { RunsLoading } from "./runs-loading"; + +interface AutomationRunsSectionProps { + automationId: number; +} + +const LIMIT = 20; + +/** + * Run history card. Shows the most recent ``LIMIT`` runs; pagination is + * intentionally deferred — for the foreseeable v1 surface (one-trigger + * automations firing daily), 20 covers ~3 weeks of history which is + * enough to tell whether things are working. Real "load more" lands if + * we see usage spike past that. + */ +export function AutomationRunsSection({ automationId }: AutomationRunsSectionProps) { + const { data, isLoading, error } = useAutomationRuns(automationId, { limit: LIMIT }); + const runs = data?.items ?? []; + + return ( + + +
+ + + Recent runs + +

+ Most recent first. Click a row to inspect step results, output and artifacts. +

+
+ {!isLoading && !error && data && ( + {data.total} total + )} + + + {isLoading ? ( + + ) : error ? ( +

+ Couldn't load runs{error.message ? `: ${error.message}` : "."} +

+ ) : runs.length === 0 ? ( +
+ +

No runs yet

+

+ This automation hasn't fired. Once a trigger fires (or you invoke it manually), runs + will appear here. +

+
+ ) : ( +
+ {runs.map((run) => ( + + ))} +
+ )} + + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx new file mode 100644 index 000000000..558a089ac --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx @@ -0,0 +1,58 @@ +"use client"; +import { CalendarClock } from "lucide-react"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { Trigger } from "@/contracts/types/automation.types"; +import { TriggerCard } from "./trigger-card"; + +interface AutomationTriggersSectionProps { + triggers: Trigger[]; + automationId: number; + canUpdate: boolean; + canDelete: boolean; +} + +/** + * The Triggers card. Lists each attached trigger with its own enable + * toggle and remove button. v1 attaches triggers at automation-creation + * time only; there is no in-place "add trigger" affordance here. + */ +export function AutomationTriggersSection({ + triggers, + automationId, + canUpdate, + canDelete, +}: AutomationTriggersSectionProps) { + return ( + + + Triggers +

+ When this automation fires. v1 supports scheduled triggers only. +

+ + + {triggers.length === 0 ? ( +
+ +

No triggers attached

+

+ This automation can still be invoked, but nothing will fire it on its own. +

+
+ ) : ( +
+ {triggers.map((trigger) => ( + + ))} +
+ )} + + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/delete-trigger-dialog.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/delete-trigger-dialog.tsx new file mode 100644 index 000000000..71e905724 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/delete-trigger-dialog.tsx @@ -0,0 +1,80 @@ +"use client"; +import { useAtomValue } from "jotai"; +import { useState } from "react"; +import { removeTriggerMutationAtom } from "@/atoms/automations/automations-mutation.atoms"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Spinner } from "@/components/ui/spinner"; + +interface DeleteTriggerDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + automationId: number; + triggerId: number; + triggerLabel: string; +} + +/** + * Confirm + detach one trigger from its automation. The automation itself + * is untouched; only this trigger row is removed. The mutation atom + * invalidates the parent automation detail so the page rerenders. + */ +export function DeleteTriggerDialog({ + open, + onOpenChange, + automationId, + triggerId, + triggerLabel, +}: DeleteTriggerDialogProps) { + const { mutateAsync: removeTrigger } = useAtomValue(removeTriggerMutationAtom); + const [submitting, setSubmitting] = useState(false); + + async function handleConfirm() { + setSubmitting(true); + try { + await removeTrigger({ automationId, triggerId }); + onOpenChange(false); + } finally { + setSubmitting(false); + } + } + + return ( + + + + Remove this trigger? + + {triggerLabel} will be detached. + The automation itself stays, but it won't fire on this trigger anymore. + + + + Cancel + + {submitting ? ( + + + Removing… + + ) : ( + "Remove" + )} + + + + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx new file mode 100644 index 000000000..5c4dc381c --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx @@ -0,0 +1,37 @@ +"use client"; +import type { Execution } from "@/contracts/types/automation.types"; + +interface ExecutionSummaryProps { + execution: Execution; +} + +/** + * Compact view of an automation's execution defaults (wall-clock cap, + * retries, backoff, concurrency, on_failure presence). Per-step overrides + * are shown inside each PlanStepCard, not here. + */ +export function ExecutionSummary({ execution }: ExecutionSummaryProps) { + return ( +
+ + + + + {execution.on_failure.length > 0 && ( + + )} +
+ ); +} + +function Item({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx new file mode 100644 index 000000000..29d79d99b --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx @@ -0,0 +1,21 @@ +"use client"; +import { JsonView } from "@/components/json-view"; +import type { Inputs } from "@/contracts/types/automation.types"; + +interface InputsSchemaPreviewProps { + inputs: Inputs; +} + +/** + * Read-only preview of an automation's accepted-inputs schema. Most + * automations don't define inputs (defaults are baked into the trigger's + * static_inputs), so the parent skips rendering this card when ``inputs`` + * is null. + */ +export function InputsSchemaPreview({ inputs }: InputsSchemaPreviewProps) { + return ( +
+ +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx new file mode 100644 index 000000000..27cecf3bf --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx @@ -0,0 +1,74 @@ +"use client"; +import { ArrowRightCircle, GitCommitHorizontal } from "lucide-react"; +import { JsonView } from "@/components/json-view"; +import type { PlanStep } from "@/contracts/types/automation.types"; + +interface PlanStepCardProps { + step: PlanStep; + index: number; +} + +/** + * Read-only view of one plan step. Renders the step_id + action prominently, + * then a definition list of the per-step knobs, and finally the params as + * formatted JSON. Editable mode is out of scope here — definition edits live + * on the (future) raw-JSON path. + */ +export function PlanStepCard({ step, index }: PlanStepCardProps) { + return ( +
+
+ + {index + 1} + + {step.step_id} + + {step.action} +
+ +
+ {(step.when || + step.output_as || + step.max_retries != null || + step.timeout_seconds != null) && ( +
+ {step.when && ( + {step.when}} /> + )} + {step.output_as && ( + {step.output_as}} + /> + )} + {step.max_retries != null && ( + + )} + {step.timeout_seconds != null && ( + + )} +
+ )} + +
+
+ + Params +
+
+ +
+
+
+
+ ); +} + +function DefRow({ label, value }: { label: string; value: React.ReactNode }) { + return ( +
+
{label}:
+
{value}
+
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx new file mode 100644 index 000000000..f9c6fbb5a --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx @@ -0,0 +1,117 @@ +"use client"; +import { AlertCircle, FileOutput, GitCommitHorizontal, Package, Settings2 } from "lucide-react"; +import { JsonView } from "@/components/json-view"; +import { Skeleton } from "@/components/ui/skeleton"; +import { useAutomationRun } from "@/hooks/use-automation-runs"; + +interface RunDetailsPanelProps { + automationId: number; + runId: number; +} + +/** + * Expanded view of a single run. Fetches lazily — the parent only renders + * this once the row is opened, so the list view stays cheap. + * + * We surface the four most actionable sections (error first when present, + * then output, step results, artifacts, inputs). The full + * ``definition_snapshot`` is omitted because it usually mirrors the live + * definition — surfacing it would dominate the panel without informing + * what the user is trying to learn ("did this work? what did it do?"). + */ +export function RunDetailsPanel({ automationId, runId }: RunDetailsPanelProps) { + const { data: run, isLoading, error } = useAutomationRun(automationId, runId); + + if (isLoading) { + return ( +
+ + +
+ ); + } + + if (error || !run) { + return ( +
+ Couldn't load run details{error?.message ? `: ${error.message}` : "."} +
+ ); + } + + const hasError = run.error && Object.keys(run.error).length > 0; + const hasOutput = run.output && Object.keys(run.output).length > 0; + const hasInputs = Object.keys(run.inputs ?? {}).length > 0; + + return ( +
+ {hasError && ( +
+ +
+ )} + + {hasOutput && ( +
+ +
+ )} + +
+ {run.step_results.length === 0 ? ( +

No steps recorded.

+ ) : ( + + )} +
+ + {run.artifacts.length > 0 && ( +
+ +
+ )} + + {hasInputs && ( +
+ +
+ )} +
+ ); +} + +function Section({ + icon: Icon, + label, + tone = "default", + children, +}: { + icon: typeof AlertCircle; + label: string; + tone?: "default" | "destructive"; + children: React.ReactNode; +}) { + return ( +
+
+ + {label} +
+ {children} +
+ ); +} + +function JsonBlock({ value }: { value: unknown }) { + return ( +
+ +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx new file mode 100644 index 000000000..02ca0569c --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx @@ -0,0 +1,75 @@ +"use client"; +import { ChevronDown, ChevronRight, Hand } from "lucide-react"; +import { useState } from "react"; +import type { RunSummary } from "@/contracts/types/automation.types"; +import { formatRelativeDate } from "@/lib/format-date"; +import { RunDetailsPanel } from "./run-details-panel"; +import { RunStatusBadge } from "./run-status-badge"; + +interface RunRowProps { + run: RunSummary; + automationId: number; +} + +/** + * One run row. Click to expand → fetches the full run and shows the + * details panel inline. State is local to each row so multiple panels + * can be open at once (or none). + */ +export function RunRow({ run, automationId }: RunRowProps) { + const [open, setOpen] = useState(false); + const duration = computeDuration(run.started_at, run.finished_at); + const startedLabel = run.started_at + ? formatRelativeDate(run.started_at) + : formatRelativeDate(run.created_at); + + return ( +
+ + + {open && } +
+ ); +} + +function TriggerSource({ triggerId }: { triggerId: number | null }) { + if (triggerId == null) { + return ( + + + Manual + + ); + } + return via trigger #{triggerId}; +} + +function computeDuration(started: string | null | undefined, finished: string | null | undefined) { + if (!started || !finished) return null; + const ms = new Date(finished).getTime() - new Date(started).getTime(); + if (!Number.isFinite(ms) || ms < 0) return null; + if (ms < 1000) return `${ms}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + const minutes = Math.floor(ms / 60_000); + const seconds = Math.floor((ms % 60_000) / 1000); + return `${minutes}m ${seconds}s`; +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-status-badge.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-status-badge.tsx new file mode 100644 index 000000000..e5532a500 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-status-badge.tsx @@ -0,0 +1,57 @@ +"use client"; +import { AlertCircle, CheckCircle2, Clock, Loader2, TimerOff, XCircle } from "lucide-react"; +import type { RunStatus } from "@/contracts/types/automation.types"; +import { cn } from "@/lib/utils"; + +const STATUS_STYLES: Record< + RunStatus, + { label: string; icon: typeof CheckCircle2; classes: string; spin?: boolean } +> = { + pending: { + label: "Pending", + icon: Clock, + classes: "bg-muted text-muted-foreground border-border/60", + }, + running: { + label: "Running", + icon: Loader2, + classes: "bg-blue-500/10 text-blue-600 border-blue-500/20", + spin: true, + }, + succeeded: { + label: "Succeeded", + icon: CheckCircle2, + classes: "bg-emerald-500/10 text-emerald-600 border-emerald-500/20", + }, + failed: { + label: "Failed", + icon: XCircle, + classes: "bg-destructive/10 text-destructive border-destructive/20", + }, + cancelled: { + label: "Cancelled", + icon: AlertCircle, + classes: "bg-muted text-muted-foreground border-border/60", + }, + timed_out: { + label: "Timed out", + icon: TimerOff, + classes: "bg-amber-500/10 text-amber-600 border-amber-500/20", + }, +}; + +export function RunStatusBadge({ status, className }: { status: RunStatus; className?: string }) { + const { label, icon: Icon, classes, spin } = STATUS_STYLES[status]; + return ( + + + {label} + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/runs-loading.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/runs-loading.tsx new file mode 100644 index 000000000..61ce25e32 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/runs-loading.tsx @@ -0,0 +1,23 @@ +"use client"; +import { Skeleton } from "@/components/ui/skeleton"; + +const ROW_KEYS = ["a", "b", "c"] as const; + +export function RunsLoading() { + return ( +
+ {ROW_KEYS.map((key) => ( +
+
+ + +
+ +
+ ))} +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx new file mode 100644 index 000000000..681877523 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx @@ -0,0 +1,274 @@ +"use client"; +import { useAtomValue } from "jotai"; +import { AlertCircle, CalendarClock, Clock, Pencil, Save, Trash2 } from "lucide-react"; +import { useState } from "react"; +import { updateTriggerMutationAtom } from "@/atoms/automations/automations-mutation.atoms"; +import { JsonView } from "@/components/json-view"; +import { Button } from "@/components/ui/button"; +import { Spinner } from "@/components/ui/spinner"; +import { Switch } from "@/components/ui/switch"; +import { type Trigger, triggerUpdateRequest } from "@/contracts/types/automation.types"; +import { describeCron } from "@/lib/automations/describe-cron"; +import { formatRelativeDate, formatRelativeFutureDate } from "@/lib/format-date"; +import { DeleteTriggerDialog } from "./delete-trigger-dialog"; + +interface TriggerCardProps { + trigger: Trigger; + automationId: number; + canUpdate: boolean; + canDelete: boolean; +} + +interface TriggerDraft { + params: Record; + static_inputs: Record; +} + +function draftFromTrigger(trigger: Trigger): TriggerDraft { + return { + params: trigger.params, + static_inputs: trigger.static_inputs ?? {}, + }; +} + +/** + * One trigger row in the Triggers section of the detail page. Renders: + * - type icon + human-readable schedule + timezone + * - last_fired_at / next_fire_at hints + * - static_inputs as formatted JSON (when present) + * - enable toggle + remove button + inline edit (each gated independently) + * + * Inline edit covers ``params`` and ``static_inputs`` — the two fields the + * backend ``PATCH /triggers/[id]`` endpoint accepts beyond ``enabled``. + * ``enabled`` stays on the Switch so the two surfaces don't fight. + */ +export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: TriggerCardProps) { + const { mutateAsync: updateTrigger, isPending: updating } = + useAtomValue(updateTriggerMutationAtom); + const [deleteOpen, setDeleteOpen] = useState(false); + const [isEditing, setIsEditing] = useState(false); + const [draft, setDraft] = useState(() => draftFromTrigger(trigger)); + const [issues, setIssues] = useState([]); + + const cron = typeof trigger.params.cron === "string" ? trigger.params.cron : undefined; + const tz = typeof trigger.params.timezone === "string" ? trigger.params.timezone : "UTC"; + const human = cron ? describeCron(cron) : trigger.type; + const triggerLabel = cron ? `${human} · ${tz}` : trigger.type; + const hasStaticInputs = Object.keys(trigger.static_inputs ?? {}).length > 0; + + async function handleToggle(checked: boolean) { + await updateTrigger({ + automationId, + triggerId: trigger.id, + patch: { enabled: checked }, + }); + } + + function startEdit() { + setDraft(draftFromTrigger(trigger)); + setIssues([]); + setIsEditing(true); + } + + function cancelEdit() { + setIsEditing(false); + setIssues([]); + } + + async function saveEdit() { + setIssues([]); + const result = triggerUpdateRequest.safeParse(draft); + if (!result.success) { + setIssues( + result.error.issues.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) + ); + return; + } + try { + await updateTrigger({ + automationId, + triggerId: trigger.id, + patch: result.data, + }); + setIsEditing(false); + } catch (err) { + setIssues([(err as Error).message ?? "Update failed"]); + } + } + + return ( + <> +
+
+
+ +
+
+ {human} + · {tz} +
+ {cron && {cron}} +
+
+ +
+ {canUpdate && ( +
+ + {trigger.enabled ? "Enabled" : "Off"} + + +
+ )} + {canUpdate && !isEditing && ( + + )} + {canDelete && ( + + )} +
+
+ +
+ {isEditing ? ( + <> +
+ setDraft(next as TriggerDraft)} + collapsed={false} + /> +
+ + {issues.length > 0 && ( +
+
+ + {issues.length === 1 ? "1 issue" : `${issues.length} issues`} +
+
    + {issues.map((issue) => ( +
  • {issue}
    • + ))} +
+
+ )} + +
+ + +
+ + ) : ( + <> + {(trigger.last_fired_at || trigger.next_fire_at) && ( +
+ {trigger.next_fire_at && ( + + )} + {trigger.last_fired_at && ( + + )} +
+ )} + + {hasStaticInputs && ( +
+
Static inputs
+
+ +
+
+ )} + + )} +
+
+ + {canDelete && ( + + )} + + ); +} + +function TimeRow({ + label, + iso, + tense, + highlight = false, +}: { + label: string; + iso: string; + tense: "past" | "future"; + highlight?: boolean; +}) { + const formatted = tense === "future" ? formatRelativeFutureDate(iso) : formatRelativeDate(iso); + return ( + <> +
+ + {label} +
+
+ {formatted} +
+ + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx new file mode 100644 index 000000000..219552a1a --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx @@ -0,0 +1,56 @@ +"use client"; +import { ShieldAlert } from "lucide-react"; +import { useAutomation } from "@/hooks/use-automation"; +import { useAutomationPermissions } from "../../hooks/use-automation-permissions"; +import { AutomationDetailLoading } from "../components/automation-detail-loading"; +import { AutomationNotFound } from "../components/automation-not-found"; +import { AutomationEditForm } from "./components/automation-edit-form"; + +interface AutomationEditContentProps { + searchSpaceId: number; + automationId: number; +} + +/** + * Client orchestrator for the edit route. Mirrors detail-content's branch + * structure but gates on ``canUpdate`` instead of ``canRead``: a user who + * can read but not update is bounced to the access-denied panel. + */ +export function AutomationEditContent({ + searchSpaceId, + automationId, +}: AutomationEditContentProps) { + const perms = useAutomationPermissions(); + const validId = Number.isInteger(automationId) && automationId > 0; + const { data: automation, isLoading, error } = useAutomation(validId ? automationId : undefined); + + if (perms.loading) { + return ; + } + + if (!perms.canUpdate) { + return ( +
+ +

Access denied

+

+ You don't have permission to edit automations in this search space. +

+
+ ); + } + + if (!validId) { + return ; + } + + if (isLoading) { + return ; + } + + if (error || !automation) { + return ; + } + + return ; +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-form.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-form.tsx new file mode 100644 index 000000000..86b355838 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-form.tsx @@ -0,0 +1,121 @@ +"use client"; +import { useAtomValue } from "jotai"; +import { AlertCircle, ArrowLeft, Save } from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; +import { updateAutomationMutationAtom } from "@/atoms/automations/automations-mutation.atoms"; +import { JsonView } from "@/components/json-view"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Spinner } from "@/components/ui/spinner"; +import { + type Automation, + automationUpdateRequest, +} from "@/contracts/types/automation.types"; + +interface AutomationEditFormProps { + automation: Automation; + searchSpaceId: number; +} + +/** + * Edit-existing-automation form. Surfaces the four mutable fields + * (name, description, status, definition) as one editable JSON tree; + * triggers stay on the detail page where they have their own management + * UI. Validates with the same Zod schema the API expects, then PATCHes + * the changed shape back. + */ +export function AutomationEditForm({ automation, searchSpaceId }: AutomationEditFormProps) { + const router = useRouter(); + const { mutateAsync: updateAutomation, isPending } = useAtomValue(updateAutomationMutationAtom); + const detailHref = `/dashboard/${searchSpaceId}/automations/${automation.id}`; + + const [value, setValue] = useState(() => ({ + name: automation.name, + description: automation.description ?? null, + status: automation.status, + definition: automation.definition, + })); + const [issues, setIssues] = useState([]); + + async function handleSave() { + setIssues([]); + const result = automationUpdateRequest.safeParse(value); + if (!result.success) { + setIssues( + result.error.issues.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) + ); + return; + } + try { + await updateAutomation({ automationId: automation.id, patch: result.data }); + router.push(detailHref); + } catch (err) { + setIssues([(err as Error).message ?? "Update failed"]); + } + } + + return ( + <> +
+ +
+

+ Edit automation +

+

{automation.name}

+
+
+ + + + Definition + + +
+ setValue(next as typeof value)} + collapsed={false} + /> +
+ + {issues.length > 0 && ( +
+
+ + {issues.length === 1 ? "1 issue" : `${issues.length} issues`} +
+
    + {issues.map((issue) => ( +
  • {issue}
    • + ))} +
+
+ )} + +
+ + +
+ + + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/page.tsx new file mode 100644 index 000000000..8477b9e12 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/page.tsx @@ -0,0 +1,18 @@ +import { AutomationEditContent } from "./automation-edit-content"; + +export default async function AutomationEditPage({ + params, +}: { + params: Promise<{ search_space_id: string; automation_id: string }>; +}) { + const { search_space_id, automation_id } = await params; + + return ( +
+ +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/page.tsx new file mode 100644 index 000000000..dbaceecdd --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/page.tsx @@ -0,0 +1,18 @@ +import { AutomationDetailContent } from "./automation-detail-content"; + +export default async function AutomationDetailPage({ + params, +}: { + params: Promise<{ search_space_id: string; automation_id: string }>; +}) { + const { search_space_id, automation_id } = await params; + + return ( +
+ +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx new file mode 100644 index 000000000..756221d38 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx @@ -0,0 +1,102 @@ +"use client"; +import { ShieldAlert } from "lucide-react"; +import { useAutomations } from "@/hooks/use-automations"; +import { AutomationsEmptyState } from "./components/automations-empty-state"; +import { AutomationsHeader } from "./components/automations-header"; +import { AutomationsTable } from "./components/automations-table"; +import { useAutomationPermissions } from "./hooks/use-automation-permissions"; + +interface AutomationsContentProps { + searchSpaceId: number; +} + +/** + * Client orchestrator for the automations list page. Pulls the active + * search space's first page (via ``useAutomations`` → ``automationsListAtom``) + * and the user's permissions, then decides between empty / loading / table. + * + * Read access is mandatory; anything else is hidden behind RBAC. The + * permissions hook is co-located in this slice so adding/removing + * surfaces is a one-file change. + */ +export function AutomationsContent({ searchSpaceId }: AutomationsContentProps) { + const { automations, total, loading, error } = useAutomations(); + const perms = useAutomationPermissions(); + + if (perms.loading) { + // Permissions gate the entire page; defer everything until we know. + return ( + <> + + + + ); + } + + if (!perms.canRead) { + return ( +
+ +

Access denied

+

+ You don't have permission to view automations in this search space. +

+
+ ); + } + + if (error) { + return ( + <> + +
+

Couldn't load automations. {error.message}

+
+ + ); + } + + if (!loading && automations.length === 0) { + return ( + <> + + + + ); + } + + return ( + <> + + + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx new file mode 100644 index 000000000..229a417dc --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx @@ -0,0 +1,98 @@ +"use client"; +import { useAtomValue } from "jotai"; +import { MoreHorizontal, Pause, Play, Trash2 } from "lucide-react"; +import { useState } from "react"; +import { updateAutomationMutationAtom } from "@/atoms/automations/automations-mutation.atoms"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import type { AutomationSummary } from "@/contracts/types/automation.types"; +import { DeleteAutomationDialog } from "./delete-automation-dialog"; + +interface AutomationRowActionsProps { + automation: AutomationSummary; + searchSpaceId: number; + canUpdate: boolean; + canDelete: boolean; +} + +/** + * Three-dot menu on each row: pause/resume (if updatable) and delete + * (if deletable). The menu itself is hidden when the user has neither + * permission so we don't render an empty trigger. + */ +export function AutomationRowActions({ + automation, + searchSpaceId, + canUpdate, + canDelete, +}: AutomationRowActionsProps) { + const { mutateAsync: updateAutomation, isPending: updating } = useAtomValue( + updateAutomationMutationAtom + ); + const [deleteOpen, setDeleteOpen] = useState(false); + + if (!canUpdate && !canDelete) return null; + + const nextStatus = automation.status === "active" ? "paused" : "active"; + const pauseLabel = automation.status === "active" ? "Pause" : "Resume"; + const PauseIcon = automation.status === "active" ? Pause : Play; + const canToggle = canUpdate && automation.status !== "archived"; + + async function handleTogglePause() { + await updateAutomation({ + automationId: automation.id, + patch: { status: nextStatus }, + }); + } + + return ( + <> + + + + + + {canToggle && ( + + + {pauseLabel} + + )} + {canToggle && canDelete && } + {canDelete && ( + setDeleteOpen(true)} + className="text-destructive focus:text-destructive" + > + + Delete + + )} + + + + {canDelete && ( + + )} + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx new file mode 100644 index 000000000..a59fb4527 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx @@ -0,0 +1,61 @@ +"use client"; +import Link from "next/link"; +import { TableCell, TableRow } from "@/components/ui/table"; +import type { AutomationSummary } from "@/contracts/types/automation.types"; +import { formatRelativeDate } from "@/lib/format-date"; +import { AutomationRowActions } from "./automation-row-actions"; +import { AutomationStatusBadge } from "./automation-status-badge"; + +interface AutomationRowProps { + automation: AutomationSummary; + searchSpaceId: number; + canUpdate: boolean; + canDelete: boolean; +} + +/** + * One row in the automations table. The name links to the detail page; + * actions are gated by ``canUpdate`` / ``canDelete``. Trigger summary + * is intentionally left to the detail page — list responses don't + * include triggers and we want to avoid N+1 detail fetches. + */ +export function AutomationRow({ + automation, + searchSpaceId, + canUpdate, + canDelete, +}: AutomationRowProps) { + return ( + + +
+ + {automation.name} + + {automation.description && ( + + {automation.description} + + )} +
+ + + + + + {formatRelativeDate(automation.updated_at)} + + + + + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx new file mode 100644 index 000000000..ecf171e78 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx @@ -0,0 +1,49 @@ +"use client"; +import { Archive, CircleDot, Pause } from "lucide-react"; +import type { AutomationStatus } from "@/contracts/types/automation.types"; +import { cn } from "@/lib/utils"; + +interface AutomationStatusBadgeProps { + status: AutomationStatus; + className?: string; +} + +// Color + icon per status. Active = green, paused = amber, archived = muted. +const STATUS_STYLES: Record< + AutomationStatus, + { label: string; icon: typeof CircleDot; classes: string } +> = { + active: { + label: "Active", + icon: CircleDot, + classes: + "bg-emerald-50 text-emerald-700 border border-emerald-200 dark:bg-emerald-950/40 dark:text-emerald-300 dark:border-emerald-900/50", + }, + paused: { + label: "Paused", + icon: Pause, + classes: + "bg-amber-50 text-amber-700 border border-amber-200 dark:bg-amber-950/40 dark:text-amber-300 dark:border-amber-900/50", + }, + archived: { + label: "Archived", + icon: Archive, + classes: "bg-muted text-muted-foreground border border-border/60", + }, +}; + +export function AutomationStatusBadge({ status, className }: AutomationStatusBadgeProps) { + const { label, icon: Icon, classes } = STATUS_STYLES[status]; + return ( + + + {label} + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-triggers-summary.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-triggers-summary.tsx new file mode 100644 index 000000000..270a1f844 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-triggers-summary.tsx @@ -0,0 +1,52 @@ +"use client"; +import { CalendarClock, Pause } from "lucide-react"; +import type { Trigger } from "@/contracts/types/automation.types"; +import { describeCron } from "@/lib/automations/describe-cron"; + +interface AutomationTriggersSummaryProps { + triggers: Trigger[]; +} + +/** + * One-line summary of an automation's triggers for the list view. + * + * v1 only registers ``schedule`` so this stays compact: + * - 0 triggers → "No triggers" + * - 1 schedule trigger → "Mon–Fri at 09:00 · UTC" + disabled badge if off + * - >1 → "N triggers" + * + * The detail page renders the full per-trigger editor. + */ +export function AutomationTriggersSummary({ triggers }: AutomationTriggersSummaryProps) { + if (triggers.length === 0) { + return No triggers; + } + + if (triggers.length > 1) { + return {triggers.length} triggers; + } + + const [trigger] = triggers; + + if (trigger.type === "schedule") { + const cron = typeof trigger.params.cron === "string" ? trigger.params.cron : undefined; + const tz = typeof trigger.params.timezone === "string" ? trigger.params.timezone : "UTC"; + const human = cron ? describeCron(cron) : "Schedule"; + + return ( + + + {human} + · {tz} + {!trigger.enabled && ( + + + Off + + )} + + ); + } + + return {trigger.type}; +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx new file mode 100644 index 000000000..83fa52fa8 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx @@ -0,0 +1,50 @@ +"use client"; +import { FileJson, MessageSquarePlus, Workflow } from "lucide-react"; +import Link from "next/link"; +import { Button } from "@/components/ui/button"; + +interface AutomationsEmptyStateProps { + searchSpaceId: number; + canCreate: boolean; +} + +/** + * Zero-state for the automations list. The primary CTA points to a new + * chat — creation happens via the ``create_automation`` HITL tool, not a + * "new automation" form. We surface the chat path explicitly so users + * don't go hunting for an "add" button that doesn't exist. + */ +export function AutomationsEmptyState({ searchSpaceId, canCreate }: AutomationsEmptyStateProps) { + return ( +
+
+ +
+

No automations yet

+

+ Automations let SurfSense run agent tasks on a schedule. Describe what you want in chat and + SurfSense drafts the automation for your approval. +

+ {canCreate ? ( +
+ + +
+ ) : ( +

+ You don't have permission to create automations in this search space. +

+ )} +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx new file mode 100644 index 000000000..544c6b7ac --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx @@ -0,0 +1,59 @@ +"use client"; +import { FileJson, MessageSquarePlus } from "lucide-react"; +import Link from "next/link"; +import { Button } from "@/components/ui/button"; + +interface AutomationsHeaderProps { + searchSpaceId: number; + total: number; + loading: boolean; + canCreate: boolean; + /** + * Render the header's Create CTA. Defaults to true; the empty state owns + * the primary CTA on its own card, so the orchestrator turns this off + * there to avoid a duplicate button. + */ + showCreateCta?: boolean; +} + +/** + * Page header: title + count + "Create via chat" CTA. Creation is intent-driven + * (the create_automation tool runs inside chat with a HITL approval card), so + * the CTA links to a new chat rather than opening a form. + */ +export function AutomationsHeader({ + searchSpaceId, + total, + loading, + canCreate, + showCreateCta = true, +}: AutomationsHeaderProps) { + return ( +
+
+

Automations

+ {!loading && ( + + {total} {total === 1 ? "automation" : "automations"} + + )} +
+ {canCreate && showCreateCta && ( +
+ + +
+ )} +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-loading.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-loading.tsx new file mode 100644 index 000000000..1156be3f6 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-loading.tsx @@ -0,0 +1,36 @@ +"use client"; +import { Skeleton } from "@/components/ui/skeleton"; +import { TableCell, TableRow } from "@/components/ui/table"; + +const ROW_KEYS = ["sk-1", "sk-2", "sk-3"]; + +/** + * Skeleton rows for the automations table. Number of rows is fixed since + * we don't know the count ahead of time and three placeholders is enough + * to communicate "loading" without flashing too much chrome. + */ +export function AutomationsLoadingRows() { + return ( + <> + {ROW_KEYS.map((key) => ( + + +
+ + +
+ + + + + + + + + + + + ))} + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx new file mode 100644 index 000000000..ec3aeeef5 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx @@ -0,0 +1,73 @@ +"use client"; +import { Activity, CalendarDays, Workflow } from "lucide-react"; +import { Table, TableBody, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import type { AutomationSummary } from "@/contracts/types/automation.types"; +import { AutomationRow } from "./automation-row"; +import { AutomationsLoadingRows } from "./automations-loading"; + +interface AutomationsTableProps { + automations: AutomationSummary[]; + searchSpaceId: number; + loading: boolean; + canUpdate: boolean; + canDelete: boolean; +} + +/** + * Table shell + header. Rows render below — loading state renders skeleton + * rows in the same shell so the layout doesn't shift on data arrival. + */ +export function AutomationsTable({ + automations, + searchSpaceId, + loading, + canUpdate, + canDelete, +}: AutomationsTableProps) { + return ( +
+ + + + + + + Name + + + + + + Status + + + + + + Updated + + + + Actions + + + + + {loading ? ( + + ) : ( + automations.map((automation) => ( + + )) + )} + +
+
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/delete-automation-dialog.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/delete-automation-dialog.tsx new file mode 100644 index 000000000..23fc522ca --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/delete-automation-dialog.tsx @@ -0,0 +1,88 @@ +"use client"; +import { useAtomValue } from "jotai"; +import { useState } from "react"; +import { deleteAutomationMutationAtom } from "@/atoms/automations/automations-mutation.atoms"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Spinner } from "@/components/ui/spinner"; + +interface DeleteAutomationDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + automationId: number; + automationName: string; + searchSpaceId: number; + /** + * Fired after a successful delete, before the dialog closes. The detail + * page uses this to navigate back to the list (the row simply vanishes + * on the list page so no callback is needed there). + */ + onDeleted?: () => void; +} + +/** + * Confirm + delete one automation. FK cascade on the backend wipes attached + * triggers and runs, so we mention it explicitly. List re-fetch is handled + * by the mutation atom's onSuccess. + */ +export function DeleteAutomationDialog({ + open, + onOpenChange, + automationId, + automationName, + searchSpaceId, + onDeleted, +}: DeleteAutomationDialogProps) { + const { mutateAsync: deleteAutomation } = useAtomValue(deleteAutomationMutationAtom); + const [submitting, setSubmitting] = useState(false); + + async function handleConfirm() { + setSubmitting(true); + try { + await deleteAutomation({ automationId, searchSpaceId }); + onDeleted?.(); + onOpenChange(false); + } finally { + setSubmitting(false); + } + } + + return ( + + + + Delete this automation? + + {automationName} and all of its + triggers and run history will be removed. This cannot be undone. + + + + Cancel + + {submitting ? ( + + + Deleting… + + ) : ( + "Delete" + )} + + + + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/hooks/use-automation-permissions.ts b/surfsense_web/app/dashboard/[search_space_id]/automations/hooks/use-automation-permissions.ts new file mode 100644 index 000000000..293688710 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/hooks/use-automation-permissions.ts @@ -0,0 +1,37 @@ +"use client"; +import { useAtomValue } from "jotai"; +import { useMemo } from "react"; +import { canPerform, myAccessAtom } from "@/atoms/members/members-query.atoms"; + +/** + * Centralized RBAC gates for the automations slice. Co-located with the + * route so adding/removing surfaces stays a one-file change. Backed by + * the same ``myAccessAtom`` the rest of the app uses; owners short-circuit + * to ``true`` for every action. + * + * Mirrors backend permissions in ``app.db.permissions`` (automations:*). + */ +export interface AutomationPermissions { + loading: boolean; + canCreate: boolean; + canRead: boolean; + canUpdate: boolean; + canDelete: boolean; + canExecute: boolean; +} + +export function useAutomationPermissions(): AutomationPermissions { + const { data: access, isLoading } = useAtomValue(myAccessAtom); + + return useMemo( + () => ({ + loading: isLoading, + canCreate: canPerform(access, "automations:create"), + canRead: canPerform(access, "automations:read"), + canUpdate: canPerform(access, "automations:update"), + canDelete: canPerform(access, "automations:delete"), + canExecute: canPerform(access, "automations:execute"), + }), + [access, isLoading] + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/new/automation-new-content.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/new/automation-new-content.tsx new file mode 100644 index 000000000..f03b3f4c8 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/new/automation-new-content.tsx @@ -0,0 +1,42 @@ +"use client"; +import { ShieldAlert } from "lucide-react"; +import { useAutomationPermissions } from "../hooks/use-automation-permissions"; +import { AutomationJsonForm } from "./components/automation-json-form"; +import { AutomationNewHeader } from "./components/automation-new-header"; + +interface AutomationNewContentProps { + searchSpaceId: number; +} + +/** + * Orchestrator for the raw-JSON create route. Gates on + * ``automations:create`` so users who can't create don't even see the + * form; same panel as the detail page's access-denied state for + * consistency. + */ +export function AutomationNewContent({ searchSpaceId }: AutomationNewContentProps) { + const perms = useAutomationPermissions(); + + if (perms.loading) { + return
; + } + + if (!perms.canCreate) { + return ( +
+ +

Access denied

+

+ You don't have permission to create automations in this search space. +

+
+ ); + } + + return ( + <> + + + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-json-form.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-json-form.tsx new file mode 100644 index 000000000..94b608b8f --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-json-form.tsx @@ -0,0 +1,98 @@ +"use client"; +import { useAtomValue } from "jotai"; +import { AlertCircle, FileJson, Save } from "lucide-react"; +import { useRouter } from "next/navigation"; +import { useState } from "react"; +import { createAutomationMutationAtom } from "@/atoms/automations/automations-mutation.atoms"; +import { JsonView } from "@/components/json-view"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Spinner } from "@/components/ui/spinner"; +import { automationCreateRequest } from "@/contracts/types/automation.types"; +import { DEFAULT_AUTOMATION_TEMPLATE } from "@/lib/automations/default-template"; + +interface AutomationJsonFormProps { + searchSpaceId: number; +} + +/** + * Raw-JSON create form. Lets power users skip the chat drafter when they + * already know the shape they want. Flow: + * edit tree → inject search_space_id → Zod validate → POST → navigate + * + * ``search_space_id`` is injected here rather than required in the edited + * tree — the user shouldn't have to know their numeric id, and it keeps + * the template copy-paste-friendly across search spaces. + */ +export function AutomationJsonForm({ searchSpaceId }: AutomationJsonFormProps) { + const router = useRouter(); + const { mutateAsync: createAutomation, isPending } = useAtomValue(createAutomationMutationAtom); + const [value, setValue] = useState>( + () => DEFAULT_AUTOMATION_TEMPLATE as Record + ); + const [issues, setIssues] = useState([]); + + async function handleSubmit() { + setIssues([]); + + const payload = { ...value, search_space_id: searchSpaceId }; + const result = automationCreateRequest.safeParse(payload); + if (!result.success) { + setIssues( + result.error.issues.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) + ); + return; + } + + try { + const created = await createAutomation(result.data); + router.push(`/dashboard/${searchSpaceId}/automations/${created.id}`); + } catch (err) { + setIssues([(err as Error).message ?? "Submit failed"]); + } + } + + const hasIssues = issues.length > 0; + + return ( + + + + + Definition + triggers + + + +
+ setValue(next as Record)} + collapsed={false} + /> +
+ + {hasIssues && ( +
+
+ + {issues.length === 1 ? "1 issue" : `${issues.length} issues`} +
+
    + {issues.map((issue) => ( +
  • {issue}
    • + ))} +
+
+ )} + +
+ +
+ + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx new file mode 100644 index 000000000..aef2744d5 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx @@ -0,0 +1,42 @@ +"use client"; +import { ArrowLeft, MessageSquarePlus } from "lucide-react"; +import Link from "next/link"; +import { Button } from "@/components/ui/button"; + +interface AutomationNewHeaderProps { + searchSpaceId: number; +} + +export function AutomationNewHeader({ searchSpaceId }: AutomationNewHeaderProps) { + return ( +
+ + +
+
+

+ New automation · raw JSON +

+

+ Paste an ``AutomationCreate`` payload and submit. Validated against the schema before + save. Prefer natural language? Use chat instead. +

+
+ +
+
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/new/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/new/page.tsx new file mode 100644 index 000000000..f6e8e0008 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/new/page.tsx @@ -0,0 +1,15 @@ +import { AutomationNewContent } from "./automation-new-content"; + +export default async function NewAutomationPage({ + params, +}: { + params: Promise<{ search_space_id: string }>; +}) { + const { search_space_id } = await params; + + return ( +
+ +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx new file mode 100644 index 000000000..b77cb20f4 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx @@ -0,0 +1,15 @@ +import { AutomationsContent } from "./automations-content"; + +export default async function AutomationsPage({ + params, +}: { + params: Promise<{ search_space_id: string }>; +}) { + const { search_space_id } = await params; + + return ( +
+ +
+ ); +} diff --git a/surfsense_web/atoms/automations/automations-mutation.atoms.ts b/surfsense_web/atoms/automations/automations-mutation.atoms.ts new file mode 100644 index 000000000..f5e4fd5f4 --- /dev/null +++ b/surfsense_web/atoms/automations/automations-mutation.atoms.ts @@ -0,0 +1,127 @@ +import { atomWithMutation } from "jotai-tanstack-query"; +import { toast } from "sonner"; +import type { + AutomationCreateRequest, + AutomationUpdateRequest, + TriggerCreateRequest, + TriggerUpdateRequest, +} from "@/contracts/types/automation.types"; +import { automationsApiService } from "@/lib/apis/automations-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queryClient } from "@/lib/query-client/client"; + +// Cache invalidation strategy: +// - Automation writes invalidate the search-space list + the touched detail. +// - Trigger writes only invalidate the parent automation detail (triggers +// come back inline in AutomationDetail). +// We deliberately invalidate the whole "automations" prefix on the list side +// because list is keyed by (searchSpaceId, limit, offset) and we don't track +// the active pagination in this layer. + +function invalidateList(searchSpaceId: number) { + queryClient.invalidateQueries({ queryKey: ["automations", "list", searchSpaceId] }); +} + +function invalidateDetail(automationId: number) { + queryClient.invalidateQueries({ + queryKey: cacheKeys.automations.detail(automationId), + }); +} + +export const createAutomationMutationAtom = atomWithMutation(() => ({ + meta: { suppressGlobalErrorToast: true }, + mutationFn: async (request: AutomationCreateRequest) => { + return automationsApiService.createAutomation(request); + }, + onSuccess: (_, variables) => { + invalidateList(variables.search_space_id); + toast.success("Automation created"); + }, + onError: (error: Error) => { + console.error("Error creating automation:", error); + toast.error("Failed to create automation"); + }, +})); + +export const updateAutomationMutationAtom = atomWithMutation(() => ({ + meta: { suppressGlobalErrorToast: true }, + mutationFn: async (vars: { automationId: number; patch: AutomationUpdateRequest }) => { + return automationsApiService.updateAutomation(vars.automationId, vars.patch); + }, + onSuccess: (automation, vars) => { + invalidateDetail(vars.automationId); + invalidateList(automation.search_space_id); + toast.success("Automation updated"); + }, + onError: (error: Error) => { + console.error("Error updating automation:", error); + toast.error("Failed to update automation"); + }, +})); + +export const deleteAutomationMutationAtom = atomWithMutation(() => ({ + meta: { suppressGlobalErrorToast: true }, + mutationFn: async (vars: { automationId: number; searchSpaceId: number }) => { + await automationsApiService.deleteAutomation(vars.automationId); + return vars; + }, + onSuccess: (vars) => { + invalidateList(vars.searchSpaceId); + invalidateDetail(vars.automationId); + toast.success("Automation deleted"); + }, + onError: (error: Error) => { + console.error("Error deleting automation:", error); + toast.error("Failed to delete automation"); + }, +})); + +export const addTriggerMutationAtom = atomWithMutation(() => ({ + meta: { suppressGlobalErrorToast: true }, + mutationFn: async (vars: { automationId: number; payload: TriggerCreateRequest }) => { + return automationsApiService.addTrigger(vars.automationId, vars.payload); + }, + onSuccess: (_, vars) => { + invalidateDetail(vars.automationId); + toast.success("Trigger added"); + }, + onError: (error: Error) => { + console.error("Error adding trigger:", error); + toast.error("Failed to add trigger"); + }, +})); + +export const updateTriggerMutationAtom = atomWithMutation(() => ({ + meta: { suppressGlobalErrorToast: true }, + mutationFn: async (vars: { + automationId: number; + triggerId: number; + patch: TriggerUpdateRequest; + }) => { + return automationsApiService.updateTrigger(vars.automationId, vars.triggerId, vars.patch); + }, + onSuccess: (_, vars) => { + invalidateDetail(vars.automationId); + toast.success("Trigger updated"); + }, + onError: (error: Error) => { + console.error("Error updating trigger:", error); + toast.error("Failed to update trigger"); + }, +})); + +export const removeTriggerMutationAtom = atomWithMutation(() => ({ + meta: { suppressGlobalErrorToast: true }, + mutationFn: async (vars: { automationId: number; triggerId: number }) => { + await automationsApiService.removeTrigger(vars.automationId, vars.triggerId); + return vars; + }, + onSuccess: (vars) => { + invalidateDetail(vars.automationId); + toast.success("Trigger removed"); + }, + onError: (error: Error) => { + console.error("Error removing trigger:", error); + toast.error("Failed to remove trigger"); + }, +})); diff --git a/surfsense_web/atoms/automations/automations-query.atoms.ts b/surfsense_web/atoms/automations/automations-query.atoms.ts new file mode 100644 index 000000000..4117f9bc8 --- /dev/null +++ b/surfsense_web/atoms/automations/automations-query.atoms.ts @@ -0,0 +1,31 @@ +import { atomWithQuery } from "jotai-tanstack-query"; +import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; +import { automationsApiService } from "@/lib/apis/automations-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; + +// First page of the active search space's automations. +// Detail + paginated/parameterized reads live in hooks (see use-automation.ts, +// use-automation-runs.ts) so atoms stay tied to "current scope" and don't +// proliferate atom families for every (id, limit, offset) tuple. +const DEFAULT_LIMIT = 50; +const DEFAULT_OFFSET = 0; + +export const automationsListAtom = atomWithQuery((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + queryKey: cacheKeys.automations.list(Number(searchSpaceId ?? 0), DEFAULT_LIMIT, DEFAULT_OFFSET), + enabled: !!searchSpaceId, + staleTime: 60 * 1000, + queryFn: async () => { + if (!searchSpaceId) { + return { items: [], total: 0 }; + } + return automationsApiService.listAutomations({ + search_space_id: Number(searchSpaceId), + limit: DEFAULT_LIMIT, + offset: DEFAULT_OFFSET, + }); + }, + }; +}); diff --git a/surfsense_web/components/json-metadata-viewer.tsx b/surfsense_web/components/json-metadata-viewer.tsx index cc87a75c5..8ebbbc84e 100644 --- a/surfsense_web/components/json-metadata-viewer.tsx +++ b/surfsense_web/components/json-metadata-viewer.tsx @@ -1,6 +1,6 @@ import { FileJson } from "lucide-react"; import React from "react"; -import { defaultStyles, JsonView } from "react-json-view-lite"; +import { JsonView } from "@/components/json-view"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -10,7 +10,6 @@ import { DialogTrigger, } from "@/components/ui/dialog"; import { Spinner } from "@/components/ui/spinner"; -import "react-json-view-lite/dist/index.css"; interface JsonMetadataViewerProps { title: string; @@ -56,13 +55,13 @@ export function JsonMetadataViewer({ {title} - Metadata -
+
{loading ? (
) : ( - + )}
@@ -87,8 +86,8 @@ export function JsonMetadataViewer({ {title} - Metadata -
- +
+
diff --git a/surfsense_web/components/json-view.tsx b/surfsense_web/components/json-view.tsx new file mode 100644 index 000000000..c293828b3 --- /dev/null +++ b/surfsense_web/components/json-view.tsx @@ -0,0 +1,93 @@ +"use client"; + +import ReactJson, { type InteractionProps } from "@microlink/react-json-view"; +import { useTheme } from "next-themes"; +import { useCallback, useMemo } from "react"; + +/** + * Shared JSON viewer/editor wrapper around @microlink/react-json-view. + * + * One component, dual mode: passing ``editable`` + ``onChange`` enables + * inline value editing, key renaming, add and delete. Omitting them + * yields a read-only viewer. The underlying library is uncontrolled — it + * mutates its own internal copy of ``src`` and surfaces the final tree on + * each interaction via ``updated_src``, which we forward to ``onChange``. + * + * Theme follows ``next-themes``: a dark base-16 palette in dark mode, the + * library's neutral default in light mode. Defaults are tuned for our + * compact UI surfaces (no data-type labels, no key quotes, triangle icons, + * tight indent). + */ +export interface JsonViewProps { + /** The JSON value to display. Primitives are wrapped under ``{ value }`` + * because the underlying library requires an object root. */ + src: unknown; + /** Enables value/key editing + add + delete. Requires ``onChange`` to + * observe the result; without it the toggle is silently a no-op. */ + editable?: boolean; + /** Called with the full updated tree on every accepted interaction. */ + onChange?: (next: unknown) => void; + /** Collapse depth. ``true`` collapses everything past the root; a number + * collapses from that depth onward. */ + collapsed?: boolean | number; + /** Root label. Default ``false`` (no label — saves vertical space). */ + name?: string | false; + className?: string; +} + +const DARK_THEME = "monokai" as const; +const LIGHT_THEME = "rjv-default" as const; + +const SHARED_DEFAULTS = { + iconStyle: "triangle" as const, + indentWidth: 2, + enableClipboard: true, + displayDataTypes: false, + displayObjectSize: true, + quotesOnKeys: false, + collapseStringsAfterLength: 80, +}; + +export function JsonView({ + src, + editable = false, + onChange, + collapsed = 2, + name = false, + className, +}: JsonViewProps) { + const { resolvedTheme } = useTheme(); + const theme = resolvedTheme === "dark" ? DARK_THEME : LIGHT_THEME; + + // The library throws on non-object roots. Wrap primitives and null/undefined. + const safeSrc = useMemo(() => { + if (src && typeof src === "object") return src as object; + return { value: src }; + }, [src]); + + const handleChange = useCallback( + (interaction: InteractionProps) => { + onChange?.(interaction.updated_src); + return true; + }, + [onChange] + ); + + const interactive = editable && onChange ? handleChange : (false as const); + + return ( +
+ +
+ ); +} diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 73b81d439..4284e3da7 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -2,7 +2,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useAtom, useAtomValue, useSetAtom } from "jotai"; -import { AlertTriangle, Inbox, LibraryBig } from "lucide-react"; +import { AlertTriangle, Inbox, LibraryBig, Workflow } from "lucide-react"; import { useParams, usePathname, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useTheme } from "next-themes"; @@ -334,9 +334,10 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid }, [threadsData, searchSpaceId]); // Navigation items - // Inbox is rendered explicitly below "New chat" in the sidebar (it is also - // surfaced in the icon rail's collapsed mode via this list). Announcements - // has been moved to the avatar dropdown and is no longer a nav item. + // Inbox, Automations, and Documents are rendered explicitly below "New chat" + // in the sidebar (also surfaced in the icon rail's collapsed mode via this + // list). Announcements has been moved to the avatar dropdown. + const isAutomationsActive = pathname?.includes("/automations") === true; const navItems: NavItem[] = useMemo( () => ( @@ -348,6 +349,12 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid isActive: isInboxSidebarOpen, badge: totalUnreadCount > 0 ? formatInboxCount(totalUnreadCount) : undefined, }, + { + title: "Automations", + url: `/dashboard/${searchSpaceId}/automations`, + icon: Workflow, + isActive: isAutomationsActive, + }, isMobile ? { title: "Documents", @@ -358,7 +365,14 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid : null, ] as (NavItem | null)[] ).filter((item): item is NavItem => item !== null), - [isMobile, isInboxSidebarOpen, isDocumentsSidebarOpen, totalUnreadCount] + [ + isMobile, + isInboxSidebarOpen, + isDocumentsSidebarOpen, + totalUnreadCount, + searchSpaceId, + isAutomationsActive, + ] ); // Handlers @@ -659,12 +673,14 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid const isUserSettingsPage = pathname?.includes("/user-settings") === true; const isSearchSpaceSettingsPage = pathname?.includes("/search-space-settings") === true; const isTeamPage = pathname?.endsWith("/team") === true; + const isAutomationsPage = pathname?.includes("/automations") === true; const useWorkspacePanel = pathname?.endsWith("/buy-more") === true || pathname?.endsWith("/more-pages") === true || isUserSettingsPage || isSearchSpaceSettingsPage || - isTeamPage; + isTeamPage || + isAutomationsPage; return ( <> @@ -704,12 +720,16 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid isChatPage={isChatPage} useWorkspacePanel={useWorkspacePanel} workspacePanelViewportClassName={ - isUserSettingsPage || isSearchSpaceSettingsPage || isTeamPage + isUserSettingsPage || isSearchSpaceSettingsPage || isTeamPage || isAutomationsPage ? "items-start justify-center px-6 py-8 md:px-10 md:py-10" : undefined } workspacePanelContentClassName={ - isUserSettingsPage || isSearchSpaceSettingsPage || isTeamPage ? "max-w-5xl" : undefined + isAutomationsPage + ? "max-w-none" + : isUserSettingsPage || isSearchSpaceSettingsPage || isTeamPage + ? "max-w-5xl" + : undefined } isLoadingChats={isLoadingThreads} activeSlideoutPanel={activeSlideoutPanel} diff --git a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx index e0cb3072a..805f8bfd3 100644 --- a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx @@ -140,16 +140,26 @@ export function Sidebar({ const t = useTranslations("sidebar"); const [openDropdownChatId, setOpenDropdownChatId] = useState(null); - // Inbox and Documents are rendered explicitly right below New Chat. Pull - // them out of the nav items list so they don't also appear in the bottom - // NavSection. Documents is only present in navItems on mobile. + // Inbox, Automations, and Documents are rendered explicitly right below + // New Chat. Pull them out of the nav items list so they don't also appear + // in the bottom NavSection. Documents is only present in navItems on + // mobile; Automations is identified by URL suffix so the same code path + // works across search spaces. const inboxItem = useMemo(() => navItems.find((item) => item.url === "#inbox"), [navItems]); + const automationsItem = useMemo( + () => navItems.find((item) => item.url.endsWith("/automations")), + [navItems] + ); const documentsItem = useMemo( () => navItems.find((item) => item.url === "#documents"), [navItems] ); const footerNavItems = useMemo( - () => navItems.filter((item) => item.url !== "#inbox" && item.url !== "#documents"), + () => + navItems.filter( + (item) => + item.url !== "#inbox" && item.url !== "#documents" && !item.url.endsWith("/automations") + ), [navItems] ); @@ -227,6 +237,16 @@ export function Sidebar({ } /> )} + {automationsItem && ( + onNavItemClick?.(automationsItem)} + isCollapsed={isCollapsed} + isActive={automationsItem.isActive} + tooltipContent={isCollapsed ? automationsItem.title : undefined} + /> + )} {documentsItem && ( ; + static_inputs: Record; + enabled: boolean; +} + +interface DraftPlanStep { + step_id: string; + action: string; + when?: string | null; +} + +interface AutomationDraft { + name: string; + description?: string | null; + definition: { + goal?: string | null; + plan: DraftPlanStep[]; + }; + triggers: DraftTrigger[]; +} + +interface AutomationDraftPreviewProps { + draft: AutomationDraft; + /** Full unmodified args dict — surfaced as the "raw JSON" escape hatch. */ + raw: Record; +} + +/** + * Structured preview of a drafted automation rendered inside the chat + * approval card. + * + * Three layers, top to bottom: + * 1. Name + description (and goal when present). + * 2. Triggers — humanised cron string + timezone + static_inputs hint. + * 3. Plan steps — ordered list of ``step_id → action``. + * + * A "View raw JSON" toggle reveals the full payload for power users who + * want to inspect every field; it's collapsed by default so the card + * stays scannable for the common case. + */ +export function AutomationDraftPreview({ draft, raw }: AutomationDraftPreviewProps) { + const [showRaw, setShowRaw] = useState(false); + + return ( +
+
+

{draft.name}

+ {draft.description &&

{draft.description}

} +
+ + {draft.definition.goal && ( +
+

{draft.definition.goal}

+
+ )} + +
+ {draft.triggers.length === 0 ? ( +

+ No triggers — automation will need one before it can run. +

+ ) : ( +
    + {draft.triggers.map((trigger) => ( +
  • + +
    • + ))} +
+ )} +
+ +
+
    + {draft.definition.plan.map((step, idx) => ( +
  1. + + {idx + 1} + +
    + {step.step_id} + → + {step.action} + {step.when && when {step.when}} +
    +
    2. + ))} +
+
+ + + {showRaw && ( + 
+					{JSON.stringify(raw, null, 2)}
+
+ )} +
+ ); +} + +/** + * Stable key derived from the trigger's identifying fields. Drafts are + * static snapshots so collisions only happen if the LLM emits two literally + * identical triggers — harmless in practice. + */ +function triggerKey(trigger: DraftTrigger): string { + const cron = typeof trigger.params.cron === "string" ? trigger.params.cron : ""; + const tz = typeof trigger.params.timezone === "string" ? trigger.params.timezone : ""; + return `${trigger.type}|${cron}|${tz}`; +} + +function TriggerLine({ trigger }: { trigger: DraftTrigger }) { + if (trigger.type === "schedule") { + const cron = typeof trigger.params.cron === "string" ? trigger.params.cron : undefined; + const tz = typeof trigger.params.timezone === "string" ? trigger.params.timezone : "UTC"; + const human = cron ? describeCron(cron) : "Schedule"; + const staticKeys = Object.keys(trigger.static_inputs ?? {}); + return ( +
+
+ {human} + · {tz} + {!trigger.enabled && ( + + Disabled + + )} +
+ {cron && {cron}} + {staticKeys.length > 0 && ( +

+ Static inputs: {staticKeys.join(", ")} +

+ )} +
+ ); + } + return {trigger.type}; +} + +function Section({ + icon: Icon, + label, + children, +}: { + icon: typeof Target; + label: string; + children: React.ReactNode; +}) { + return ( +
+
+ + {label} +
+ {children} +
+ ); +} diff --git a/surfsense_web/components/tool-ui/automation/create-automation.tsx b/surfsense_web/components/tool-ui/automation/create-automation.tsx new file mode 100644 index 000000000..b152f9055 --- /dev/null +++ b/surfsense_web/components/tool-ui/automation/create-automation.tsx @@ -0,0 +1,427 @@ +"use client"; + +import type { ToolCallMessagePartProps } from "@assistant-ui/react"; +import { useAtomValue } from "jotai"; +import { AlertCircle, CornerDownLeftIcon, ExternalLink, Pencil, Workflow } from "lucide-react"; +import Link from "next/link"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; +import { JsonView } from "@/components/json-view"; +import { TextShimmerLoader } from "@/components/prompt-kit/loader"; +import { Button } from "@/components/ui/button"; +import { automationCreateRequest } from "@/contracts/types/automation.types"; +import type { HitlDecision, InterruptResult } from "@/features/chat-messages/hitl"; +import { isInterruptResult, useHitlDecision, useHitlPhase } from "@/features/chat-messages/hitl"; +import { AutomationDraftPreview } from "./automation-draft-preview"; + +const editArgsSchema = automationCreateRequest.omit({ search_space_id: true }); + +// ---------------------------------------------------------------------------- +// Result discrimination — mirrors the backend return shapes in +// app/agents/multi_agent_chat/main_agent/tools/automation/create.py. +// ---------------------------------------------------------------------------- + +type AutomationCreateContext = { + search_space_id?: number; +}; + +interface SavedResult { + status: "saved"; + automation_id: number; + name: string; +} + +interface RejectedResult { + status: "rejected"; + message?: string; +} + +interface InvalidResult { + status: "invalid"; + issues: string[]; + raw?: unknown; +} + +interface ErrorResult { + status: "error"; + message: string; +} + +type CreateAutomationResult = + | InterruptResult + | SavedResult + | RejectedResult + | InvalidResult + | ErrorResult; + +function hasStatus(value: unknown, status: string): boolean { + return ( + typeof value === "object" && + value !== null && + "status" in value && + (value as { status: unknown }).status === status + ); +} + +// ---------------------------------------------------------------------------- +// Approval card — pending → processing → complete / rejected. +// +// Edit toggle reuses the same primitives as the Create-via-JSON page: raw +// textarea, Format, Zod validation against ``AutomationCreate`` (minus the +// ``search_space_id`` field, which the backend injects). Approve dispatches +// an ``edit`` decision with the parsed args when edits are pending, otherwise +// a plain ``approve``. Multi-turn chat refinement still works as a fallback. +// ---------------------------------------------------------------------------- + +interface ApprovalCardProps { + args: Record; + interruptData: InterruptResult; + onDecision: (decision: HitlDecision) => void; +} + +function ApprovalCard({ args, interruptData, onDecision }: ApprovalCardProps) { + const { phase, setProcessing, setRejected } = useHitlPhase(interruptData); + + const reviewConfig = interruptData.review_configs[0]; + const allowedDecisions = reviewConfig?.allowed_decisions ?? ["approve", "reject"]; + const canApprove = allowedDecisions.includes("approve"); + const canReject = allowedDecisions.includes("reject"); + const canEdit = allowedDecisions.includes("edit"); + + const [pendingEdits, setPendingEdits] = useState | null>(null); + const [isEditing, setIsEditing] = useState(false); + + const effectiveArgs = pendingEdits ?? args; + const draft = useMemo(() => extractDraft(effectiveArgs), [effectiveArgs]); + + const handleApprove = useCallback(() => { + if (phase !== "pending" || !canApprove || isEditing) return; + setProcessing(); + onDecision({ + type: pendingEdits ? "edit" : "approve", + edited_action: { + name: interruptData.action_requests[0]?.name ?? "create_automation", + args: pendingEdits ?? args, + }, + }); + }, [phase, canApprove, isEditing, setProcessing, onDecision, interruptData, args, pendingEdits]); + + const handleReject = useCallback(() => { + if (phase !== "pending" || !canReject || isEditing) return; + setRejected(); + onDecision({ type: "reject", message: "User rejected the automation draft." }); + }, [phase, canReject, isEditing, setRejected, onDecision]); + + useEffect(() => { + if (isEditing) return; + const handler = (e: KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey && !e.ctrlKey && !e.metaKey) { + handleApprove(); + } + }; + window.addEventListener("keydown", handler); + return () => window.removeEventListener("keydown", handler); + }, [handleApprove, isEditing]); + + return ( +
+
+
+ +
+

+ {phase === "rejected" + ? "Automation cancelled" + : phase === "processing" + ? "Saving automation" + : phase === "complete" + ? "Automation saved" + : "Create automation"} +

+ {phase === "processing" ? ( + + ) : phase === "complete" ? ( +

+ {pendingEdits + ? "Automation saved with your edits" + : "Automation created from this draft"} +

+ ) : phase === "rejected" ? ( +

+ No automation was saved — ask in chat to refine and try again. +

+ ) : ( +

+ {pendingEdits + ? "Showing your edits. Approve to save, or edit again." + : "Review and approve to save. Edit for fine-tuning, or reply in chat for a redraft."} +

+ )} +
+
+ {phase === "pending" && canEdit && !isEditing && ( + + )} +
+ +
+
+ {isEditing ? ( + { + setPendingEdits(parsed); + setIsEditing(false); + }} + onCancel={() => setIsEditing(false)} + /> + ) : ( + + )} +
+ + {phase === "pending" && !isEditing && ( + <> +
+
+ {canApprove && ( + + )} + {canReject && ( + + )} +
+ + )} +
+ ); +} + +interface JsonEditorProps { + initialValue: Record; + onSave: (parsed: Record) => void; + onCancel: () => void; +} + +function JsonEditor({ initialValue, onSave, onCancel }: JsonEditorProps) { + const [value, setValue] = useState>(initialValue); + const [issues, setIssues] = useState([]); + + function handleSave() { + setIssues([]); + const result = editArgsSchema.safeParse(value); + if (!result.success) { + setIssues( + result.error.issues.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) + ); + return; + } + onSave(result.data as unknown as Record); + } + + return ( +
+
+ setValue(next as Record)} + collapsed={false} + /> +
+ {issues.length > 0 && ( +
+
+ + {issues.length} issue{issues.length === 1 ? "" : "s"} +
+
    + {issues.map((issue) => ( +
  • + {issue} +
    • + ))} +
+
+ )} +
+ + +
+
+ ); +} + +// ---------------------------------------------------------------------------- +// Terminal result cards. +// ---------------------------------------------------------------------------- + +function SavedCard({ result }: { result: SavedResult }) { + const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); + const detailHref = searchSpaceId + ? `/dashboard/${searchSpaceId}/automations/${result.automation_id}` + : null; + + return ( +
+
+ +
+

Automation saved

+

{result.name}

+
+
+ {detailHref && ( + <> +
+
+ + + Open automation #{result.automation_id} + +
+ + )} +
+ ); +} + +function InvalidCard({ result }: { result: InvalidResult }) { + return ( +
+
+

Couldn't draft this automation

+

+ The drafter produced output that didn't validate. I'll refine and retry. +

+
+ {result.issues.length > 0 && ( + <> +
+
    + {result.issues.map((issue) => ( +
  • {issue}
    • + ))} +
+ + )} +
+ ); +} + +function ErrorCard({ result }: { result: ErrorResult }) { + return ( +
+
+

Failed to create automation

+
+
+
+

{result.message}

+
+
+ ); +} + +// ---------------------------------------------------------------------------- +// Entry — dispatches between the approval card and terminal result cards. +// +// Rejection is special: we hide the standalone "rejected" card because the +// approval card itself already transitions to a "rejected" phase inline. A +// second message in the timeline would be noisy. +// ---------------------------------------------------------------------------- + +export const CreateAutomationToolUI = ({ + args, + result, +}: ToolCallMessagePartProps<{ intent: string }, CreateAutomationResult>) => { + const { dispatch } = useHitlDecision(); + + if (!result) return null; + + if (isInterruptResult(result)) { + return ( + } + interruptData={result as InterruptResult} + onDecision={(decision) => dispatch([decision])} + /> + ); + } + + if (hasStatus(result, "rejected")) return null; + if (hasStatus(result, "saved")) return ; + if (hasStatus(result, "invalid")) return ; + if (hasStatus(result, "error")) return ; + + return null; +}; + +// ---------------------------------------------------------------------------- +// Helpers. +// ---------------------------------------------------------------------------- + +/** + * Project raw args into the shape ``AutomationDraftPreview`` expects. + * + * The args dict is the full ``AutomationCreate`` payload (minus + * ``search_space_id`` which is injected server-side), so we trust the + * top-level fields but defend against missing nested defaults. + */ +function extractDraft(args: Record) { + const definition = (args.definition ?? {}) as Record; + const planSteps = Array.isArray(definition.plan) + ? (definition.plan as Array>).map((step) => ({ + step_id: String(step.step_id ?? "(unnamed)"), + action: String(step.action ?? ""), + when: typeof step.when === "string" ? step.when : null, + })) + : []; + + const triggers = Array.isArray(args.triggers) + ? (args.triggers as Array>).map((trigger) => ({ + type: String(trigger.type ?? "schedule"), + params: (trigger.params ?? {}) as Record, + static_inputs: (trigger.static_inputs ?? {}) as Record, + enabled: trigger.enabled !== false, + })) + : []; + + return { + name: String(args.name ?? "(unnamed automation)"), + description: typeof args.description === "string" ? args.description : null, + definition: { + goal: typeof definition.goal === "string" ? definition.goal : null, + plan: planSteps, + }, + triggers, + }; +} diff --git a/surfsense_web/components/tool-ui/automation/index.ts b/surfsense_web/components/tool-ui/automation/index.ts new file mode 100644 index 000000000..50cf1a478 --- /dev/null +++ b/surfsense_web/components/tool-ui/automation/index.ts @@ -0,0 +1 @@ +export { CreateAutomationToolUI } from "./create-automation"; diff --git a/surfsense_web/components/tool-ui/index.ts b/surfsense_web/components/tool-ui/index.ts index 4d885a38c..ee5072dad 100644 --- a/surfsense_web/components/tool-ui/index.ts +++ b/surfsense_web/components/tool-ui/index.ts @@ -7,6 +7,7 @@ */ export { Audio } from "./audio"; +export { CreateAutomationToolUI } from "./automation"; export { CreateDropboxFileToolUI, DeleteDropboxFileToolUI } from "./dropbox"; export { type GenerateImageArgs, diff --git a/surfsense_web/contracts/enums/toolIcons.tsx b/surfsense_web/contracts/enums/toolIcons.tsx index bb87be0ba..668cb51cd 100644 --- a/surfsense_web/contracts/enums/toolIcons.tsx +++ b/surfsense_web/contracts/enums/toolIcons.tsx @@ -25,6 +25,7 @@ import { SearchCheck, Send, Trash2, + Workflow, Wrench, } from "lucide-react"; @@ -47,6 +48,8 @@ const TOOL_ICONS: Record = { scrape_webpage: ScanLine, web_search: Globe, search_surfsense_docs: BookOpen, + // Automations + create_automation: Workflow, // Memory update_memory: Brain, // Filesystem (built-in deepagent + middleware) @@ -150,6 +153,8 @@ const TOOL_DISPLAY_NAMES: Record = { scrape_webpage: "Read webpage", web_search: "Search the web", search_surfsense_docs: "Search knowledge base", + // Automations + create_automation: "Create automation", // Memory update_memory: "Update memory", // Calendar diff --git a/surfsense_web/contracts/types/automation.types.ts b/surfsense_web/contracts/types/automation.types.ts new file mode 100644 index 000000000..a93249735 --- /dev/null +++ b/surfsense_web/contracts/types/automation.types.ts @@ -0,0 +1,193 @@ +import { z } from "zod"; + +// ============================================================================= +// Enums — mirror app/automations/persistence/enums/* +// ============================================================================= + +export const automationStatus = z.enum(["active", "paused", "archived"]); +export type AutomationStatus = z.infer; + +export const triggerType = z.enum(["schedule", "manual"]); +export type TriggerType = z.infer; + +export const runStatus = z.enum([ + "pending", + "running", + "succeeded", + "failed", + "cancelled", + "timed_out", +]); +export type RunStatus = z.infer; + +// ============================================================================= +// Definition envelope — mirror app/automations/schemas/definition/* +// ============================================================================= + +export const planStep = z.object({ + step_id: z.string().min(1), + action: z.string().min(1), + when: z.string().nullable().optional(), + params: z.record(z.string(), z.any()).default({}), + output_as: z.string().nullable().optional(), + max_retries: z.number().int().min(0).nullable().optional(), + timeout_seconds: z.number().int().positive().nullable().optional(), +}); +export type PlanStep = z.infer; + +export const definitionTriggerSpec = z.object({ + type: z.string().min(1), + params: z.record(z.string(), z.any()).default({}), +}); +export type DefinitionTriggerSpec = z.infer; + +export const execution = z.object({ + timeout_seconds: z.number().int().positive().default(600), + max_retries: z.number().int().min(0).default(2), + retry_backoff: z.enum(["exponential", "linear", "none"]).default("exponential"), + concurrency: z.enum(["drop_if_running", "queue", "always"]).default("drop_if_running"), + on_failure: z.array(planStep).default([]), +}); +export type Execution = z.infer; + +// Backend ``Metadata`` is ``extra="allow"`` — keep ``tags`` typed, accept arbitrary keys. +export const metadata = z.object({ tags: z.array(z.string()).default([]) }).catchall(z.any()); +export type Metadata = z.infer; + +// Backend ``Inputs`` serializes its ``schema_`` field as ``schema`` (alias). +export const inputs = z.object({ + schema: z.record(z.string(), z.any()), +}); +export type Inputs = z.infer; + +export const automationDefinition = z.object({ + schema_version: z.string().default("1.0"), + name: z.string().min(1).max(200), + goal: z.string().nullable().optional(), + inputs: inputs.nullable().optional(), + triggers: z.array(definitionTriggerSpec).default([]), + plan: z.array(planStep).min(1), + execution: execution.default(execution.parse({})), + metadata: metadata.default(metadata.parse({})), +}); +export type AutomationDefinition = z.infer; + +// ============================================================================= +// Triggers (sub-resource) — mirror app/automations/schemas/api/trigger.py +// ============================================================================= + +export const triggerCreateRequest = z.object({ + type: triggerType, + params: z.record(z.string(), z.any()).default({}), + static_inputs: z.record(z.string(), z.any()).default({}), + enabled: z.boolean().default(true), +}); +export type TriggerCreateRequest = z.infer; + +export const triggerUpdateRequest = z.object({ + enabled: z.boolean().nullable().optional(), + params: z.record(z.string(), z.any()).nullable().optional(), + static_inputs: z.record(z.string(), z.any()).nullable().optional(), +}); +export type TriggerUpdateRequest = z.infer; + +export const trigger = z.object({ + id: z.number(), + type: triggerType, + params: z.record(z.string(), z.any()), + static_inputs: z.record(z.string(), z.any()), + enabled: z.boolean(), + last_fired_at: z.string().nullable().optional(), + next_fire_at: z.string().nullable().optional(), + created_at: z.string(), +}); +export type Trigger = z.infer; + +// ============================================================================= +// Automations — mirror app/automations/schemas/api/automation.py +// ============================================================================= + +export const automationCreateRequest = z.object({ + search_space_id: z.number(), + name: z.string().min(1).max(200), + description: z.string().nullable().optional(), + definition: automationDefinition, + triggers: z.array(triggerCreateRequest).default([]), +}); +export type AutomationCreateRequest = z.infer; + +export const automationUpdateRequest = z.object({ + name: z.string().min(1).max(200).nullable().optional(), + description: z.string().nullable().optional(), + status: automationStatus.nullable().optional(), + definition: automationDefinition.nullable().optional(), +}); +export type AutomationUpdateRequest = z.infer; + +export const automationSummary = z.object({ + id: z.number(), + search_space_id: z.number(), + name: z.string(), + description: z.string().nullable().optional(), + status: automationStatus, + version: z.number(), + created_at: z.string(), + updated_at: z.string(), +}); +export type AutomationSummary = z.infer; + +export const automation = automationSummary.extend({ + definition: automationDefinition, + triggers: z.array(trigger).default([]), +}); +export type Automation = z.infer; + +export const automationListResponse = z.object({ + items: z.array(automationSummary), + total: z.number(), +}); +export type AutomationListResponse = z.infer; + +export const automationListParams = z.object({ + search_space_id: z.number(), + limit: z.number().int().min(1).max(200).default(50), + offset: z.number().int().min(0).default(0), +}); +export type AutomationListParams = z.infer; + +// ============================================================================= +// Runs (sub-resource) — mirror app/automations/schemas/api/run.py +// ============================================================================= + +export const runSummary = z.object({ + id: z.number(), + automation_id: z.number(), + trigger_id: z.number().nullable().optional(), + status: runStatus, + started_at: z.string().nullable().optional(), + finished_at: z.string().nullable().optional(), + created_at: z.string(), +}); +export type RunSummary = z.infer; + +export const run = runSummary.extend({ + definition_snapshot: z.record(z.string(), z.any()), + inputs: z.record(z.string(), z.any()), + step_results: z.array(z.record(z.string(), z.any())), + output: z.record(z.string(), z.any()).nullable().optional(), + artifacts: z.array(z.record(z.string(), z.any())), + error: z.record(z.string(), z.any()).nullable().optional(), +}); +export type Run = z.infer; + +export const runListResponse = z.object({ + items: z.array(runSummary), + total: z.number(), +}); +export type RunListResponse = z.infer; + +export const runListParams = z.object({ + limit: z.number().int().min(1).max(200).default(50), + offset: z.number().int().min(0).default(0), +}); +export type RunListParams = z.infer; diff --git a/surfsense_web/features/chat-messages/timeline/tool-registry/registry.ts b/surfsense_web/features/chat-messages/timeline/tool-registry/registry.ts index 8acc6b4fa..c4cfe7cd3 100644 --- a/surfsense_web/features/chat-messages/timeline/tool-registry/registry.ts +++ b/surfsense_web/features/chat-messages/timeline/tool-registry/registry.ts @@ -17,6 +17,11 @@ const UpdateMemoryToolUI = dynamic( () => import("@/components/tool-ui/user-memory").then((m) => ({ default: m.UpdateMemoryToolUI })), { ssr: false } ); +const CreateAutomationToolUI = dynamic( + () => + import("@/components/tool-ui/automation").then((m) => ({ default: m.CreateAutomationToolUI })), + { ssr: false } +); const SandboxExecuteToolUI = dynamic( () => import("@/components/tool-ui/sandbox-execute").then((m) => ({ @@ -184,6 +189,7 @@ const NullTimelineBody: TimelineToolComponent = () => null; */ const TOOLS_BY_NAME = { task: NullTimelineBody, + create_automation: CreateAutomationToolUI, update_memory: UpdateMemoryToolUI, execute: SandboxExecuteToolUI, execute_code: SandboxExecuteToolUI, diff --git a/surfsense_web/hooks/use-automation-runs.ts b/surfsense_web/hooks/use-automation-runs.ts new file mode 100644 index 000000000..c91c7bd6e --- /dev/null +++ b/surfsense_web/hooks/use-automation-runs.ts @@ -0,0 +1,42 @@ +"use client"; +import { useQuery } from "@tanstack/react-query"; +import type { Run, RunListResponse } from "@/contracts/types/automation.types"; +import { automationsApiService } from "@/lib/apis/automations-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; + +const DEFAULT_LIMIT = 50; +const DEFAULT_OFFSET = 0; + +export interface UseAutomationRunsOptions { + limit?: number; + offset?: number; + enabled?: boolean; +} + +/** Paginated run history for one automation. Newest-first per backend. */ +export function useAutomationRuns( + automationId: number | undefined, + { limit = DEFAULT_LIMIT, offset = DEFAULT_OFFSET, enabled = true }: UseAutomationRunsOptions = {} +) { + return useQuery({ + queryKey: cacheKeys.automations.runs(automationId ?? 0, limit, offset), + queryFn: () => automationsApiService.listRuns(automationId as number, { limit, offset }), + enabled: enabled && !!automationId, + staleTime: 30_000, + }); +} + +/** Single run with the full snapshot, step results, output and artifacts. */ +export function useAutomationRun( + automationId: number | undefined, + runId: number | undefined, + options: { enabled?: boolean } = {} +) { + const { enabled = true } = options; + return useQuery({ + queryKey: cacheKeys.automations.run(automationId ?? 0, runId ?? 0), + queryFn: () => automationsApiService.getRun(automationId as number, runId as number), + enabled: enabled && !!automationId && !!runId, + staleTime: 30_000, + }); +} diff --git a/surfsense_web/hooks/use-automation.ts b/surfsense_web/hooks/use-automation.ts new file mode 100644 index 000000000..d49ec03a1 --- /dev/null +++ b/surfsense_web/hooks/use-automation.ts @@ -0,0 +1,19 @@ +"use client"; +import { useQuery } from "@tanstack/react-query"; +import type { Automation } from "@/contracts/types/automation.types"; +import { automationsApiService } from "@/lib/apis/automations-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; + +/** + * Fetch a single automation with its definition and triggers. + * Lives outside the jotai atom layer because it's keyed by id, not by the + * "current scope" the atom layer assumes. + */ +export function useAutomation(automationId: number | undefined) { + return useQuery({ + queryKey: cacheKeys.automations.detail(automationId ?? 0), + queryFn: () => automationsApiService.getAutomation(automationId as number), + enabled: !!automationId, + staleTime: 60_000, + }); +} diff --git a/surfsense_web/hooks/use-automations.ts b/surfsense_web/hooks/use-automations.ts new file mode 100644 index 000000000..945e91866 --- /dev/null +++ b/surfsense_web/hooks/use-automations.ts @@ -0,0 +1,24 @@ +"use client"; +import { useAtomValue } from "jotai"; +import { automationsListAtom } from "@/atoms/automations/automations-query.atoms"; + +/** + * List automations in the active search space (first page). + * Pagination knobs live in detail/list hooks below; v1 surfaces only the + * first page since automation counts are expected to be small. + */ +export function useAutomations() { + const { data, isLoading, error, refetch } = useAutomationsRaw(); + return { + automations: data?.items ?? [], + total: data?.total ?? 0, + loading: isLoading, + error, + refresh: refetch, + }; +} + +// Exposed for callers that prefer the raw react-query result shape. +export function useAutomationsRaw() { + return useAtomValue(automationsListAtom); +} diff --git a/surfsense_web/lib/apis/automations-api.service.ts b/surfsense_web/lib/apis/automations-api.service.ts new file mode 100644 index 000000000..ebe72bea5 --- /dev/null +++ b/surfsense_web/lib/apis/automations-api.service.ts @@ -0,0 +1,102 @@ +import { + type AutomationCreateRequest, + type AutomationListParams, + type AutomationUpdateRequest, + automation, + automationCreateRequest, + automationListResponse, + automationUpdateRequest, + type RunListParams, + run, + runListResponse, + type TriggerCreateRequest, + type TriggerUpdateRequest, + trigger, + triggerCreateRequest, + triggerUpdateRequest, +} from "@/contracts/types/automation.types"; +import { ValidationError } from "../error"; +import { baseApiService } from "./base-api.service"; + +const BASE = "/api/v1/automations"; + +function rejectIfInvalid( + parsed: { success: true; data: T } | { success: false; error: { issues: { message: string }[] } } +): T { + if (!parsed.success) { + throw new ValidationError( + `Invalid request: ${parsed.error.issues.map((i) => i.message).join(", ")}` + ); + } + return parsed.data; +} + +class AutomationsApiService { + // ---- Automations --------------------------------------------------------- + + listAutomations = async (params: AutomationListParams) => { + const qs = new URLSearchParams({ + search_space_id: String(params.search_space_id), + limit: String(params.limit), + offset: String(params.offset), + }); + return baseApiService.get(`${BASE}?${qs.toString()}`, automationListResponse); + }; + + getAutomation = async (automationId: number) => { + return baseApiService.get(`${BASE}/${automationId}`, automation); + }; + + createAutomation = async (request: AutomationCreateRequest) => { + const data = rejectIfInvalid(automationCreateRequest.safeParse(request)); + return baseApiService.post(BASE, automation, { body: data }); + }; + + updateAutomation = async (automationId: number, request: AutomationUpdateRequest) => { + const data = rejectIfInvalid(automationUpdateRequest.safeParse(request)); + return baseApiService.patch(`${BASE}/${automationId}`, automation, { body: data }); + }; + + // Server returns 204; baseApiService now resolves to null and skips schema validation. + deleteAutomation = async (automationId: number) => { + return baseApiService.delete(`${BASE}/${automationId}`); + }; + + // ---- Triggers (sub-resource) -------------------------------------------- + + addTrigger = async (automationId: number, request: TriggerCreateRequest) => { + const data = rejectIfInvalid(triggerCreateRequest.safeParse(request)); + return baseApiService.post(`${BASE}/${automationId}/triggers`, trigger, { body: data }); + }; + + updateTrigger = async ( + automationId: number, + triggerId: number, + request: TriggerUpdateRequest + ) => { + const data = rejectIfInvalid(triggerUpdateRequest.safeParse(request)); + return baseApiService.patch(`${BASE}/${automationId}/triggers/${triggerId}`, trigger, { + body: data, + }); + }; + + removeTrigger = async (automationId: number, triggerId: number) => { + return baseApiService.delete(`${BASE}/${automationId}/triggers/${triggerId}`); + }; + + // ---- Runs (sub-resource, read-only) ------------------------------------- + + listRuns = async (automationId: number, params: RunListParams) => { + const qs = new URLSearchParams({ + limit: String(params.limit), + offset: String(params.offset), + }); + return baseApiService.get(`${BASE}/${automationId}/runs?${qs.toString()}`, runListResponse); + }; + + getRun = async (automationId: number, runId: number) => { + return baseApiService.get(`${BASE}/${automationId}/runs/${runId}`, run); + }; +} + +export const automationsApiService = new AutomationsApiService(); diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts index 0819cbc7c..a0039b63a 100644 --- a/surfsense_web/lib/apis/base-api.service.ts +++ b/surfsense_web/lib/apis/base-api.service.ts @@ -1,4 +1,5 @@ import type { ZodType } from "zod"; +import { BACKEND_URL } from "@/lib/env-config"; import { getClientPlatform } from "../agent-filesystem"; import { getBearerToken, handleUnauthorized, refreshAccessToken } from "../auth-utils"; import { @@ -9,7 +10,7 @@ import { NetworkError, NotFoundError, } from "../error"; -import { BACKEND_URL } from "@/lib/env-config"; + enum ResponseType { JSON = "json", TEXT = "text", @@ -122,8 +123,9 @@ class BaseApiService { if (contentType === "application/json" && typeof mergedOptions.body === "object") { fetchOptions.body = JSON.stringify(mergedOptions.body); } else { - // Pass body as-is for other content types (e.g., form data, already stringified) - fetchOptions.body = mergedOptions.body; + // Pass body as-is for other content types (form data, already stringified). + // Caller is responsible for passing a real BodyInit when Content-Type is not JSON. + fetchOptions.body = mergedOptions.body as BodyInit; } } @@ -210,32 +212,39 @@ class BaseApiService { let data; const responseType = mergedOptions.responseType; - try { - switch (responseType) { - case ResponseType.JSON: - data = await response.json(); - break; - case ResponseType.TEXT: - data = await response.text(); - break; - case ResponseType.BLOB: - data = await response.blob(); - break; - case ResponseType.ARRAY_BUFFER: - data = await response.arrayBuffer(); - break; - // Add more cases as needed - default: - data = await response.json(); + if (response.status === 204) { + // 204 No Content has no body; .json() would throw SyntaxError. + // Leave data as null and skip schema validation below so endpoints + // that opt out of bodies (REST-style DELETE) don't error on success. + data = null; + } else { + try { + switch (responseType) { + case ResponseType.JSON: + data = await response.json(); + break; + case ResponseType.TEXT: + data = await response.text(); + break; + case ResponseType.BLOB: + data = await response.blob(); + break; + case ResponseType.ARRAY_BUFFER: + data = await response.arrayBuffer(); + break; + // Add more cases as needed + default: + data = await response.json(); + } + } catch (error) { + console.error("Failed to parse response as JSON:", error); + throw new AppError("Failed to parse response", response.status, response.statusText); } - } catch (error) { - console.error("Failed to parse response as JSON:", error); - throw new AppError("Failed to parse response", response.status, response.statusText); } // Validate response if (responseType === ResponseType.JSON) { - if (!responseSchema) { + if (!responseSchema || response.status === 204) { return data; } const parsedData = responseSchema.safeParse(data); diff --git a/surfsense_web/lib/automations/default-template.ts b/surfsense_web/lib/automations/default-template.ts new file mode 100644 index 000000000..8963992cb --- /dev/null +++ b/surfsense_web/lib/automations/default-template.ts @@ -0,0 +1,44 @@ +/** + * Minimal valid ``AutomationCreate`` skeleton used to seed the raw-JSON + * create form. ``search_space_id`` is omitted on purpose — the form + * injects it from the route so users never have to know their id. + * + * The shape matches the Pydantic ``AutomationCreate`` model less the + * search_space_id field; Zod validates the merged payload before submit. + */ +export const DEFAULT_AUTOMATION_TEMPLATE = { + name: "My automation", + description: null, + definition: { + name: "My automation", + goal: null, + plan: [ + { + step_id: "step_1", + action: "agent_task", + params: { + query: "Summarize new docs added to folder 12 since the last run.", + }, + }, + ], + execution: { + timeout_seconds: 600, + max_retries: 2, + retry_backoff: "exponential", + concurrency: "drop_if_running", + on_failure: [], + }, + metadata: { tags: [] }, + }, + triggers: [ + { + type: "schedule", + params: { + cron: "0 9 * * 1-5", + timezone: "UTC", + }, + static_inputs: {}, + enabled: true, + }, + ], +} as const; diff --git a/surfsense_web/lib/automations/describe-cron.ts b/surfsense_web/lib/automations/describe-cron.ts new file mode 100644 index 000000000..19f7ff991 --- /dev/null +++ b/surfsense_web/lib/automations/describe-cron.ts @@ -0,0 +1,67 @@ +/** + * Minimal cron describer for the 5-field patterns the SurfSense drafter LLM + * actually produces (daily, weekdays, weekly, monthly, hourly). Falls back + * to the raw expression when unrecognized so the user still sees something + * honest instead of a guess. + * + * Lives under ``lib/automations/`` because both the dashboard slice and the + * chat ``create_automation`` approval card render schedule descriptions — + * keeping the helper outside either feature avoids a layering violation. + */ + +const DAY_NAMES = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; + +export function describeCron(cron: string): string { + const parts = cron.trim().split(/\s+/); + if (parts.length !== 5) return cron; + + const [minute, hour, dom, month, dow] = parts; + + // Daily at H:MM ("0 9 * * *") + if (month === "*" && dom === "*" && dow === "*" && /^\d+$/.test(minute) && /^\d+$/.test(hour)) { + return `Daily at ${formatTime(hour, minute)}`; + } + + // Weekdays at H:MM ("0 9 * * 1-5") + if (month === "*" && dom === "*" && dow === "1-5" && /^\d+$/.test(minute) && /^\d+$/.test(hour)) { + return `Mon–Fri at ${formatTime(hour, minute)}`; + } + + // Specific weekday(s) ("0 9 * * 1" or "0 9 * * 1,3,5") + if ( + month === "*" && + dom === "*" && + /^\d+$/.test(minute) && + /^\d+$/.test(hour) && + /^[\d,]+$/.test(dow) + ) { + const days = dow + .split(",") + .map((d) => DAY_NAMES[Number(d) % 7]) + .filter(Boolean) + .join(", "); + if (days) return `${days} at ${formatTime(hour, minute)}`; + } + + // Monthly on day N ("0 9 1 * *") + if ( + month === "*" && + dow === "*" && + /^\d+$/.test(dom) && + /^\d+$/.test(hour) && + /^\d+$/.test(minute) + ) { + return `Day ${dom} of each month at ${formatTime(hour, minute)}`; + } + + // Hourly ("0 * * * *") + if (month === "*" && dom === "*" && dow === "*" && hour === "*" && /^\d+$/.test(minute)) { + return minute === "0" ? "Every hour" : `Every hour at :${minute.padStart(2, "0")}`; + } + + return cron; +} + +function formatTime(hour: string, minute: string): string { + return `${hour.padStart(2, "0")}:${minute.padStart(2, "0")}`; +} diff --git a/surfsense_web/lib/format-date.ts b/surfsense_web/lib/format-date.ts index 9decd3402..c2f445537 100644 --- a/surfsense_web/lib/format-date.ts +++ b/surfsense_web/lib/format-date.ts @@ -1,4 +1,12 @@ -import { differenceInDays, differenceInMinutes, format, isToday, isYesterday } from "date-fns"; +import { + differenceInDays, + differenceInMinutes, + format, + isThisYear, + isToday, + isTomorrow, + isYesterday, +} from "date-fns"; /** * Format a date string as a human-readable relative time @@ -23,6 +31,36 @@ export function formatRelativeDate(dateString: string): string { return format(date, "MMM d, yyyy"); } +/** + * Format a future date string as a human-readable countdown. + * - < 1 min: "Any moment" + * - < 60 min: "in 15m" + * - Today: "Today, 2:30 PM" + * - Tomorrow: "Tomorrow, 2:30 PM" + * - < 7 days: "in 3d" + * - This year: "May 30, 2:30 PM" + * - Older: "Jan 15, 2027" + * + * Mirrors {@link formatRelativeDate} but for moments strictly after now. + * Falls back to the past-relative formatter if the timestamp is not in + * the future (defensive — guards against stale "next_fire_at" values). + */ +export function formatRelativeFutureDate(dateString: string): string { + const date = new Date(dateString); + const now = new Date(); + const minutesAhead = differenceInMinutes(date, now); + const daysAhead = differenceInDays(date, now); + + if (minutesAhead <= 0) return formatRelativeDate(dateString); + if (minutesAhead < 1) return "Any moment"; + if (minutesAhead < 60) return `in ${minutesAhead}m`; + if (isToday(date)) return `Today, ${format(date, "h:mm a")}`; + if (isTomorrow(date)) return `Tomorrow, ${format(date, "h:mm a")}`; + if (daysAhead < 7) return `in ${daysAhead}d`; + if (isThisYear(date)) return format(date, "MMM d, h:mm a"); + return format(date, "MMM d, yyyy"); +} + /** * Format a thread's last-updated timestamp for the chats sidebars. * Example: "Mar 23, 2026 at 4:30 PM" diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index ce45ee143..8943d6842 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -126,4 +126,14 @@ export const cacheKeys = { batchUnreadCounts: (searchSpaceId: number | null) => ["notifications", "unread-counts-batch", searchSpaceId] as const, }, + automations: { + // list endpoint is keyed by pagination too so distinct pages don't collide + list: (searchSpaceId: number, limit: number, offset: number) => + ["automations", "list", searchSpaceId, limit, offset] as const, + detail: (automationId: number) => ["automations", "detail", automationId] as const, + runs: (automationId: number, limit: number, offset: number) => + ["automations", "runs", automationId, limit, offset] as const, + run: (automationId: number, runId: number) => + ["automations", "runs", automationId, runId] as const, + }, }; diff --git a/surfsense_web/package.json b/surfsense_web/package.json index 213adbaad..6ac32160b 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -36,6 +36,7 @@ "@babel/standalone": "^7.29.2", "@hookform/resolvers": "^5.2.2", "@marsidev/react-turnstile": "^1.5.0", + "@microlink/react-json-view": "^1.31.20", "@monaco-editor/react": "^4.7.0", "@number-flow/react": "^0.5.10", "@platejs/autoformat": "^52.0.11", @@ -134,7 +135,6 @@ "react-dom": "^19.2.3", "react-dropzone": "^14.3.8", "react-hook-form": "^7.61.1", - "react-json-view-lite": "^2.4.1", "react-syntax-highlighter": "^15.6.1", "react-wrap-balancer": "^1.1.1", "rehype-katex": "^7.0.1", diff --git a/surfsense_web/pnpm-lock.yaml b/surfsense_web/pnpm-lock.yaml index 8602feb8d..7cbff6923 100644 --- a/surfsense_web/pnpm-lock.yaml +++ b/surfsense_web/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: '@marsidev/react-turnstile': specifier: ^1.5.0 version: 1.5.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@microlink/react-json-view': + specifier: ^1.31.20 + version: 1.31.20(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@monaco-editor/react': specifier: ^4.7.0 version: 4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -323,9 +326,6 @@ importers: react-hook-form: specifier: ^7.61.1 version: 7.71.2(react@19.2.4) - react-json-view-lite: - specifier: ^2.4.1 - version: 2.5.0(react@19.2.4) react-syntax-highlighter: specifier: ^15.6.1 version: 15.6.6(react@19.2.4) @@ -1143,24 +1143,28 @@ packages: engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] + libc: [musl] '@biomejs/cli-linux-arm64@2.4.6': resolution: {integrity: sha512-kMLaI7OF5GN1Q8Doymjro1P8rVEoy7BKQALNz6fiR8IC1WKduoNyteBtJlHT7ASIL0Cx2jR6VUOBIbcB1B8pew==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] + libc: [glibc] '@biomejs/cli-linux-x64-musl@2.4.6': resolution: {integrity: sha512-C9s98IPDu7DYarjlZNuzJKTjVHN03RUnmHV5htvqsx6vEUXCDSJ59DNwjKVD5XYoSS4N+BYhq3RTBAL8X6svEg==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] + libc: [musl] '@biomejs/cli-linux-x64@2.4.6': resolution: {integrity: sha512-oHXmUFEoH8Lql1xfc3QkFLiC1hGR7qedv5eKNlC185or+o4/4HiaU7vYODAH3peRCfsuLr1g6v2fK9dFFOYdyw==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] + libc: [glibc] '@biomejs/cli-win32-arm64@2.4.6': resolution: {integrity: sha512-xzThn87Pf3YrOGTEODFGONmqXpTwUNxovQb72iaUOdcw8sBSY3+3WD8Hm9IhMYLnPi0n32s3L3NWU6+eSjfqFg==} @@ -1836,89 +1840,105 @@ packages: resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.4': resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.4': resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-riscv64@1.2.4': resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} cpu: [riscv64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.4': resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.4': resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.2.4': resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.4': resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-linux-arm64@0.34.5': resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-linux-arm@0.34.5': resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-linux-ppc64@0.34.5': resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-linux-riscv64@0.34.5': resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [riscv64] os: [linux] + libc: [glibc] '@img/sharp-linux-s390x@0.34.5': resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-linux-x64@0.34.5': resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-linuxmusl-arm64@0.34.5': resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-linuxmusl-x64@0.34.5': resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-wasm32@0.34.5': resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} @@ -1989,6 +2009,13 @@ packages: peerDependencies: mediabunny: ^1.0.0 + '@microlink/react-json-view@1.31.20': + resolution: {integrity: sha512-gNLkGvjFDeAqVGvK3H7lfoDqetn/9lW2ugiYiJhchc7jQU1ZaKsZnt97ANluXWFfd/wifoA9TrVOTsUXwXCJwA==} + engines: {node: '>=17'} + peerDependencies: + react: '>= 15' + react-dom: '>= 15' + '@monaco-editor/loader@1.7.0': resolution: {integrity: sha512-gIwR1HrJrrx+vfyOhYmCZ0/JcWqG5kbfG7+d3f/C1LXk2EvzAbHSg3MQ5lO2sMlo9izoAZ04shohfKLVT6crVA==} @@ -2028,30 +2055,35 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@napi-rs/canvas-linux-arm64-musl@0.1.97': resolution: {integrity: sha512-kKmSkQVnWeqg7qdsiXvYxKhAFuHz3tkBjW/zyQv5YKUPhotpaVhpBGv5LqCngzyuRV85SXoe+OFj+Tv0a0QXkQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@napi-rs/canvas-linux-riscv64-gnu@0.1.97': resolution: {integrity: sha512-Jc7I3A51jnEOIAXeLsN/M/+Z28LUeakcsXs07FLq9prXc0eYOtVwsDEv913Gr+06IRo34gJJVgT0TXvmz+N2VA==} engines: {node: '>= 10'} cpu: [riscv64] os: [linux] + libc: [glibc] '@napi-rs/canvas-linux-x64-gnu@0.1.97': resolution: {integrity: sha512-iDUBe7AilfuBSRbSa8/IGX38Mf+iCSBqoVKLSQ5XaY2JLOaqz1TVyPFEyIck7wT6mRQhQt5sN6ogfjIDfi74tg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@napi-rs/canvas-linux-x64-musl@0.1.97': resolution: {integrity: sha512-AKLFd/v0Z5fvgqBDqhvqtAdx+fHMJ5t9JcUNKq4FIZ5WH+iegGm8HPdj00NFlCSnm83Fp3Ln8I2f7uq1aIiWaA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@napi-rs/canvas-win32-arm64-msvc@0.1.97': resolution: {integrity: sha512-u883Yr6A6fO7Vpsy9YE4FVCIxzzo5sO+7pIUjjoDLjS3vQaNMkVzx5bdIpEL+ob+gU88WDK4VcxYMZ6nmnoX9A==} @@ -2095,24 +2127,28 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@next/swc-linux-arm64-musl@16.1.6': resolution: {integrity: sha512-S4J2v+8tT3NIO9u2q+S0G5KdvNDjXfAv06OhfOzNDaBn5rw84DGXWndOEB7d5/x852A20sW1M56vhC/tRVbccQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@next/swc-linux-x64-gnu@16.1.6': resolution: {integrity: sha512-2eEBDkFlMMNQnkTyPBhQOAyn2qMxyG2eE7GPH2WIDGEpEILcBPI/jdSv4t6xupSP+ot/jkfrCShLAa7+ZUPcJQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@next/swc-linux-x64-musl@16.1.6': resolution: {integrity: sha512-oicJwRlyOoZXVlxmIMaTq7f8pN9QNbdes0q2FXfRsPhfCi8n8JmOZJm5oo1pwDaFbnnD421rVU409M3evFbIqg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@next/swc-win32-arm64-msvc@16.1.6': resolution: {integrity: sha512-gQmm8izDTPgs+DCWH22kcDmuUp7NyiJgEl18bcr8irXA5N2m2O+JQIr6f3ct42GOs9c0h8QF3L5SzIxcYAAXXw==} @@ -2768,48 +2804,56 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [glibc] '@oxfmt/binding-linux-arm64-musl@0.45.0': resolution: {integrity: sha512-XQKXZIKYJC3GQJ8FnD3iMntpw69Wd9kDDK/Xt79p6xnFYlGGxSNv2vIBvRTDg5CKByWFWWZLCRDOXoP/m6YN4g==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [musl] '@oxfmt/binding-linux-ppc64-gnu@0.45.0': resolution: {integrity: sha512-+g5RiG+xOkdrCWkKodv407nTvMq4vYM18Uox2MhZBm/YoqFxxJpWKsloskFFG5NU13HGPw1wzYjjOVcyd9moCA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@oxfmt/binding-linux-riscv64-gnu@0.45.0': resolution: {integrity: sha512-V7dXKoSyEbWAkkSF4JJNtF+NJZDmJoSarSoP30WCsB3X636Rehd3CvxBj49FIJxEBFWhvcUjGSHVeU8Erck1bQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [riscv64] os: [linux] + libc: [glibc] '@oxfmt/binding-linux-riscv64-musl@0.45.0': resolution: {integrity: sha512-Vdelft1sAEYojVGgcODEFXSWYQYlIvoyIGWebKCuUibd1tvS1TjTx413xG2ZLuHpYj45CkN/ztMLMX6jrgqpgg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [riscv64] os: [linux] + libc: [musl] '@oxfmt/binding-linux-s390x-gnu@0.45.0': resolution: {integrity: sha512-RR7xKgNpqwENnK0aYCGYg0JycY2n93J0reNjHyes+I9Gq52dH95x+CBlnlAQHCPfz6FGnKA9HirgUl14WO6o7w==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] + libc: [glibc] '@oxfmt/binding-linux-x64-gnu@0.45.0': resolution: {integrity: sha512-U/QQ0+BQNSHxjuXR/utvXnQ50Vu5kUuqEomZvQ1/3mhgbBiMc2WU9q5kZ5WwLp3gnFIx9ibkveoRSe2EZubkqg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [glibc] '@oxfmt/binding-linux-x64-musl@0.45.0': resolution: {integrity: sha512-o5TLOUCF0RWQjsIS06yVC+kFgp092/yLe6qBGSUvtnmTVw9gxjpdQSXc3VN5Cnive4K11HNstEZF8ROKHfDFSw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [musl] '@oxfmt/binding-openharmony-arm64@0.45.0': resolution: {integrity: sha512-RnGcV3HgPuOjsGx/k9oyRNKmOp+NBLGzZTdPDYbc19r7NGeYPplnUU/BfU35bX2Y/O4ejvHxcfkvW2WoYL/gsg==} @@ -2864,36 +2908,42 @@ packages: engines: {node: '>= 10.0.0'} cpu: [arm] os: [linux] + libc: [glibc] '@parcel/watcher-linux-arm-musl@2.5.6': resolution: {integrity: sha512-Ve3gUCG57nuUUSyjBq/MAM0CzArtuIOxsBdQ+ftz6ho8n7s1i9E1Nmk/xmP323r2YL0SONs1EuwqBp2u1k5fxg==} engines: {node: '>= 10.0.0'} cpu: [arm] os: [linux] + libc: [musl] '@parcel/watcher-linux-arm64-glibc@2.5.6': resolution: {integrity: sha512-f2g/DT3NhGPdBmMWYoxixqYr3v/UXcmLOYy16Bx0TM20Tchduwr4EaCbmxh1321TABqPGDpS8D/ggOTaljijOA==} engines: {node: '>= 10.0.0'} cpu: [arm64] os: [linux] + libc: [glibc] '@parcel/watcher-linux-arm64-musl@2.5.6': resolution: {integrity: sha512-qb6naMDGlbCwdhLj6hgoVKJl2odL34z2sqkC7Z6kzir8b5W65WYDpLB6R06KabvZdgoHI/zxke4b3zR0wAbDTA==} engines: {node: '>= 10.0.0'} cpu: [arm64] os: [linux] + libc: [musl] '@parcel/watcher-linux-x64-glibc@2.5.6': resolution: {integrity: sha512-kbT5wvNQlx7NaGjzPFu8nVIW1rWqV780O7ZtkjuWaPUgpv2NMFpjYERVi0UYj1msZNyCzGlaCWEtzc+exjMGbQ==} engines: {node: '>= 10.0.0'} cpu: [x64] os: [linux] + libc: [glibc] '@parcel/watcher-linux-x64-musl@2.5.6': resolution: {integrity: sha512-1JRFeC+h7RdXwldHzTsmdtYR/Ku8SylLgTU/reMuqdVD7CtLwf0VR1FqeprZ0eHQkO0vqsbvFLXUmYm/uNKJBg==} engines: {node: '>= 10.0.0'} cpu: [x64] os: [linux] + libc: [musl] '@parcel/watcher-win32-arm64@2.5.6': resolution: {integrity: sha512-3ukyebjc6eGlw9yRt678DxVF7rjXatWiHvTXqphZLvo7aC5NdEgFufVwjFfY51ijYEWpXbqF5jtrK275z52D4Q==} @@ -4222,66 +4272,79 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} @@ -4472,24 +4535,28 @@ packages: engines: {node: '>=10'} cpu: [arm64] os: [linux] + libc: [glibc] '@swc/core-linux-arm64-musl@1.15.13': resolution: {integrity: sha512-SmZ9m+XqCB35NddHCctvHFLqPZDAs5j8IgD36GoutufDJmeq2VNfgk5rQoqNqKmAK3Y7iFdEmI76QoHIWiCLyw==} engines: {node: '>=10'} cpu: [arm64] os: [linux] + libc: [musl] '@swc/core-linux-x64-gnu@1.15.13': resolution: {integrity: sha512-5rij+vB9a29aNkHq72EXI2ihDZPszJb4zlApJY4aCC/q6utgqFA6CkrfTfIb+O8hxtG3zP5KERETz8mfFK6A0A==} engines: {node: '>=10'} cpu: [x64] os: [linux] + libc: [glibc] '@swc/core-linux-x64-musl@1.15.13': resolution: {integrity: sha512-OlSlaOK9JplQ5qn07WiBLibkOw7iml2++ojEXhhR3rbWrNEKCD7sd8+6wSavsInyFdw4PhLA+Hy6YyDBIE23Yw==} engines: {node: '>=10'} cpu: [x64] os: [linux] + libc: [musl] '@swc/core-win32-arm64-msvc@1.15.13': resolution: {integrity: sha512-zwQii5YVdsfG8Ti9gIKgBKZg8qMkRZxl+OlYWUT5D93Jl4NuNBRausP20tfEkQdAPSRrMCSUZBM6FhW7izAZRg==} @@ -4573,24 +4640,28 @@ packages: engines: {node: '>= 20'} cpu: [arm64] os: [linux] + libc: [glibc] '@tailwindcss/oxide-linux-arm64-musl@4.2.1': resolution: {integrity: sha512-WZA0CHRL/SP1TRbA5mp9htsppSEkWuQ4KsSUumYQnyl8ZdT39ntwqmz4IUHGN6p4XdSlYfJwM4rRzZLShHsGAQ==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] + libc: [musl] '@tailwindcss/oxide-linux-x64-gnu@4.2.1': resolution: {integrity: sha512-qMFzxI2YlBOLW5PhblzuSWlWfwLHaneBE0xHzLrBgNtqN6mWfs+qYbhryGSXQjFYB1Dzf5w+LN5qbUTPhW7Y5g==} engines: {node: '>= 20'} cpu: [x64] os: [linux] + libc: [glibc] '@tailwindcss/oxide-linux-x64-musl@4.2.1': resolution: {integrity: sha512-5r1X2FKnCMUPlXTWRYpHdPYUY6a1Ar/t7P24OuiEdEOmms5lyqjDRvVY1yy9Rmioh+AunQ0rWiOTPE8F9A3v5g==} engines: {node: '>= 20'} cpu: [x64] os: [linux] + libc: [musl] '@tailwindcss/oxide-wasm32-wasi@4.2.1': resolution: {integrity: sha512-MGFB5cVPvshR85MTJkEvqDUnuNoysrsRxd6vnk1Lf2tbiqNlXpHYZqkqOQalydienEWOHHFyyuTSYRsLfxFJ2Q==} @@ -4732,6 +4803,9 @@ packages: '@types/katex@0.16.8': resolution: {integrity: sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==} + '@types/lodash@4.17.24': + resolution: {integrity: sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==} + '@types/mdast@4.0.4': resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} @@ -4915,41 +4989,49 @@ packages: resolution: {integrity: sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==} cpu: [arm64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-arm64-musl@1.11.1': resolution: {integrity: sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==} cpu: [arm64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-ppc64-gnu@1.11.1': resolution: {integrity: sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-gnu@1.11.1': resolution: {integrity: sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==} cpu: [riscv64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-musl@1.11.1': resolution: {integrity: sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==} cpu: [riscv64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-s390x-gnu@1.11.1': resolution: {integrity: sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==} cpu: [s390x] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-gnu@1.11.1': resolution: {integrity: sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==} cpu: [x64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-musl@1.11.1': resolution: {integrity: sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==} cpu: [x64] os: [linux] + libc: [musl] '@unrs/resolver-binding-wasm32-wasi@1.11.1': resolution: {integrity: sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==} @@ -5323,6 +5405,13 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + color-string@1.9.1: + resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==} + + color@4.2.3: + resolution: {integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==} + engines: {node: '>=12.5.0'} + comma-separated-tokens@1.0.8: resolution: {integrity: sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==} @@ -6471,6 +6560,9 @@ packages: is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + is-arrayish@0.3.4: + resolution: {integrity: sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==} + is-async-function@2.1.1: resolution: {integrity: sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ==} engines: {node: '>= 0.4'} @@ -6838,24 +6930,28 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [glibc] lightningcss-linux-arm64-musl@1.31.1: resolution: {integrity: sha512-mVZ7Pg2zIbe3XlNbZJdjs86YViQFoJSpc41CbVmKBPiGmC4YrfeOyz65ms2qpAobVd7WQsbW4PdsSJEMymyIMg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [musl] lightningcss-linux-x64-gnu@1.31.1: resolution: {integrity: sha512-xGlFWRMl+0KvUhgySdIaReQdB4FNudfUTARn7q0hh/V67PVGCs3ADFjw+6++kG1RNd0zdGRlEKa+T13/tQjPMA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [glibc] lightningcss-linux-x64-musl@1.31.1: resolution: {integrity: sha512-eowF8PrKHw9LpoZii5tdZwnBcYDxRw2rRCyvAXLi34iyeYfqCQNA9rmUM0ce62NlPhCvof1+9ivRaTY6pSKDaA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [musl] lightningcss-win32-arm64-msvc@1.31.1: resolution: {integrity: sha512-aJReEbSEQzx1uBlQizAOBSjcmr9dCdL3XuC/6HLXAxmtErsj2ICo5yYggg1qOODQMtnjNQv2UHb9NpOuFtYe4w==} @@ -6880,6 +6976,9 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} + lodash-es@4.18.1: + resolution: {integrity: sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==} + lodash.camelcase@4.3.0: resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} @@ -7673,6 +7772,9 @@ packages: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true + react-base16-styling@0.10.0: + resolution: {integrity: sha512-H1k2eFB6M45OaiRru3PBXkuCcn2qNmx+gzLb4a9IPMR7tMH8oBRXU5jGbPDYG1Hz+82d88ED0vjR8BmqU3pQdg==} + react-compiler-runtime@1.0.0: resolution: {integrity: sha512-rRfjYv66HlG8896yPUDONgKzG5BxZD1nV9U6rkm+7VCuvQc903C4MjcoZR4zPw53IKSOX9wMQVpA1IAbRtzQ7w==} peerDependencies: @@ -7729,11 +7831,8 @@ packages: react-is@16.13.1: resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} - react-json-view-lite@2.5.0: - resolution: {integrity: sha512-tk7o7QG9oYyELWHL8xiMQ8x4WzjCzbWNyig3uexmkLb54r8jO0yH3WCWx8UZS0c49eSA4QUmG5caiRJ8fAn58g==} - engines: {node: '>=18'} - peerDependencies: - react: ^18.0.0 || ^19.0.0 + react-lifecycles-compat@3.0.4: + resolution: {integrity: sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA==} react-markdown@10.1.0: resolution: {integrity: sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ==} @@ -8121,6 +8220,9 @@ packages: simple-get@4.0.1: resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + simple-swizzle@0.2.4: + resolution: {integrity: sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==} + slate-dom@0.119.0: resolution: {integrity: sha512-foc8a2NkE+1SldDIYaoqjhVKupt8RSuvHI868rfYOcypD4we5TT7qunjRKJ852EIRh/Ql8sSTepXgXKOUJnt1w==} peerDependencies: @@ -10316,6 +10418,16 @@ snapshots: dependencies: mediabunny: 1.39.2 + '@microlink/react-json-view@1.31.20(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + react: 19.2.4 + react-base16-styling: 0.10.0 + react-dom: 19.2.4(react@19.2.4) + react-lifecycles-compat: 3.0.4 + react-textarea-autosize: 8.5.9(@types/react@19.2.14)(react@19.2.4) + transitivePeerDependencies: + - '@types/react' + '@monaco-editor/loader@1.7.0': dependencies: state-local: 1.0.7 @@ -13283,6 +13395,8 @@ snapshots: '@types/katex@0.16.8': {} + '@types/lodash@4.17.24': {} + '@types/mdast@4.0.4': dependencies: '@types/unist': 3.0.3 @@ -13905,6 +14019,16 @@ snapshots: color-name@1.1.4: {} + color-string@1.9.1: + dependencies: + color-name: 1.1.4 + simple-swizzle: 0.2.4 + + color@4.2.3: + dependencies: + color-convert: 2.0.1 + color-string: 1.9.1 + comma-separated-tokens@1.0.8: {} comma-separated-tokens@2.0.3: {} @@ -15327,6 +15451,8 @@ snapshots: is-arrayish@0.2.1: {} + is-arrayish@0.3.4: {} + is-async-function@2.1.1: dependencies: async-function: 1.0.0 @@ -15662,6 +15788,8 @@ snapshots: dependencies: p-locate: 5.0.0 + lodash-es@4.18.1: {} + lodash.camelcase@4.3.0: {} lodash.debounce@4.0.8: {} @@ -16843,6 +16971,13 @@ snapshots: minimist: 1.2.8 strip-json-comments: 2.0.1 + react-base16-styling@0.10.0: + dependencies: + '@types/lodash': 4.17.24 + color: 4.2.3 + csstype: 3.2.3 + lodash-es: 4.18.1 + react-compiler-runtime@1.0.0(react@19.2.4): dependencies: react: 19.2.4 @@ -16894,9 +17029,7 @@ snapshots: react-is@16.13.1: {} - react-json-view-lite@2.5.0(react@19.2.4): - dependencies: - react: 19.2.4 + react-lifecycles-compat@3.0.4: {} react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4): dependencies: @@ -17458,6 +17591,10 @@ snapshots: once: 1.4.0 simple-concat: 1.0.1 + simple-swizzle@0.2.4: + dependencies: + is-arrayish: 0.3.4 + slate-dom@0.119.0(slate@0.120.0): dependencies: '@juggle/resize-observer': 3.4.0