From 722b5fefcd0f1ee90538c03fbb0312f01605997d Mon Sep 17 00:00:00 2001 From: octo-patch Date: Tue, 2 Jun 2026 01:48:11 +0800 Subject: [PATCH 01/12] feat: upgrade MiniMax default model to M3 - Add MiniMax-M3 to the model selection list (set as the new default) - Add MiniMax-M2.7 and MiniMax-M2.7-highspeed as alternatives - Remove deprecated MiniMax-M2.5 / M2.5-highspeed entries - Update example config and Chinese setup docs to reference M3 (512K context) --- docs/chinese-llm-setup.md | 19 ++++++++++--------- .../app/config/global_llm_config.example.yaml | 10 +++++----- surfsense_web/contracts/enums/llm-models.ts | 14 ++++++++++---- .../contracts/enums/llm-providers.ts | 4 ++-- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/docs/chinese-llm-setup.md b/docs/chinese-llm-setup.md index 6638dbba1..ac03a7902 100644 --- a/docs/chinese-llm-setup.md +++ b/docs/chinese-llm-setup.md @@ -212,9 +212,9 @@ API Base URL: https://open.bigmodel.cn/api/paas/v4 | 字段 | 值 | 说明 | |------|-----|------| -| **Configuration Name** | `MiniMax M2.5` | 配置名称（自定义） | +| **Configuration Name** | `MiniMax M3` | 配置名称（自定义） | | **Provider** | `MINIMAX` | 选择 MiniMax | -| **Model Name** | `MiniMax-M2.5` | 推荐模型
其他选项: `MiniMax-M2.5-highspeed` | +| **Model Name** | `MiniMax-M3` | 推荐模型
其他选项: `MiniMax-M2.7`、`MiniMax-M2.7-highspeed` | | **API Key** | `eyJ...` | 你的 MiniMax API Key | | **API Base URL** | `https://api.minimax.io/v1` | MiniMax API 地址 | | **Parameters** | `{"temperature": 1.0}` | 注意：temperature 必须在 (0.0, 1.0] 范围内，不能为 0 | @@ -222,22 +222,23 @@ API Base URL: https://open.bigmodel.cn/api/paas/v4 ### 示例配置 ``` -Configuration Name: MiniMax M2.5 +Configuration Name: MiniMax M3 Provider: MINIMAX -Model Name: MiniMax-M2.5 +Model Name: MiniMax-M3 API Key: eyJxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx API Base URL: https://api.minimax.io/v1 ``` ### 可用模型 -- **MiniMax-M2.5**: 高性能通用模型，204K 上下文窗口（推荐） -- **MiniMax-M2.5-highspeed**: 高速推理版本，204K 上下文窗口 +- **MiniMax-M3**: 旗舰模型，512K 上下文窗口（推荐） +- **MiniMax-M2.7**: 上一代通用模型，204K 上下文窗口 +- **MiniMax-M2.7-highspeed**: 上一代高速推理版本，204K 上下文窗口 ### 注意事项 - **temperature 参数**: MiniMax 要求 temperature 必须在 (0.0, 1.0] 范围内，不能设置为 0。建议使用 1.0。 -- 两个模型都支持 204K 超长上下文窗口，适合处理长文本任务。 +- M3 支持 512K 超长上下文，M2.7 系列保留 204K，适合按需求选择。 ### 定价 - 请访问 [MiniMax 定价页面](https://platform.minimaxi.com/document/Price) 查看最新价格 @@ -315,8 +316,8 @@ docker compose logs backend | grep -i "error" |---------|---------|------| | **文档摘要** | Qwen-Plus, GLM-4 | 平衡性能和成本 | | **代码分析** | DeepSeek-Coder | 代码专用 | -| **长文本处理** | Kimi 128K, MiniMax-M2.5 (204K) | 超长上下文 | -| **快速响应** | Qwen-Turbo, GLM-4-Flash, MiniMax-M2.5-highspeed | 速度优先 | +| **长文本处理** | Kimi 128K, MiniMax-M3 (512K) | 超长上下文 | +| **快速响应** | Qwen-Turbo, GLM-4-Flash, MiniMax-M2.7-highspeed | 速度优先 | ### 2. 成本优化 diff --git a/surfsense_backend/app/config/global_llm_config.example.yaml b/surfsense_backend/app/config/global_llm_config.example.yaml index 83d556754..1c09a91ac 100644 --- a/surfsense_backend/app/config/global_llm_config.example.yaml +++ b/surfsense_backend/app/config/global_llm_config.example.yaml @@ -236,17 +236,17 @@ global_llm_configs: use_default_system_instructions: true citations_enabled: true - # Example: MiniMax M2.5 - High-performance with 204K context window + # Example: MiniMax M3 - High-performance with 512K context window - id: -8 - name: "Global MiniMax M2.5" - description: "MiniMax M2.5 with 204K context window and competitive pricing" + name: "Global MiniMax M3" + description: "MiniMax M3 with 512K context window and competitive pricing" billing_tier: "free" anonymous_enabled: true seo_enabled: true - seo_slug: "minimax-m2.5" + seo_slug: "minimax-m3" quota_reserve_tokens: 4000 provider: "MINIMAX" - model_name: "MiniMax-M2.5" + model_name: "MiniMax-M3" api_key: "your-minimax-api-key-here" api_base: "https://api.minimax.io/v1" rpm: 60 diff --git a/surfsense_web/contracts/enums/llm-models.ts b/surfsense_web/contracts/enums/llm-models.ts index 74cc056f3..9647c9d31 100644 --- a/surfsense_web/contracts/enums/llm-models.ts +++ b/surfsense_web/contracts/enums/llm-models.ts @@ -1528,14 +1528,20 @@ export const LLM_MODELS: LLMModel[] = [ // MiniMax { - value: "MiniMax-M2.5", - label: "MiniMax M2.5", + value: "MiniMax-M3", + label: "MiniMax M3", + provider: "MINIMAX", + contextWindow: "512K", + }, + { + value: "MiniMax-M2.7", + label: "MiniMax M2.7", provider: "MINIMAX", contextWindow: "204K", }, { - value: "MiniMax-M2.5-highspeed", - label: "MiniMax M2.5 Highspeed", + value: "MiniMax-M2.7-highspeed", + label: "MiniMax M2.7 Highspeed", provider: "MINIMAX", contextWindow: "204K", }, diff --git a/surfsense_web/contracts/enums/llm-providers.ts b/surfsense_web/contracts/enums/llm-providers.ts index ce2b6afe9..c04a44923 100644 --- a/surfsense_web/contracts/enums/llm-providers.ts +++ b/surfsense_web/contracts/enums/llm-providers.ts @@ -184,8 +184,8 @@ export const LLM_PROVIDERS: LLMProvider[] = [ { value: "MINIMAX", label: "MiniMax", - example: "MiniMax-M2.5, MiniMax-M2.5-highspeed", - description: "High-performance models with 204K context", + example: "MiniMax-M3, MiniMax-M2.7", + description: "High-performance models with up to 512K context", apiBase: "https://api.minimax.io/v1", }, { From b2b70cfa3e572bcc3f826dc4d127f756d94f0982 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 20:11:25 +0200 Subject: [PATCH 02/12] fix: make migration 144 idempotent --- .../versions/144_add_automation_tables.py | 96 +++++++++++++------ 1 file changed, 66 insertions(+), 30 deletions(-) diff --git a/surfsense_backend/alembic/versions/144_add_automation_tables.py b/surfsense_backend/alembic/versions/144_add_automation_tables.py index 39f927417..296c33585 100644 --- a/surfsense_backend/alembic/versions/144_add_automation_tables.py +++ b/surfsense_backend/alembic/versions/144_add_automation_tables.py @@ -25,34 +25,60 @@ depends_on: str | Sequence[str] | None = None def upgrade() -> None: - # ENUM types (PostgreSQL requires types created before tables that use them) + # Guard every object so the migration is safe to re-run after a partial + # apply (the types/tables outlive a failed run that never advanced + # alembic_version). Types must precede the tables that reference them. op.execute( """ - CREATE TYPE automation_status AS ENUM ( - 'active', 'paused', 'archived' - ); + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type WHERE typname = 'automation_status' + ) THEN + CREATE TYPE automation_status AS ENUM ( + 'active', 'paused', 'archived' + ); + END IF; + END + $$; """ ) op.execute( """ - CREATE TYPE automation_trigger_type AS ENUM ( - 'schedule', 'manual' - ); + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type WHERE typname = 'automation_trigger_type' + ) THEN + CREATE TYPE automation_trigger_type AS ENUM ( + 'schedule', 'manual' + ); + END IF; + END + $$; """ ) op.execute( """ - CREATE TYPE automation_run_status AS ENUM ( - 'pending', 'running', 'succeeded', 'failed', - 'cancelled', 'timed_out' - ); + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type WHERE typname = 'automation_run_status' + ) THEN + CREATE TYPE automation_run_status AS ENUM ( + 'pending', 'running', 'succeeded', 'failed', + 'cancelled', 'timed_out' + ); + END IF; + END + $$; """ ) # automations — the editable, versioned automation definition op.execute( """ - CREATE TABLE automations ( + CREATE TABLE IF NOT EXISTS automations ( id SERIAL PRIMARY KEY, search_space_id INTEGER NOT NULL REFERENCES searchspaces(id) ON DELETE CASCADE, @@ -69,19 +95,25 @@ def upgrade() -> None: """ ) op.execute( - "CREATE INDEX ix_automations_search_space_id ON automations(search_space_id);" + "CREATE INDEX IF NOT EXISTS ix_automations_search_space_id ON automations(search_space_id);" ) op.execute( - "CREATE INDEX ix_automations_created_by_user_id ON automations(created_by_user_id);" + "CREATE INDEX IF NOT EXISTS ix_automations_created_by_user_id ON automations(created_by_user_id);" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_automations_status ON automations(status);" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_automations_created_at ON automations(created_at);" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_automations_updated_at ON automations(updated_at);" ) - op.execute("CREATE INDEX ix_automations_status ON automations(status);") - op.execute("CREATE INDEX ix_automations_created_at ON automations(created_at);") - op.execute("CREATE INDEX ix_automations_updated_at ON automations(updated_at);") # automation_triggers — one row per (automation, trigger-instance) pair op.execute( """ - CREATE TABLE automation_triggers ( + CREATE TABLE IF NOT EXISTS automation_triggers ( id SERIAL PRIMARY KEY, automation_id INTEGER NOT NULL REFERENCES automations(id) ON DELETE CASCADE, @@ -96,20 +128,22 @@ def upgrade() -> None: """ ) op.execute( - "CREATE INDEX ix_automation_triggers_automation_id ON automation_triggers(automation_id);" - ) - op.execute("CREATE INDEX ix_automation_triggers_type ON automation_triggers(type);") - op.execute( - "CREATE INDEX ix_automation_triggers_enabled ON automation_triggers(enabled);" + "CREATE INDEX IF NOT EXISTS ix_automation_triggers_automation_id ON automation_triggers(automation_id);" ) op.execute( - "CREATE INDEX ix_automation_triggers_created_at ON automation_triggers(created_at);" + "CREATE INDEX IF NOT EXISTS ix_automation_triggers_type ON automation_triggers(type);" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_automation_triggers_enabled ON automation_triggers(enabled);" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_automation_triggers_created_at ON automation_triggers(created_at);" ) # Partial index for the schedule tick: only enabled schedule triggers # with a scheduled next fire are ever scanned for due rows. op.execute( """ - CREATE INDEX ix_automation_triggers_due + CREATE INDEX IF NOT EXISTS ix_automation_triggers_due ON automation_triggers (next_fire_at) WHERE enabled = true AND type = 'schedule' @@ -120,7 +154,7 @@ def upgrade() -> None: # automation_runs — the immutable per-fire execution record op.execute( """ - CREATE TABLE automation_runs ( + CREATE TABLE IF NOT EXISTS automation_runs ( id SERIAL PRIMARY KEY, automation_id INTEGER NOT NULL REFERENCES automations(id) ON DELETE CASCADE, @@ -140,14 +174,16 @@ def upgrade() -> None: """ ) op.execute( - "CREATE INDEX ix_automation_runs_automation_id ON automation_runs(automation_id);" + "CREATE INDEX IF NOT EXISTS ix_automation_runs_automation_id ON automation_runs(automation_id);" ) op.execute( - "CREATE INDEX ix_automation_runs_trigger_id ON automation_runs(trigger_id);" + "CREATE INDEX IF NOT EXISTS ix_automation_runs_trigger_id ON automation_runs(trigger_id);" ) - op.execute("CREATE INDEX ix_automation_runs_status ON automation_runs(status);") op.execute( - "CREATE INDEX ix_automation_runs_created_at ON automation_runs(created_at);" + "CREATE INDEX IF NOT EXISTS ix_automation_runs_status ON automation_runs(status);" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_automation_runs_created_at ON automation_runs(created_at);" ) From c752bdd4fbf2062f9ad31dbd7cfc35cc23a76926 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 20:23:11 +0200 Subject: [PATCH 03/12] docs: clarify automation checkpointer TODO --- .../builtin/agent_task/dependencies.py | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py b/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py index e3736cc95..4ef8c52bf 100644 --- a/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py +++ b/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py @@ -85,23 +85,16 @@ async def build_dependencies( connector_service, firecrawl_api_key = await setup_connector_and_firecrawl( session, search_space_id=search_space_id ) - # Quick fix: use an in-memory checkpointer for automation runs. + # Per-task InMemorySaver: the shared Postgres checkpointer's connection + # pool binds connections to the loop that opened them, but Celery uses a + # fresh loop per task, so the next task hangs 30s on a dead-loop connection + # (`PoolTimeout`). InMemorySaver has no pool and dies with the task — fine + # while runs are one-shot (the checkpoint only spans one graph execution). # - # The shared Postgres checkpointer caches DB connections in a - # module-level pool. Each cached connection is bound to the asyncio - # loop that opened it. Celery throws away the loop after every task, - # so the pool ends up full of connections pointing to a dead loop, - # and the next Celery task (running on a fresh loop) can't use any - # of them — it hangs 30s and fails with - # `PoolTimeout: couldn't get a connection after 30.00 sec`. - # - # InMemorySaver has no cached connections, no loop binding — each - # Celery task creates one and drops it on exit. - # - # TODO(checkpointer): proper fix is to dispose the checkpointer - # pool around each Celery task in `run_async_celery_task`, the same - # way `_dispose_shared_db_engine` already does for the SQLAlchemy - # pool. Then this site can switch back to the shared checkpointer. + # TODO(checkpointer): when runs need durability (crash-resume or HITL + # interrupt/resume across tasks), dispose the checkpointer pool around each + # Celery task in `run_async_celery_task` — as `_dispose_shared_db_engine` + # already does for the SQLAlchemy pool — then use the shared checkpointer. checkpointer = InMemorySaver() return AgentDependencies( llm=llm, From ec2b57bfb328ca7480ccad02d86b9b09d14e2473 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 21:02:16 +0200 Subject: [PATCH 04/12] feat(db): add automation_runs to zero_publication (thin column list) Resyncs zero_publication via the canonical ALTER PUBLICATION ... SET TABLE pattern (mirrors migration 143) to publish a thin live view of automation_runs for dashboard real-time updates. Heavy JSONB fields (definition_snapshot, inputs, output, artifacts, error) stay on REST. --- ...add_automation_runs_to_zero_publication.py | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py diff --git a/surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py b/surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py new file mode 100644 index 000000000..1b25be753 --- /dev/null +++ b/surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py @@ -0,0 +1,177 @@ +"""add automation_runs to zero_publication with thin column list + +Publishes ``automation_runs`` so the dashboard can replace polling with a +live run status + per-step ticker. Only the columns the list and ticker +read are exposed (``id, automation_id, trigger_id, status, step_results, +started_at, finished_at, created_at``); heavy JSONB +(``definition_snapshot``, ``inputs``, ``output``, ``artifacts``, ``error``) +stays on REST and is fetched lazily on detail expand. + +Uses the canonical ``ALTER PUBLICATION ... SET TABLE`` + ``COMMENT`` +bookend pattern (see migration 143) -- the shape Zero ``>=1.0`` requires +to fire its schema-change hook. Existing tables are re-emitted unchanged. + +Revision ID: 148 +Revises: 147 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "148" +down_revision: str | None = "147" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +PUBLICATION_NAME = "zero_publication" + +# Mirrors migration 143. Kept in sync explicitly: any change to these lists +# must be re-emitted in a new resync migration with COMMENT bookends. +DOCUMENT_COLS = [ + "id", + "title", + "document_type", + "search_space_id", + "folder_id", + "created_by_id", + "status", + "created_at", + "updated_at", +] + +USER_COLS = [ + "id", + "pages_limit", + "pages_used", + "premium_credit_micros_limit", + "premium_credit_micros_used", +] + +# Thin set: status + lightweight progress only. Heavy JSONB stays on REST. +AUTOMATION_RUN_COLS = [ + "id", + "automation_id", + "trigger_id", + "status", + "step_results", + "started_at", + "finished_at", + "created_at", +] + + +def _has_zero_version(conn, table: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = :tbl AND column_name = '_0_version'" + ), + {"tbl": table}, + ).fetchone() + is not None + ) + + +def _build_set_table_ddl( + *, documents_has_zero_ver: bool, user_has_zero_ver: bool +) -> str: + doc_cols = DOCUMENT_COLS + (['"_0_version"'] if documents_has_zero_ver else []) + user_cols = USER_COLS + (['"_0_version"'] if user_has_zero_ver else []) + doc_col_list = ", ".join(doc_cols) + user_col_list = ", ".join(user_cols) + run_col_list = ", ".join(AUTOMATION_RUN_COLS) + return ( + f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + f"notifications, " + f"documents ({doc_col_list}), " + f"folders, " + f"search_source_connectors, " + f"new_chat_messages, " + f"chat_comments, " + f"chat_session_state, " + f'"user" ({user_col_list}), ' + f"automation_runs ({run_col_list})" + ) + + +def upgrade() -> None: + conn = op.get_bind() + + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if not exists: + return + + documents_has_zero_ver = _has_zero_version(conn, "documents") + user_has_zero_ver = _has_zero_version(conn, "user") + + # COMMENT-ALTER-COMMENT trio must be one transaction so Zero observes + # them as one schema-change event. Matches the SAVEPOINT pattern used + # in migrations 117 / 139 / 140 / 143. + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-148-resync'") + ) + conn.execute( + sa.text( + _build_set_table_ddl( + documents_has_zero_ver=documents_has_zero_ver, + user_has_zero_ver=user_has_zero_ver, + ) + ) + ) + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-148-resync'") + ) + + +def downgrade() -> None: + """Re-emit migration 143's shape (no automation_runs).""" + conn = op.get_bind() + + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if not exists: + return + + documents_has_zero_ver = _has_zero_version(conn, "documents") + user_has_zero_ver = _has_zero_version(conn, "user") + + doc_cols = DOCUMENT_COLS + (['"_0_version"'] if documents_has_zero_ver else []) + user_cols = USER_COLS + (['"_0_version"'] if user_has_zero_ver else []) + doc_col_list = ", ".join(doc_cols) + user_col_list = ", ".join(user_cols) + ddl = ( + f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + f"notifications, " + f"documents ({doc_col_list}), " + f"folders, " + f"search_source_connectors, " + f"new_chat_messages, " + f"chat_comments, " + f"chat_session_state, " + f'"user" ({user_col_list})' + ) + + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute( + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-148-downgrade'" + ) + ) + conn.execute(sa.text(ddl)) + conn.execute( + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-148-downgrade'" + ) + ) From c64781252d297ae5c19f81f3d0cadeb8c8c92dc5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 21:02:20 +0200 Subject: [PATCH 05/12] feat(zero): define automationRunTable schema --- surfsense_web/zero/schema/automations.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 surfsense_web/zero/schema/automations.ts diff --git a/surfsense_web/zero/schema/automations.ts b/surfsense_web/zero/schema/automations.ts new file mode 100644 index 000000000..4d6ebfac7 --- /dev/null +++ b/surfsense_web/zero/schema/automations.ts @@ -0,0 +1,18 @@ +import { json, number, string, table } from "@rocicorp/zero"; + +// Thin live row: status + per-step progress only. Heavy fields +// (definition_snapshot, inputs, output, artifacts, error) stay on REST +// (`GET /automations/{id}/runs/{run_id}`) and load on detail expand. +// Mirrors the publication shape in migration 148. +export const automationRunTable = table("automation_runs") + .columns({ + id: number(), + automationId: number().from("automation_id"), + triggerId: number().optional().from("trigger_id"), + status: string(), + stepResults: json().from("step_results"), + startedAt: number().optional().from("started_at"), + finishedAt: number().optional().from("finished_at"), + createdAt: number().from("created_at"), + }) + .primaryKey("id"); From a458a56e53dbab9a290552beb8a6435542e9b14f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 21:02:25 +0200 Subject: [PATCH 06/12] feat(zero): register automationRunTable in schema --- surfsense_web/zero/schema/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_web/zero/schema/index.ts b/surfsense_web/zero/schema/index.ts index 3cca0f24a..d6731e371 100644 --- a/surfsense_web/zero/schema/index.ts +++ b/surfsense_web/zero/schema/index.ts @@ -1,4 +1,5 @@ import { createBuilder, createSchema, relationships } from "@rocicorp/zero"; +import { automationRunTable } from "./automations"; import { chatCommentTable, chatSessionStateTable, newChatMessageTable } from "./chat"; import { documentTable, searchSourceConnectorTable } from "./documents"; import { folderTable } from "./folders"; @@ -36,6 +37,7 @@ export const schema = createSchema({ chatCommentTable, chatSessionStateTable, userTable, + automationRunTable, ], relationships: [chatCommentRelationships, newChatMessageRelationships], }); From 9dbe864d6a0f3339e0c6495717094a08cf7ba9cc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 21:02:30 +0200 Subject: [PATCH 07/12] feat(zero): add runs-by-automation query --- surfsense_web/zero/queries/automations.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 surfsense_web/zero/queries/automations.ts diff --git a/surfsense_web/zero/queries/automations.ts b/surfsense_web/zero/queries/automations.ts new file mode 100644 index 000000000..27b274acc --- /dev/null +++ b/surfsense_web/zero/queries/automations.ts @@ -0,0 +1,14 @@ +import { defineQuery } from "@rocicorp/zero"; +import { z } from "zod"; +import { zql } from "../schema/index"; + +// Mirrors chat byThread: client passes the parent id, the REST route still +// authorizes via `automation_id -> search_space`. No search_space_id on the +// table by design. +export const automationRunQueries = { + byAutomation: defineQuery( + z.object({ automationId: z.number() }), + ({ args: { automationId } }) => + zql.automation_runs.where("automationId", automationId).orderBy("createdAt", "desc") + ), +}; From c73f7ef03a4cb7f530845844625cf2f2ff6fd946 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 21:02:34 +0200 Subject: [PATCH 08/12] feat(zero): register automationRuns queries --- surfsense_web/zero/queries/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/surfsense_web/zero/queries/index.ts b/surfsense_web/zero/queries/index.ts index fbf1bd76e..fe711f5d3 100644 --- a/surfsense_web/zero/queries/index.ts +++ b/surfsense_web/zero/queries/index.ts @@ -1,4 +1,5 @@ import { defineQueries } from "@rocicorp/zero"; +import { automationRunQueries } from "./automations"; import { chatSessionQueries, commentQueries, messageQueries } from "./chat"; import { connectorQueries, documentQueries } from "./documents"; import { folderQueries } from "./folders"; @@ -14,4 +15,5 @@ export const queries = defineQueries({ comments: commentQueries, chatSession: chatSessionQueries, user: userQueries, + automationRuns: automationRunQueries, }); From 69eb64db08d16384522ce2f63bde46a347048fec Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 21:02:40 +0200 Subject: [PATCH 09/12] refactor(automations): source runs list from Zero useAutomationRuns now reads from the zero_publication thin column set and adapts rows to LiveRunSummary (RunSummary + step_results). The detail hook stays on REST for the heavy fields. --- surfsense_web/hooks/use-automation-runs.ts | 99 ++++++++++++++++++---- 1 file changed, 83 insertions(+), 16 deletions(-) diff --git a/surfsense_web/hooks/use-automation-runs.ts b/surfsense_web/hooks/use-automation-runs.ts index c91c7bd6e..69e51ddc6 100644 --- a/surfsense_web/hooks/use-automation-runs.ts +++ b/surfsense_web/hooks/use-automation-runs.ts @@ -1,42 +1,109 @@ "use client"; -import { useQuery } from "@tanstack/react-query"; -import type { Run, RunListResponse } from "@/contracts/types/automation.types"; +import { useQuery as useZeroQuery } from "@rocicorp/zero/react"; +import { useQuery as useReactQuery } from "@tanstack/react-query"; +import { useMemo } from "react"; +import type { Run, RunStepResult, RunSummary } from "@/contracts/types/automation.types"; import { automationsApiService } from "@/lib/apis/automations-api.service"; import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queries } from "@/zero/queries"; const DEFAULT_LIMIT = 50; -const DEFAULT_OFFSET = 0; + +/** + * Thin live row sourced from Zero. Strict superset of {@link RunSummary} — + * existing consumers that only look at the summary fields keep working, + * while the run detail panel can read ``step_results`` directly for the + * live step ticker without a second REST round-trip. + */ +export interface LiveRunSummary extends RunSummary { + step_results: RunStepResult[]; +} export interface UseAutomationRunsOptions { limit?: number; - offset?: number; - enabled?: boolean; } -/** Paginated run history for one automation. Newest-first per backend. */ +interface UseAutomationRunsResult { + data: { items: LiveRunSummary[]; total: number } | undefined; + isLoading: boolean; + error: Error | null; +} + +/** + * Live run history for one automation, newest-first. Sourced from Zero's + * thin ``automation_runs`` publication so status and per-step progress + * tick in real time without polling. Heavy fields (output, artifacts, + * inputs, error, definition_snapshot) are still fetched lazily via + * {@link useAutomationRun}. + */ export function useAutomationRuns( automationId: number | undefined, - { limit = DEFAULT_LIMIT, offset = DEFAULT_OFFSET, enabled = true }: UseAutomationRunsOptions = {} -) { - return useQuery({ - queryKey: cacheKeys.automations.runs(automationId ?? 0, limit, offset), - queryFn: () => automationsApiService.listRuns(automationId as number, { limit, offset }), - enabled: enabled && !!automationId, - staleTime: 30_000, - }); + { limit = DEFAULT_LIMIT }: UseAutomationRunsOptions = {} +): UseAutomationRunsResult { + const [rows, result] = useZeroQuery( + queries.automationRuns.byAutomation({ automationId: automationId ?? -1 }) + ); + + const items = useMemo(() => { + if (!automationId) return []; + return rows.slice(0, limit).map(toLiveRunSummary); + }, [automationId, rows, limit]); + + const total = automationId ? rows.length : 0; + + // Pre-hydration window: nothing visible AND Zero hasn't confirmed + // completeness yet. After the first sync (even an empty set) we stop + // showing the skeleton so the empty-state copy can take over. + const isLoading = !!automationId && result.type !== "complete" && rows.length === 0; + + return { + data: automationId ? { items, total } : undefined, + isLoading, + error: null, + }; } -/** Single run with the full snapshot, step results, output and artifacts. */ +/** + * Full run record (definition snapshot, inputs, output, artifacts, error). + * Stays on REST: these fields are large and largely static after the run + * finishes, so they're not worth replicating to every connected client. + */ export function useAutomationRun( automationId: number | undefined, runId: number | undefined, options: { enabled?: boolean } = {} ) { const { enabled = true } = options; - return useQuery({ + return useReactQuery({ queryKey: cacheKeys.automations.run(automationId ?? 0, runId ?? 0), queryFn: () => automationsApiService.getRun(automationId as number, runId as number), enabled: enabled && !!automationId && !!runId, staleTime: 30_000, }); } + +interface ZeroAutomationRunRow { + id: number; + automationId: number; + triggerId?: number | null; + status: string; + stepResults: unknown; + startedAt?: number | null; + finishedAt?: number | null; + createdAt: number; +} + +/** Adapt a Zero camelCase row (epoch ms timestamps) to the snake_case + * ISO-string ``RunSummary`` shape the existing UI already consumes. */ +function toLiveRunSummary(row: ZeroAutomationRunRow): LiveRunSummary { + return { + id: row.id, + automation_id: row.automationId, + trigger_id: row.triggerId ?? null, + status: row.status as RunSummary["status"], + started_at: row.startedAt ? new Date(row.startedAt).toISOString() : null, + finished_at: row.finishedAt ? new Date(row.finishedAt).toISOString() : null, + created_at: new Date(row.createdAt).toISOString(), + step_results: Array.isArray(row.stepResults) ? (row.stepResults as RunStepResult[]) : [], + }; +} From d8db3159d6aae6a2357f53d99a2ebf5fa906bdf8 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 21:02:47 +0200 Subject: [PATCH 10/12] feat(automations): forward live status and steps to run details panel --- .../[automation_id]/components/run-row.tsx | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx index 3f6a39c35..b48230e3f 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx @@ -1,21 +1,21 @@ "use client"; import { ChevronDown, ChevronRight, Hand } from "lucide-react"; import { useState } from "react"; -import type { RunSummary } from "@/contracts/types/automation.types"; +import type { LiveRunSummary } from "@/hooks/use-automation-runs"; import { formatDuration } from "@/lib/automations/run-duration"; import { formatRelativeDate } from "@/lib/format-date"; import { RunDetailsPanel } from "./run-details-panel"; import { RunStatusBadge } from "./run-status-badge"; interface RunRowProps { - run: RunSummary; + run: LiveRunSummary; automationId: number; } /** - * One run row. Click to expand → fetches the full run and shows the - * details panel inline. State is local to each row so multiple panels - * can be open at once (or none). + * One run row. Click to expand → renders the details panel inline. + * Status and step_results come live from the parent's Zero query; the + * panel itself only fetches the heavy REST fields on first expand. */ export function RunRow({ run, automationId }: RunRowProps) { const [open, setOpen] = useState(false); @@ -47,7 +47,14 @@ export function RunRow({ run, automationId }: RunRowProps) { - {open && } + {open && ( + + )} ); } From ca66bff02b42f251c0dfc1c68c62ef80cbc8d83d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 21:02:54 +0200 Subject: [PATCH 11/12] feat(automations): render live step ticker, defer REST until terminal Step results now render from the synced Zero row so the panel ticks forward as the run progresses. The REST getRun call is gated on the run reaching a terminal status, since output/artifacts/error are only written at terminal mark. --- .../components/run-details-panel.tsx | 108 ++++++++++-------- 1 file changed, 60 insertions(+), 48 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx index 164f156e5..1a54ac0e5 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx @@ -15,7 +15,7 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component import { ScrollArea } from "@/components/ui/scroll-area"; import { Separator } from "@/components/ui/separator"; import { Skeleton } from "@/components/ui/skeleton"; -import type { RunStepResult } from "@/contracts/types/automation.types"; +import type { RunStatus, RunStepResult } from "@/contracts/types/automation.types"; import { useAutomationRun } from "@/hooks/use-automation-runs"; import { cn } from "@/lib/utils"; import { RunStepResultCard } from "./run-step-result-card"; @@ -23,44 +23,46 @@ import { RunStepResultCard } from "./run-step-result-card"; interface RunDetailsPanelProps { automationId: number; runId: number; + /** Live step entries from Zero; rendered while the run is in-flight and + * also kept as the authoritative source once it finishes. */ + liveSteps: RunStepResult[]; + /** Live run status from Zero. Used to hide diagnostic sections that + * only make sense after the run reaches a terminal state. */ + liveStatus: RunStatus; } /** - * Expanded view of a single run. Fetches lazily — the parent only renders - * this once the row is opened, so the list view stays cheap. + * Expanded view of a single run. Steps render immediately from the live + * Zero row so the panel updates as the run progresses; the heavy REST + * payload (output, artifacts, resolved inputs, run-level error) is + * fetched lazily and merged in when it arrives. * - * We surface the run outcome readably: a run-level error first (when - * present), then per-step cards that render the agent's markdown - * ``final_message`` directly, and finally the structural artifacts/inputs. - * The full ``definition_snapshot`` is omitted because it usually mirrors the - * live definition — surfacing it would dominate the panel without informing + * Surfacing order is outcome-first: a run-level error (when present), + * then per-step cards that render the agent's markdown ``final_message`` + * directly, and finally the structural artifacts/inputs. The full + * ``definition_snapshot`` is omitted because it usually mirrors the live + * definition — surfacing it would dominate the panel without informing * what the user is trying to learn ("did this work? what did it do?"). */ -export function RunDetailsPanel({ automationId, runId }: RunDetailsPanelProps) { - const { data: run, isLoading, error } = useAutomationRun(automationId, runId); +export function RunDetailsPanel({ + automationId, + runId, + liveSteps, + liveStatus, +}: RunDetailsPanelProps) { + const isTerminal = liveStatus !== "pending" && liveStatus !== "running"; + // Defer the REST round-trip until the run can actually carry heavy + // fields — output/artifacts/error are only written at terminal mark. + const { data: run, isLoading, error } = useAutomationRun(automationId, runId, { + enabled: isTerminal, + }); - if (isLoading) { - return ( -
- - -
- ); - } - - if (error || !run) { - return ( -
- Couldn't load run details{error?.message ? `: ${error.message}` : "."} -
- ); - } - - const runError = run.error && Object.keys(run.error).length > 0 ? run.error : null; - const hasOutput = run.output && Object.keys(run.output).length > 0; - const hasInputs = Object.keys(run.inputs ?? {}).length > 0; - const steps = run.step_results as RunStepResult[]; - const hasDiagnostics = run.artifacts.length > 0 || hasInputs; + const runError = run?.error && Object.keys(run.error).length > 0 ? run.error : null; + const hasOutput = !!run?.output && Object.keys(run.output).length > 0; + const hasInputs = !!run && Object.keys(run.inputs ?? {}).length > 0; + const hasDiagnostics = !!run && (run.artifacts.length > 0 || hasInputs); + const heavyLoading = isTerminal && isLoading && !run; + const heavyError = isTerminal && !!error; return (
@@ -72,30 +74,40 @@ export function RunDetailsPanel({ automationId, runId }: RunDetailsPanelProps) { ) : null} -
- {steps.length === 0 ? ( -

No steps recorded.

+
+ {liveSteps.length === 0 ? ( +

+ {isTerminal ? "No steps recorded." : "Waiting for first step…"} +

) : (
- {steps.map((step, index) => ( + {liveSteps.map((step, index) => ( ))}
)}
- {hasDiagnostics ? : null} - - {run.artifacts.length > 0 ? ( -
- -
- ) : null} - - {hasInputs ? ( -
- -
+ {heavyLoading ? ( + + ) : heavyError ? ( +

+ Couldn't load run details{error?.message ? `: ${error.message}` : "."} +

+ ) : hasDiagnostics ? ( + <> + + {run && run.artifacts.length > 0 ? ( +
+ +
+ ) : null} + {hasInputs ? ( +
+ +
+ ) : null} + ) : null}
); From 00ee6d04ee94e5ec4f81ec31c288378e63b1b341 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Mon, 1 Jun 2026 21:02:58 +0200 Subject: [PATCH 12/12] docs(zero-sync): list automation_runs in synced tables --- surfsense_web/content/docs/how-to/zero-sync.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/surfsense_web/content/docs/how-to/zero-sync.mdx b/surfsense_web/content/docs/how-to/zero-sync.mdx index 1a7762f23..7007e6637 100644 --- a/surfsense_web/content/docs/how-to/zero-sync.mdx +++ b/surfsense_web/content/docs/how-to/zero-sync.mdx @@ -104,6 +104,7 @@ Zero syncs the following tables for real-time features: | `new_chat_messages` | Live chat message sync for shared chats | | `chat_comments` | Real-time comment threads on AI responses | | `chat_session_state` | Collaboration indicators (who is typing) | +| `automation_runs` | Live run status and per-step progress (thin column set; heavy fields stay on REST) | ## Troubleshooting