Merge remote-tracking branch 'upstream/dev' into feat/whatsapp-gateway-integration

surfsense_backend/alembic/versions/144_add_automation_tables.py

@@ -0,0 +1,177 @@
+"""Add automation tables (automations, automation_triggers, automation_runs)
+
+Revision ID: 144
+Revises: 143
+Create Date: 2026-05-26
+
+Adds the three tables that back the v1 automation engine, plus the
+three PostgreSQL ENUM types they reference. Matches the SQLAlchemy
+models under ``app.automations.persistence.models`` and the v1 data
+model in ``automation-design-plan.md`` §9.
+
+v1 ships these three tables only. ``domain_events`` is deferred to
+Phase 3 with the event trigger; ``mcp_connections`` / ``mcp_tools``
+are deferred to Phase 4 with the MCP integration.
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "144"
+down_revision: str | None = "143"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ENUM types (PostgreSQL requires types created before tables that use them)
+    op.execute(
+        """
+        CREATE TYPE automation_status AS ENUM (
+            'active', 'paused', 'archived'
+        );
+        """
+    )
+    op.execute(
+        """
+        CREATE TYPE automation_trigger_type AS ENUM (
+            'schedule', 'manual'
+        );
+        """
+    )
+    op.execute(
+        """
+        CREATE TYPE automation_run_status AS ENUM (
+            'pending', 'running', 'succeeded', 'failed',
+            'cancelled', 'timed_out'
+        );
+        """
+    )
+
+    # automations — the editable, versioned automation definition
+    op.execute(
+        """
+        CREATE TABLE automations (
+            id SERIAL PRIMARY KEY,
+            search_space_id INTEGER NOT NULL
+                REFERENCES searchspaces(id) ON DELETE CASCADE,
+            created_by_user_id UUID
+                REFERENCES "user"(id) ON DELETE SET NULL,
+            name VARCHAR(200) NOT NULL,
+            description TEXT,
+            status automation_status NOT NULL DEFAULT 'active',
+            definition JSONB NOT NULL,
+            version INTEGER NOT NULL DEFAULT 1,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+            updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+        );
+        """
+    )
+    op.execute(
+        "CREATE INDEX ix_automations_search_space_id ON automations(search_space_id);"
+    )
+    op.execute(
+        "CREATE INDEX ix_automations_created_by_user_id ON automations(created_by_user_id);"
+    )
+    op.execute("CREATE INDEX ix_automations_status ON automations(status);")
+    op.execute("CREATE INDEX ix_automations_created_at ON automations(created_at);")
+    op.execute("CREATE INDEX ix_automations_updated_at ON automations(updated_at);")
+
+    # automation_triggers — one row per (automation, trigger-instance) pair
+    op.execute(
+        """
+        CREATE TABLE automation_triggers (
+            id SERIAL PRIMARY KEY,
+            automation_id INTEGER NOT NULL
+                REFERENCES automations(id) ON DELETE CASCADE,
+            type automation_trigger_type NOT NULL,
+            params JSONB NOT NULL,
+            static_inputs JSONB NOT NULL DEFAULT '{}'::jsonb,
+            enabled BOOLEAN NOT NULL DEFAULT true,
+            last_fired_at TIMESTAMP WITH TIME ZONE,
+            next_fire_at TIMESTAMP WITH TIME ZONE,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+        );
+        """
+    )
+    op.execute(
+        "CREATE INDEX ix_automation_triggers_automation_id ON automation_triggers(automation_id);"
+    )
+    op.execute("CREATE INDEX ix_automation_triggers_type ON automation_triggers(type);")
+    op.execute(
+        "CREATE INDEX ix_automation_triggers_enabled ON automation_triggers(enabled);"
+    )
+    op.execute(
+        "CREATE INDEX ix_automation_triggers_created_at ON automation_triggers(created_at);"
+    )
+    # Partial index for the schedule tick: only enabled schedule triggers
+    # with a scheduled next fire are ever scanned for due rows.
+    op.execute(
+        """
+        CREATE INDEX ix_automation_triggers_due
+            ON automation_triggers (next_fire_at)
+            WHERE enabled = true
+              AND type = 'schedule'
+              AND next_fire_at IS NOT NULL;
+        """
+    )
+
+    # automation_runs — the immutable per-fire execution record
+    op.execute(
+        """
+        CREATE TABLE automation_runs (
+            id SERIAL PRIMARY KEY,
+            automation_id INTEGER NOT NULL
+                REFERENCES automations(id) ON DELETE CASCADE,
+            trigger_id INTEGER
+                REFERENCES automation_triggers(id) ON DELETE SET NULL,
+            status automation_run_status NOT NULL DEFAULT 'pending',
+            definition_snapshot JSONB NOT NULL,
+            inputs JSONB NOT NULL DEFAULT '{}'::jsonb,
+            step_results JSONB NOT NULL DEFAULT '[]'::jsonb,
+            output JSONB,
+            artifacts JSONB NOT NULL DEFAULT '[]'::jsonb,
+            error JSONB,
+            started_at TIMESTAMP WITH TIME ZONE,
+            finished_at TIMESTAMP WITH TIME ZONE,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+        );
+        """
+    )
+    op.execute(
+        "CREATE INDEX ix_automation_runs_automation_id ON automation_runs(automation_id);"
+    )
+    op.execute(
+        "CREATE INDEX ix_automation_runs_trigger_id ON automation_runs(trigger_id);"
+    )
+    op.execute("CREATE INDEX ix_automation_runs_status ON automation_runs(status);")
+    op.execute(
+        "CREATE INDEX ix_automation_runs_created_at ON automation_runs(created_at);"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS ix_automation_runs_created_at;")
+    op.execute("DROP INDEX IF EXISTS ix_automation_runs_status;")
+    op.execute("DROP INDEX IF EXISTS ix_automation_runs_trigger_id;")
+    op.execute("DROP INDEX IF EXISTS ix_automation_runs_automation_id;")
+    op.execute("DROP TABLE IF EXISTS automation_runs;")
+
+    op.execute("DROP INDEX IF EXISTS ix_automation_triggers_due;")
+    op.execute("DROP INDEX IF EXISTS ix_automation_triggers_created_at;")
+    op.execute("DROP INDEX IF EXISTS ix_automation_triggers_enabled;")
+    op.execute("DROP INDEX IF EXISTS ix_automation_triggers_type;")
+    op.execute("DROP INDEX IF EXISTS ix_automation_triggers_automation_id;")
+    op.execute("DROP TABLE IF EXISTS automation_triggers;")
+
+    op.execute("DROP INDEX IF EXISTS ix_automations_updated_at;")
+    op.execute("DROP INDEX IF EXISTS ix_automations_created_at;")
+    op.execute("DROP INDEX IF EXISTS ix_automations_status;")
+    op.execute("DROP INDEX IF EXISTS ix_automations_created_by_user_id;")
+    op.execute("DROP INDEX IF EXISTS ix_automations_search_space_id;")
+    op.execute("DROP TABLE IF EXISTS automations;")
+
+    op.execute("DROP TYPE IF EXISTS automation_run_status;")
+    op.execute("DROP TYPE IF EXISTS automation_trigger_type;")
+    op.execute("DROP TYPE IF EXISTS automation_status;")

surfsense_backend/alembic/versions/145_add_automations_permissions_to_roles.py

@@ -0,0 +1,87 @@
+"""Add automations permissions to existing Editor/Viewer roles
+
+Revision ID: 145
+Revises: 144
+Create Date: 2026-05-27
+
+Owners already have ``*`` and need no backfill. Custom (non-system) roles
+are left untouched on purpose: workspace admins manage those explicitly.
+"""
+
+from collections.abc import Sequence
+
+from sqlalchemy import text
+
+from alembic import op
+
+revision: str = "145"
+down_revision: str | None = "144"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+_EDITOR_PERMISSIONS = (
+    "automations:create",
+    "automations:read",
+    "automations:update",
+    "automations:execute",
+)
+_VIEWER_PERMISSIONS = ("automations:read",)
+
+
+def upgrade():
+    connection = op.get_bind()
+
+    for permission in _EDITOR_PERMISSIONS:
+        connection.execute(
+            text(
+                """
+                UPDATE search_space_roles
+                SET permissions = array_append(permissions, :permission)
+                WHERE name = 'Editor'
+                AND NOT (:permission = ANY(permissions))
+                """
+            ),
+            {"permission": permission},
+        )
+
+    for permission in _VIEWER_PERMISSIONS:
+        connection.execute(
+            text(
+                """
+                UPDATE search_space_roles
+                SET permissions = array_append(permissions, :permission)
+                WHERE name = 'Viewer'
+                AND NOT (:permission = ANY(permissions))
+                """
+            ),
+            {"permission": permission},
+        )
+
+
+def downgrade():
+    connection = op.get_bind()
+
+    for permission in _EDITOR_PERMISSIONS:
+        connection.execute(
+            text(
+                """
+                UPDATE search_space_roles
+                SET permissions = array_remove(permissions, :permission)
+                WHERE name = 'Editor'
+                """
+            ),
+            {"permission": permission},
+        )
+
+    for permission in _VIEWER_PERMISSIONS:
+        connection.execute(
+            text(
+                """
+                UPDATE search_space_roles
+                SET permissions = array_remove(permissions, :permission)
+                WHERE name = 'Viewer'
+                """
+            ),
+            {"permission": permission},
+        )

surfsense_backend/alembic/versions/146_drop_surfsense_docs_tables.py

@@ -0,0 +1,129 @@
+"""Drop Surfsense docs tables (feature removed end to end)
+
+Revision ID: 146
+Revises: 145
+Create Date: 2026-05-28
+
+Removes the SurfSense product-documentation feature: the
+``surfsense_docs_documents`` and ``surfsense_docs_chunks`` tables (created
+in revision 60) and the GIN trigram index on the title column (added in
+revision 67). The docs were seeded at startup from local MDX files, so no
+user data is lost. Downgrade recreates the tables and indexes.
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+from app.config import config
+
+# revision identifiers, used by Alembic.
+revision: str = "146"
+down_revision: str | None = "145"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+# Embedding dimension is required to recreate the vector columns on downgrade.
+EMBEDDING_DIM = config.embedding_model_instance.dimension
+
+
+def upgrade() -> None:
+    """Drop surfsense docs tables and all their indexes."""
+    # Trigram index from revision 67
+    op.execute("DROP INDEX IF EXISTS idx_surfsense_docs_title_trgm")
+
+    # Full-text search indexes
+    op.execute("DROP INDEX IF EXISTS surfsense_docs_chunks_search_index")
+    op.execute("DROP INDEX IF EXISTS surfsense_docs_documents_search_index")
+
+    # Vector indexes
+    op.execute("DROP INDEX IF EXISTS surfsense_docs_chunks_vector_index")
+    op.execute("DROP INDEX IF EXISTS surfsense_docs_documents_vector_index")
+
+    # B-tree indexes
+    op.execute("DROP INDEX IF EXISTS ix_surfsense_docs_chunks_document_id")
+    op.execute("DROP INDEX IF EXISTS ix_surfsense_docs_documents_updated_at")
+    op.execute("DROP INDEX IF EXISTS ix_surfsense_docs_documents_content_hash")
+    op.execute("DROP INDEX IF EXISTS ix_surfsense_docs_documents_source")
+
+    # Tables (chunks first due to FK)
+    op.execute("DROP TABLE IF EXISTS surfsense_docs_chunks")
+    op.execute("DROP TABLE IF EXISTS surfsense_docs_documents")
+
+
+def downgrade() -> None:
+    """Recreate surfsense docs tables and indexes (reverses revisions 60 + 67)."""
+    op.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS surfsense_docs_documents (
+            id SERIAL PRIMARY KEY,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+            source VARCHAR NOT NULL UNIQUE,
+            title VARCHAR NOT NULL,
+            content TEXT NOT NULL,
+            content_hash VARCHAR NOT NULL,
+            embedding vector({EMBEDDING_DIM}),
+            updated_at TIMESTAMP WITH TIME ZONE
+        );
+        """
+    )
+    op.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS surfsense_docs_chunks (
+            id SERIAL PRIMARY KEY,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+            content TEXT NOT NULL,
+            embedding vector({EMBEDDING_DIM}),
+            document_id INTEGER NOT NULL REFERENCES surfsense_docs_documents(id) ON DELETE CASCADE
+        );
+        """
+    )
+
+    # B-tree indexes
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_surfsense_docs_documents_source ON surfsense_docs_documents(source)"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_surfsense_docs_documents_content_hash ON surfsense_docs_documents(content_hash)"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_surfsense_docs_documents_updated_at ON surfsense_docs_documents(updated_at)"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_surfsense_docs_chunks_document_id ON surfsense_docs_chunks(document_id)"
+    )
+
+    # Vector indexes
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS surfsense_docs_documents_vector_index
+        ON surfsense_docs_documents USING hnsw (embedding public.vector_cosine_ops);
+        """
+    )
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS surfsense_docs_chunks_vector_index
+        ON surfsense_docs_chunks USING hnsw (embedding public.vector_cosine_ops);
+        """
+    )
+
+    # Full-text search indexes
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS surfsense_docs_documents_search_index
+        ON surfsense_docs_documents USING gin (to_tsvector('english', content));
+        """
+    )
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS surfsense_docs_chunks_search_index
+        ON surfsense_docs_chunks USING gin (to_tsvector('english', content));
+        """
+    )
+
+    # Trigram index from revision 67
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS idx_surfsense_docs_title_trgm
+        ON surfsense_docs_documents USING gin (title gin_trgm_ops);
+        """
+    )

surfsense_backend/alembic/versions/147_add_event_to_automation_trigger_type.py

@@ -0,0 +1,47 @@
+"""Add 'event' to automation_trigger_type enum
+
+Revision ID: 147
+Revises: 146
+Create Date: 2026-05-29
+
+Adds the ``event`` value to the ``automation_trigger_type`` enum so automations
+can be triggered by published domain events, alongside the existing
+``schedule`` triggers.
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "147"
+down_revision: str | None = "146"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+ENUM_NAME = "automation_trigger_type"
+NEW_VALUE = "event"
+
+
+def upgrade() -> None:
+    """Safely add 'event' to automation_trigger_type enum if missing."""
+    op.execute(
+        f"""
+    DO $$
+    BEGIN
+        IF NOT EXISTS (
+            SELECT 1 FROM pg_type t
+            JOIN pg_enum e ON t.oid = e.enumtypid
+            WHERE t.typname = '{ENUM_NAME}' AND e.enumlabel = '{NEW_VALUE}'
+        ) THEN
+            ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}';
+        END IF;
+    END
+    $$;
+    """
+    )
+
+
+def downgrade() -> None:
+    """No-op: PostgreSQL does not support removing enum values."""
+    pass