Merge remote-tracking branch 'upstream/dev' into feat/ui-fixes

surfsense_backend/alembic/versions/152_add_document_files.py

@@ -0,0 +1,86 @@
+"""add document_files table for stored original uploads
+
+Revision ID: 152
+Revises: 151
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "152"
+down_revision: str | None = "151"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # The enum type must precede the table that references it.
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF NOT EXISTS (
+                SELECT 1 FROM pg_type WHERE typname = 'document_file_kind'
+            ) THEN
+                CREATE TYPE document_file_kind AS ENUM (
+                    'ORIGINAL', 'REDACTED', 'FILLED_FORM'
+                );
+            END IF;
+        END
+        $$;
+        """
+    )
+
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS document_files (
+            id SERIAL PRIMARY KEY,
+            document_id INTEGER NOT NULL
+                REFERENCES documents(id) ON DELETE CASCADE,
+            search_space_id INTEGER NOT NULL
+                REFERENCES searchspaces(id) ON DELETE CASCADE,
+            kind document_file_kind NOT NULL DEFAULT 'ORIGINAL',
+            storage_backend VARCHAR(32) NOT NULL,
+            storage_key TEXT NOT NULL,
+            original_filename TEXT NOT NULL,
+            mime_type TEXT,
+            size_bytes BIGINT NOT NULL,
+            checksum_sha256 VARCHAR(64),
+            created_by_id UUID
+                REFERENCES "user"(id) ON DELETE SET NULL,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+        );
+        """
+    )
+
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_document_files_document_id "
+        "ON document_files(document_id);"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_document_files_search_space_id "
+        "ON document_files(search_space_id);"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_document_files_kind "
+        "ON document_files(kind);"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_document_files_created_by_id "
+        "ON document_files(created_by_id);"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_document_files_created_at "
+        "ON document_files(created_at);"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS ix_document_files_created_at;")
+    op.execute("DROP INDEX IF EXISTS ix_document_files_created_by_id;")
+    op.execute("DROP INDEX IF EXISTS ix_document_files_kind;")
+    op.execute("DROP INDEX IF EXISTS ix_document_files_search_space_id;")
+    op.execute("DROP INDEX IF EXISTS ix_document_files_document_id;")
+    op.execute("DROP TABLE IF EXISTS document_files;")
+    op.execute("DROP TYPE IF EXISTS document_file_kind;")

surfsense_backend/alembic/versions/153_restore_automation_runs_to_zero_publication.py

@@ -0,0 +1,109 @@
+"""restore automation_runs to zero_publication
+
+Migration 149's ``SET TABLE`` dropped ``automation_runs`` (added in 148),
+breaking the dashboard live run ticker with a SchemaVersionNotSupported
+reload loop. Re-emit the publication with ``automation_runs`` using the
+``COMMENT`` bookend pattern so zero-cache fires its schema-change hook.
+
+Revision ID: 153
+Revises: 152
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision: str = "153"
+down_revision: str | None = "152"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+PUBLICATION_NAME = "zero_publication"
+
+DOCUMENT_COLS = [
+    "id",
+    "title",
+    "document_type",
+    "search_space_id",
+    "folder_id",
+    "created_by_id",
+    "status",
+    "created_at",
+    "updated_at",
+]
+
+USER_COLS = [
+    "id",
+    "pages_limit",
+    "pages_used",
+    "premium_credit_micros_limit",
+    "premium_credit_micros_used",
+]
+
+AUTOMATION_RUN_COLS = [
+    "id",
+    "automation_id",
+    "trigger_id",
+    "status",
+    "step_results",
+    "started_at",
+    "finished_at",
+    "created_at",
+]
+
+
+def _has_zero_version(conn, table: str) -> bool:
+    return (
+        conn.execute(
+            sa.text(
+                "SELECT 1 FROM information_schema.columns "
+                "WHERE table_name = :tbl AND column_name = '_0_version'"
+            ),
+            {"tbl": table},
+        ).fetchone()
+        is not None
+    )
+
+
+def _set_table_ddl(*, with_automation_runs: bool, conn) -> str:
+    doc_cols = DOCUMENT_COLS + (['"_0_version"'] if _has_zero_version(conn, "documents") else [])
+    user_cols = USER_COLS + (['"_0_version"'] if _has_zero_version(conn, "user") else [])
+    tables = [
+        "notifications",
+        f"documents ({', '.join(doc_cols)})",
+        "folders",
+        "search_source_connectors",
+        "new_chat_messages",
+        "chat_comments",
+        "chat_session_state",
+        f'"user" ({", ".join(user_cols)})',
+    ]
+    if with_automation_runs:
+        tables.append(f"automation_runs ({', '.join(AUTOMATION_RUN_COLS)})")
+    return f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + ", ".join(tables)
+
+
+def _resync(*, with_automation_runs: bool, tag: str) -> None:
+    conn = op.get_bind()
+    exists = conn.execute(
+        sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"),
+        {"name": PUBLICATION_NAME},
+    ).fetchone()
+    if not exists:
+        return
+
+    tx = conn.begin_nested() if conn.in_transaction() else conn.begin()
+    with tx:
+        conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'"))
+        conn.execute(sa.text(_set_table_ddl(with_automation_runs=with_automation_runs, conn=conn)))
+        conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'"))
+
+
+def upgrade() -> None:
+    _resync(with_automation_runs=True, tag="153-resync")
+
+
+def downgrade() -> None:
+    _resync(with_automation_runs=False, tag="153-downgrade")