apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-02 22:01:05 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge upstream/dev into feature/human-in-the-loop

Browse source
This commit is contained in:
CREDO23 2026-02-13 20:17:59 +02:00
commit 66a6fb685e
47 changed files with 7257 additions and 4582 deletions
Download patch file Download diff file Expand all files Collapse all files

17
surfsense_backend/Dockerfile
View file

@ -22,6 +22,23 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
git \
&& rm -rf /var/lib/apt/lists/*
# Install Pandoc 3.x from GitHub (apt ships 2.17 which has broken table rendering).
RUN ARCH=$(dpkg --print-architecture) && \
wget -qO /tmp/pandoc.deb "https://github.com/jgm/pandoc/releases/download/3.9/pandoc-3.9-1-${ARCH}.deb" && \
dpkg -i /tmp/pandoc.deb && \
rm /tmp/pandoc.deb
# Install Typst for PDF rendering (Typst has built-in professional styling
# for tables, headings, code blocks, etc., no CSS needed).
RUN ARCH=$(dpkg --print-architecture) && \
if [ "$ARCH" = "amd64" ]; then TYPST_ARCH="x86_64-unknown-linux-musl"; \
else TYPST_ARCH="aarch64-unknown-linux-musl"; fi && \
wget -qO /tmp/typst.tar.xz "https://github.com/typst/typst/releases/download/v0.14.2/typst-${TYPST_ARCH}.tar.xz" && \
tar -xf /tmp/typst.tar.xz -C /tmp && \
cp /tmp/typst-*/typst /usr/local/bin/typst && \
rm -rf /tmp/typst* && \
typst --version
# Update certificates and install SSL tools
RUN update-ca-certificates
RUN pip install --upgrade certifi pip-system-certs

69
surfsense_backend/alembic/versions/100_add_report_group_id.py Normal file
View file

@ -0,0 +1,69 @@
"""Add report_group_id for report versioning
Revision ID: 100
Revises: 99
Create Date: 2026-02-11
Adds report_group_id column to reports table for grouping report versions.
Reports with the same report_group_id are versions of the same report.
For the first version (v1), report_group_id equals the report's own id.
Migration is idempotent — safe to re-run.
"""
from collections.abc import Sequence
from alembic import op
revision: str = "100"
down_revision: str | None = "99"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# Add report_group_id column (idempotent)
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'reports' AND column_name = 'report_group_id'
) THEN
ALTER TABLE reports ADD COLUMN report_group_id INTEGER;
END IF;
END $$;
"""
)
# Backfill existing reports: set report_group_id = id (each is its own v1)
op.execute(
"""
UPDATE reports SET report_group_id = id WHERE report_group_id IS NULL;
"""
)
# Create index (idempotent)
op.execute(
"""
CREATE INDEX IF NOT EXISTS ix_reports_report_group_id
ON reports(report_group_id);
"""
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_reports_report_group_id")
op.execute(
"""
DO $$
BEGIN
IF EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'reports' AND column_name = 'report_group_id'
) THEN
ALTER TABLE reports DROP COLUMN report_group_id;
END IF;
END $$;
"""
)

66
surfsense_backend/alembic/versions/99_add_reports_table.py Normal file
View file

@ -0,0 +1,66 @@
"""Add reports table
Revision ID: 99
Revises: 98
Create Date: 2026-02-11
Adds reports table for storing generated Markdown reports.
"""
from collections.abc import Sequence
from alembic import op
revision: str = "99"
down_revision: str | None = "98"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# Create the reports table
op.execute(
"""
CREATE TABLE IF NOT EXISTS reports (
id SERIAL PRIMARY KEY,
title VARCHAR(500) NOT NULL,
content TEXT,
report_metadata JSONB,
report_style VARCHAR(100),
search_space_id INTEGER NOT NULL
REFERENCES searchspaces(id) ON DELETE CASCADE,
thread_id INTEGER
REFERENCES new_chat_threads(id) ON DELETE SET NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
);
"""
)
# Add indexes
op.execute(
"""
CREATE INDEX IF NOT EXISTS ix_reports_search_space_id
ON reports(search_space_id);
"""
)
op.execute(
"""
CREATE INDEX IF NOT EXISTS ix_reports_thread_id
ON reports(thread_id);
"""
)
op.execute(
"""
CREATE INDEX IF NOT EXISTS ix_reports_created_at
ON reports(created_at);
"""
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_reports_created_at")
op.execute("DROP INDEX IF EXISTS ix_reports_thread_id")
op.execute("DROP INDEX IF EXISTS ix_reports_search_space_id")
op.execute("DROP TABLE IF EXISTS reports")

68
surfsense_backend/app/agents/new_chat/system_prompt.py
View file

@ -50,7 +50,7 @@ def _get_system_instructions(
return SURFSENSE_SYSTEM_INSTRUCTIONS.format(resolved_today=resolved_today)
# Tools 0-6 (common to both private and shared prompts)
# Tools 0-7 (common to both private and shared prompts)
_TOOLS_INSTRUCTIONS_COMMON = """
<tools>
You have access to the following tools:
@ -92,7 +92,44 @@ You have access to the following tools:
- IMPORTANT: Only one podcast can be generated at a time. If a podcast is already being generated, the tool will return status "already_generating".
- After calling this tool, inform the user that podcast generation has started and they will see the player when it's ready (takes 3-5 minutes).
3. link_preview: Fetch metadata for a URL to display a rich preview card.
3. generate_report: Generate a structured Markdown report from provided content.
- Use this when the user asks to create, generate, write, produce, draft, or summarize into a report-style deliverable.
- DECISION RULE (HIGH PRIORITY): If the user asks for a report in any form, call `generate_report` instead of writing the full report directly in chat.
- Only skip `generate_report` if the user explicitly asks for chat-only output (e.g., "just answer in chat", "no report card", "don't generate a report").
- Trigger classes include:
* Direct trigger words: report, document, memo, letter, template
* Creation-intent phrases: "write a document/report/post/article"
* File-intent words: requests containing "save", "file", or "document" when intent is to create a report-like deliverable
* Word-doc specific triggers: professional report-style deliverable, professional document, Word doc, .docx
* Other report-like output intents: one-pager, blog post, article, standalone written content, comprehensive guide
* General artifact-style intents: analysis / writing as substantial deliverables
- Trigger phrases include:
* "generate a report about", "write a report", "produce a report"
* "create a detailed report about", "make a research report on"
* "summarize this into a report", "turn this into a report"
* "write a report/document", "draft a report"
* "create an executive summary", "make a briefing note", "write a one-pager"
* "write a blog post", "write an article", "create a comprehensive guide"
* "create a small report", "write a short report", "make a quick report", "brief report for class"
- IMPORTANT FORMAT RULE: Reports are ALWAYS generated in Markdown.
- Args:
- topic: The main topic or title of the report
- source_content: The text content to base the report on. This MUST be comprehensive and include:
* If discussing the current conversation: Include a detailed summary of the FULL chat history (all user questions and your responses)
* If based on knowledge base search: Include the key findings and insights from the search results
* You can combine both: conversation context + search results for richer reports
* The more detailed the source_content, the better the report quality
- report_style: Optional style. Options: "detailed" (default), "executive_summary", "deep_research", "brief"
- user_instructions: Optional specific instructions (e.g., "focus on financial impacts", "include recommendations")
- Returns: A dictionary with status "ready" or "failed", report_id, title, and word_count.
- The report is generated immediately in Markdown and displayed inline in the chat.
- Export/download formats (e.g., PDF/DOCX) are produced from the generated Markdown report.
- SOURCE-COLLECTION RULE:
* If the user already provided enough source material (current chat content, uploaded files, pasted text, or a summarized video/article), generate the report directly from that.
* Use search_knowledge_base first when additional context is needed or the user asks for information beyond what is already available in the conversation.
- AFTER CALLING THIS TOOL: Do NOT repeat, summarize, or reproduce the report content in the chat. The report is already displayed as an interactive card that the user can open, read, copy, and export. Simply confirm that the report was generated (e.g., "I've generated your report on [topic]. You can view the Markdown report now, and export to PDF/DOCX from the card."). NEVER write out the report text in the chat.
4. link_preview: Fetch metadata for a URL to display a rich preview card.
- IMPORTANT: Use this tool WHENEVER the user shares or mentions a URL/link in their message.
- This fetches the page's Open Graph metadata (title, description, thumbnail) to show a preview card.
- NOTE: This tool only fetches metadata, NOT the full page content. It cannot read the article text.
@ -105,7 +142,7 @@ You have access to the following tools:
- Returns: A rich preview card with title, description, thumbnail, and domain
- The preview card will automatically be displayed in the chat.
4. display_image: Display an image in the chat with metadata.
5. display_image: Display an image in the chat with metadata.
- Use this tool ONLY when you have a valid public HTTP/HTTPS image URL to show.
- This displays the image with an optional title, description, and source attribution.
- Valid use cases:
@ -130,7 +167,7 @@ You have access to the following tools:
- Returns: An image card with the image, title, and description
- The image will automatically be displayed in the chat.
5. generate_image: Generate images from text descriptions using AI image models.
6. generate_image: Generate images from text descriptions using AI image models.
- Use this when the user asks you to create, generate, draw, design, or make an image.
- Trigger phrases: "generate an image of", "create a picture of", "draw me", "make an image", "design a logo", "create artwork"
- Args:
@ -144,7 +181,7 @@ You have access to the following tools:
expand and improve the prompt with specific details about style, lighting, composition, and mood.
- If the user's request is vague (e.g., "make me an image of a cat"), enhance the prompt with artistic details.
6. scrape_webpage: Scrape and extract the main content from a webpage.
7. scrape_webpage: Scrape and extract the main content from a webpage.
- Use this when the user wants you to READ and UNDERSTAND the actual content of a webpage.
- IMPORTANT: This is different from link_preview:
* link_preview: Only fetches metadata (title, description, thumbnail) for display
@ -169,9 +206,9 @@ You have access to the following tools:
"""
# Private (user) memory: tools 7-8 + memory-specific examples
# Private (user) memory: tools 8-9 + memory-specific examples
_TOOLS_INSTRUCTIONS_MEMORY_PRIVATE = """
7. save_memory: Save facts, preferences, or context for personalized responses.
8. save_memory: Save facts, preferences, or context for personalized responses.
- Use this when the user explicitly or implicitly shares information worth remembering.
- Trigger scenarios:
* User says "remember this", "keep this in mind", "note that", or similar
@ -194,7 +231,7 @@ _TOOLS_INSTRUCTIONS_MEMORY_PRIVATE = """
- IMPORTANT: Only save information that would be genuinely useful for future conversations.
Don't save trivial or temporary information.
8. recall_memory: Retrieve relevant memories about the user for personalized responses.
9. recall_memory: Retrieve relevant memories about the user for personalized responses.
- Use this to access stored information about the user.
- Trigger scenarios:
* You need user context to give a better, more personalized answer
@ -232,7 +269,7 @@ _TOOLS_INSTRUCTIONS_MEMORY_PRIVATE = """
# Shared (team) memory: tools 7-8 + team memory examples
_TOOLS_INSTRUCTIONS_MEMORY_SHARED = """
7. save_memory: Save a fact, preference, or context to the team's shared memory for future reference.
8. save_memory: Save a fact, preference, or context to the team's shared memory for future reference.
- Use this when the user or a team member says "remember this", "keep this in mind", or similar in this shared chat.
- Use when the team agrees on something to remember (e.g., decisions, conventions).
- Someone shares a preference or fact that should be visible to the whole team.
@ -247,7 +284,7 @@ _TOOLS_INSTRUCTIONS_MEMORY_SHARED = """
- Returns: Confirmation of saved memory; returned context may include who added it (added_by).
- IMPORTANT: Only save information that would be genuinely useful for future team conversations in this space.
8. recall_memory: Recall relevant team memories for this space to provide contextual responses.
9. recall_memory: Recall relevant team memories for this space to provide contextual responses.
- Use when you need team context to answer (e.g., "where do we store X?", "what did we decide about Y?").
- Use when someone asks about something the team agreed to remember.
- Use when team preferences or conventions would improve the response.
@ -321,6 +358,17 @@ _TOOLS_INSTRUCTIONS_EXAMPLES_COMMON = """
- First search: `search_knowledge_base(query="quantum computing")`
- Then: `generate_podcast(source_content="Key insights about quantum computing from the knowledge base:\\n\\n[Comprehensive summary of all relevant search results with key facts, concepts, and findings]", podcast_title="Quantum Computing Explained")`
- User: "Generate a report about AI trends"
- First search: `search_knowledge_base(query="AI trends")`
- Then: `generate_report(topic="AI Trends Report", source_content="Key insights about AI trends from the knowledge base:\\n\\n[Comprehensive summary of all relevant search results with key facts, concepts, and findings]", report_style="detailed")`
- User: "Write a research report from this conversation"
- Call: `generate_report(topic="Research Report", source_content="Complete conversation summary:\\n\\nUser asked about [topic 1]:\\n[Your detailed response]\\n\\nUser then asked about [topic 2]:\\n[Your detailed response]\\n\\n[Continue for all exchanges in the conversation]", report_style="deep_research")`
- User: "Create a brief executive summary about our project progress"
- First search: `search_knowledge_base(query="project progress updates")`
- Then: `generate_report(topic="Project Progress Executive Summary", source_content="[Combined search results and conversation context]", report_style="executive_summary", user_instructions="Focus on milestones achieved and upcoming deadlines")`
- User: "Check out https://dev.to/some-article"
- Call: `link_preview(url="https://dev.to/some-article")`
- Call: `scrape_webpage(url="https://dev.to/some-article")`

12
surfsense_backend/app/agents/new_chat/tools/registry.py
View file

@ -56,6 +56,7 @@ from .notion import (
create_update_notion_page_tool,
)
from .podcast import create_generate_podcast_tool
from .report import create_generate_report_tool
from .scrape_webpage import create_scrape_webpage_tool
from .search_surfsense_docs import create_search_surfsense_docs_tool
from .shared_memory import (
@ -123,6 +124,17 @@ BUILTIN_TOOLS: list[ToolDefinition] = [
),
requires=["search_space_id", "db_session", "thread_id"],
),
# Report generation tool (inline, no Celery)
ToolDefinition(
name="generate_report",
description="Generate a structured Markdown report from provided content",
factory=lambda deps: create_generate_report_tool(
search_space_id=deps["search_space_id"],
db_session=deps["db_session"],
thread_id=deps["thread_id"],
),
requires=["search_space_id", "db_session", "thread_id"],
),
# Link preview tool - fetches Open Graph metadata for URLs
ToolDefinition(
name="link_preview",

396
surfsense_backend/app/agents/new_chat/tools/report.py Normal file
View file

@ -0,0 +1,396 @@
"""
Report generation tool for the SurfSense agent.
This module provides a factory function for creating the generate_report tool
that generates a structured Markdown report inline (no Celery). The LLM is
called within the tool, the result is saved to the database, and the tool
returns immediately with a ready status.
This follows the same inline pattern as generate_image and display_image,
NOT the Celery-based podcast pattern.
"""
import logging
import re
from typing import Any
from langchain_core.tools import tool
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import Report
from app.services.llm_service import get_document_summary_llm
logger = logging.getLogger(__name__)
# Prompt template for report generation (new report from scratch)
_REPORT_PROMPT = """You are an expert report writer. Generate a well-structured, comprehensive Markdown report based on the provided information.
**Topic:** {topic}
**Report Style:** {report_style}
{user_instructions_section}
{previous_version_section}
**Source Content:**
{source_content}
---
**Instructions:**
1. Write the report in well-formatted Markdown.
2. Include a clear title (as a level-1 heading), an executive summary, and logically organized sections.
3. Use headings (##, ###), bullet points, numbered lists, bold/italic text, and tables where appropriate.
4. Cite specific facts, figures, and findings from the source content.
5. Be thorough and comprehensive — include all relevant information from the source content.
6. End with a conclusion or key takeaways section.
7. The report should be professional and ready to export.
8. When including code examples, ALWAYS format them as proper fenced code blocks with the correct language identifier (e.g. ```java, ```python). Code inside code blocks MUST have proper line breaks and indentation — NEVER put multiple statements on a single line. Each statement, brace, and logical block must be on its own line with correct indentation.
9. When including Mermaid diagrams, use ```mermaid fenced code blocks. Each Mermaid statement MUST be on its own line — NEVER use semicolons to join multiple statements on one line. For line breaks inside node labels, use <br> (NOT <br/>). Example:
```mermaid
graph TD
A[Source Code] --> B[Compiler]
B --> C[Bytecode]
```
Write the report now:
"""
def _strip_wrapping_code_fences(text: str) -> str:
"""Remove wrapping code fences that LLMs often add around Markdown output.
Handles patterns like:
```markdown\\n...content...\\n```
```md\\n...content...\\n```
```\\n...content...\\n```
"""
stripped = text.strip()
# Match opening fence with optional language tag (markdown, md, or bare)
m = re.match(r"^```(?:markdown|md)?\s*\n", stripped)
if m and stripped.endswith("```"):
stripped = stripped[m.end() :] # remove opening fence
stripped = stripped[:-3].rstrip() # remove closing fence
return stripped
def _extract_metadata(content: str) -> dict[str, Any]:
"""Extract metadata from generated Markdown content."""
# Count section headings
headings = re.findall(r"^(#{1,6})\s+(.+)$", content, re.MULTILINE)
# Word count
word_count = len(content.split())
# Character count
char_count = len(content)
return {
"status": "ready",
"word_count": word_count,
"char_count": char_count,
"section_count": len(headings),
}
def create_generate_report_tool(
search_space_id: int,
db_session: AsyncSession,
thread_id: int | None = None,
):
"""
Factory function to create the generate_report tool with injected dependencies.
The tool generates a Markdown report inline using the search space's
document summary LLM, saves it to the database, and returns immediately.
Args:
search_space_id: The user's search space ID
db_session: Database session for creating the report record
thread_id: The chat thread ID for associating the report
Returns:
A configured tool function for generating reports
"""
@tool
async def generate_report(
topic: str,
source_content: str,
report_style: str = "detailed",
user_instructions: str | None = None,
parent_report_id: int | None = None,
) -> dict[str, Any]:
"""
Generate a structured Markdown report from provided content.
Use this tool when the user asks to create, generate, write, produce, draft,
or summarize into a report-style deliverable.
HIGH-PRIORITY DECISION RULE:
- If the user asks for a report in any form,
call this tool rather than writing the full report directly in chat.
- Only skip this tool when the user explicitly requests chat-only output and
says they do not want a generated report card.
Trigger classes include:
- Direct trigger words: report, document, memo, letter, template
- Creation-intent phrases: "write a document/report/post/article"
- File-intent words: requests containing "save", "file", or "document" when
intent is to create a report-like deliverable
- Word-doc specific triggers: professional report-style deliverable,
professional document, Word doc, .docx
- Other report-like output intents: one-pager, blog post, article,
standalone written content, comprehensive guide
- General artifact-style intents: analysis / writing as substantial deliverables
Common triggers include phrases like:
- "Generate a report about this"
- "Write a report from this conversation"
- "Create a detailed report about..."
- "Make a research report on..."
- "Summarize this into a report"
- "Turn this into a report"
- "Write a report/document"
- "Draft a report"
- "Create an executive summary"
- "Make a briefing note"
- "Write a one-pager"
- "Write a blog post"
- "Write an article"
- "Create a comprehensive guide"
- "Prepare a report"
- "Create a small report"
- "Write a short report"
- "Make a quick report"
- "Brief report for class"
FORMAT/EXPORT RULE:
- Always generate the report content in Markdown.
- If the user requests DOCX/Word/PDF or another file format, export from
the generated Markdown report.
SOURCE-COLLECTION RULE:
- If enough source material is already present in the conversation (chat
history, pasted text, uploaded files, or a provided video/article summary),
generate directly from that source_content.
- Use knowledge-base search first only when extra context is needed beyond
what the user already provided.
VERSIONING — parent_report_id:
- Set parent_report_id when the user wants to MODIFY, REVISE, IMPROVE,
UPDATE, EXPAND, or ADD CONTENT TO an existing report that was already
generated in this conversation.
- This includes both explicit AND implicit modification requests. If the
user references the existing report using words like "it", "this",
"here", "the report", or clearly refers to a previously generated
report, treat it as a revision request.
- The value must be the report_id from a previous generate_report
result in this same conversation.
- Do NOT set parent_report_id when:
* The user asks for a report on a completely NEW/DIFFERENT topic
* The user says "generate another report" (new report, not a revision)
* There is no prior report to reference
- When parent_report_id is set, the previous report's content will be
used as a base. Your user_instructions should describe WHAT TO CHANGE.
Examples of when to SET parent_report_id:
User: "Make that report shorter" → parent_report_id = <previous report_id>
User: "Add a cost analysis section to the report" → parent_report_id = <previous report_id>
User: "Rewrite the report in a more formal tone" → parent_report_id = <previous report_id>
User: "I want more details about pricing in here" → parent_report_id = <previous report_id>
User: "Include more examples" → parent_report_id = <previous report_id>
User: "Can you also cover security in this?" → parent_report_id = <previous report_id>
User: "Make it more detailed" → parent_report_id = <previous report_id>
User: "I want more about X for in here" → parent_report_id = <previous report_id>
Examples of when to LEAVE parent_report_id as None:
User: "Generate a report on climate change" → parent_report_id = None (new topic)
User: "Write me a report about the budget" → parent_report_id = None (new topic)
User: "Create another report, this time about marketing" → parent_report_id = None
Args:
topic: A short, concise title for the report (maximum 8 words). Keep it brief and descriptive — e.g. "AI in Healthcare Analysis: A Comprehensive Report" instead of "Comprehensive Analysis of Artificial Intelligence Applications in Modern Healthcare Systems".
source_content: The text content to base the report on. This MUST be comprehensive and include:
* If discussing the current conversation: a detailed summary of the FULL chat history
* If based on knowledge base search: the key findings and insights from search results
* You can combine both: conversation context + search results for richer reports
* The more detailed the source_content, the better the report quality
report_style: Style of the report. Options: "detailed", "executive_summary", "deep_research", "brief". Default: "detailed"
user_instructions: Optional specific instructions for the report (e.g., "focus on financial impacts", "include recommendations"). When revising an existing report (parent_report_id is set), this should describe the changes to make.
parent_report_id: Optional ID of a previously generated report to revise. When set, the new report is created as a new version in the same version group. The previous report's content is included as context for the LLM to refine.
Returns:
A dictionary containing:
- status: "ready" or "failed"
- report_id: The report ID
- title: The report title
- word_count: Number of words in the report
- message: Status message (or "error" field if failed)
"""
# Resolve the parent report and its group (if versioning)
parent_report: Report | None = None
report_group_id: int | None = None
if parent_report_id:
parent_report = await db_session.get(Report, parent_report_id)
if parent_report:
report_group_id = parent_report.report_group_id
logger.info(
f"[generate_report] Creating new version from parent {parent_report_id} "
f"(group {report_group_id})"
)
else:
logger.warning(
f"[generate_report] parent_report_id={parent_report_id} not found, "
"creating standalone report"
)
async def _save_failed_report(error_msg: str) -> int | None:
"""Persist a failed report row so the error is visible later."""
try:
failed_report = Report(
title=topic,
content=None,
report_metadata={
"status": "failed",
"error_message": error_msg,
},
report_style=report_style,
search_space_id=search_space_id,
thread_id=thread_id,
report_group_id=report_group_id,
)
db_session.add(failed_report)
await db_session.commit()
await db_session.refresh(failed_report)
# If this is a new group (v1 failed), set group to self
if not failed_report.report_group_id:
failed_report.report_group_id = failed_report.id
await db_session.commit()
logger.info(
f"[generate_report] Saved failed report {failed_report.id}: {error_msg}"
)
return failed_report.id
except Exception:
logger.exception(
"[generate_report] Could not persist failed report row"
)
return None
try:
# Get the LLM instance for this search space
llm = await get_document_summary_llm(db_session, search_space_id)
if not llm:
error_msg = (
"No LLM configured. Please configure a language model in Settings."
)
report_id = await _save_failed_report(error_msg)
return {
"status": "failed",
"error": error_msg,
"report_id": report_id,
"title": topic,
}
# Build the prompt
user_instructions_section = ""
if user_instructions:
user_instructions_section = (
f"**Additional Instructions:** {user_instructions}"
)
# If revising, include previous version content
previous_version_section = ""
if parent_report and parent_report.content:
previous_version_section = (
"**Previous Version of This Report (refine this based on the instructions above — "
"preserve structure and quality, apply only the requested changes):**\n\n"
f"{parent_report.content}"
)
prompt = _REPORT_PROMPT.format(
topic=topic,
report_style=report_style,
user_instructions_section=user_instructions_section,
previous_version_section=previous_version_section,
source_content=source_content[:100000], # Cap source content
)
# Call the LLM inline
from langchain_core.messages import HumanMessage
response = await llm.ainvoke([HumanMessage(content=prompt)])
report_content = response.content
if not report_content or not isinstance(report_content, str):
error_msg = "LLM returned empty or invalid content"
report_id = await _save_failed_report(error_msg)
return {
"status": "failed",
"error": error_msg,
"report_id": report_id,
"title": topic,
}
# LLMs often wrap output in ```markdown ... ``` fences — strip them
# so the stored content is clean Markdown.
report_content = _strip_wrapping_code_fences(report_content)
if not report_content:
error_msg = "LLM returned empty or invalid content"
report_id = await _save_failed_report(error_msg)
return {
"status": "failed",
"error": error_msg,
"report_id": report_id,
"title": topic,
}
# Extract metadata (includes "status": "ready")
metadata = _extract_metadata(report_content)
# Save to database
report = Report(
title=topic,
content=report_content,
report_metadata=metadata,
report_style=report_style,
search_space_id=search_space_id,
thread_id=thread_id,
report_group_id=report_group_id, # None for v1, inherited for v2+
)
db_session.add(report)
await db_session.commit()
await db_session.refresh(report)
# If this is a brand-new report (v1), set report_group_id = own id
if not report.report_group_id:
report.report_group_id = report.id
await db_session.commit()
logger.info(
f"[generate_report] Created report {report.id} "
f"(group={report.report_group_id}): "
f"{metadata.get('word_count', 0)} words, "
f"{metadata.get('section_count', 0)} sections"
)
return {
"status": "ready",
"report_id": report.id,
"title": topic,
"word_count": metadata.get("word_count", 0),
"message": f"Report generated successfully: {topic}",
}
except Exception as e:
error_message = str(e)
logger.exception(f"[generate_report] Error: {error_message}")
report_id = await _save_failed_report(error_message)
return {
"status": "failed",
"error": error_message,
"report_id": report_id,
"title": topic,
}
return generate_report

36
surfsense_backend/app/db.py
View file

@ -1031,6 +1031,36 @@ class Podcast(BaseModel, TimestampMixin):
thread = relationship("NewChatThread")
class Report(BaseModel, TimestampMixin):
"""Report model for storing generated Markdown reports."""
__tablename__ = "reports"
title = Column(String(500), nullable=False)
content = Column(Text, nullable=True) # Markdown body
report_metadata = Column(JSONB, nullable=True) # section headings, word count, etc.
report_style = Column(
String(100), nullable=True
) # e.g. "executive_summary", "deep_research"
search_space_id = Column(
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False
)
search_space = relationship("SearchSpace", back_populates="reports")
# Versioning: reports sharing the same report_group_id are versions of the same report.
# For v1, report_group_id = the report's own id (set after insert).
report_group_id = Column(Integer, nullable=True, index=True)
thread_id = Column(
Integer,
ForeignKey("new_chat_threads.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
thread = relationship("NewChatThread")
class ImageGenerationConfig(BaseModel, TimestampMixin):
"""
Dedicated configuration table for image generation models.
@ -1185,6 +1215,12 @@ class SearchSpace(BaseModel, TimestampMixin):
order_by="Podcast.id.desc()",
cascade="all, delete-orphan",
)
reports = relationship(
"Report",
back_populates="search_space",
order_by="Report.id.desc()",
cascade="all, delete-orphan",
)
image_generations = relationship(
"ImageGeneration",
back_populates="search_space",

2
surfsense_backend/app/routes/__init__.py
View file

@ -34,6 +34,7 @@ from .notion_add_connector_route import router as notion_add_connector_router
from .podcasts_routes import router as podcasts_router
from .public_chat_routes import router as public_chat_router
from .rbac_routes import router as rbac_router
from .reports_routes import router as reports_router
from .search_source_connectors_routes import router as search_source_connectors_router
from .search_spaces_routes import router as search_spaces_router
from .slack_add_connector_route import router as slack_add_connector_router
@ -50,6 +51,7 @@ router.include_router(notes_router)
router.include_router(new_chat_router) # Chat with assistant-ui persistence
router.include_router(chat_comments_router)
router.include_router(podcasts_router) # Podcast task status and audio
router.include_router(reports_router) # Report CRUD and export (PDF/DOCX)
router.include_router(image_generation_router) # Image generation via litellm
router.include_router(search_source_connectors_router)
router.include_router(google_calendar_add_connector_router)

35
surfsense_backend/app/routes/public_chat_routes.py
View file

@ -20,6 +20,7 @@ from app.services.public_chat_service import (
clone_from_snapshot,
get_public_chat,
get_snapshot_podcast,
get_snapshot_report,
)
from app.users import current_active_user
@ -114,3 +115,37 @@ async def stream_public_podcast(
"Content-Disposition": f"inline; filename={os.path.basename(file_path)}",
},
)
@router.get("/{share_token}/reports/{report_id}/content")
async def get_public_report_content(
share_token: str,
report_id: int,
session: AsyncSession = Depends(get_async_session),
):
"""
Get report content from a public chat snapshot.
No authentication required - the share_token provides access.
Returns report content including title, markdown body, metadata, and versions.
"""
from app.services.public_chat_service import get_snapshot_report_versions
report_info = await get_snapshot_report(session, share_token, report_id)
if not report_info:
raise HTTPException(status_code=404, detail="Report not found")
# Get version siblings from the same snapshot
versions = await get_snapshot_report_versions(
session, share_token, report_info.get("report_group_id")
)
return {
"id": report_info.get("original_id"),
"title": report_info.get("title"),
"content": report_info.get("content"),
"report_metadata": report_info.get("report_metadata"),
"report_group_id": report_info.get("report_group_id"),
"versions": versions,
}

337
surfsense_backend/app/routes/reports_routes.py Normal file
View file

@ -0,0 +1,337 @@
"""
Report routes for read, export (PDF/DOCX), and delete operations.
No create or update endpoints here — reports are generated inline by the
agent tool during chat and stored as Markdown in the database.
Export to PDF/DOCX is on-demand — PDF uses pypandoc (Markdown→Typst) + typst-py
(Typst→PDF); DOCX uses pypandoc directly.
Authorization: lightweight search-space membership checks (no granular RBAC)
since reports are chat-generated artifacts, not standalone managed resources.
"""
import asyncio
import io
import logging
import os
import re
import tempfile
from enum import Enum
import pypandoc
import typst
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import StreamingResponse
from sqlalchemy import select
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import (
Report,
SearchSpace,
SearchSpaceMembership,
User,
get_async_session,
)
from app.schemas import ReportContentRead, ReportRead
from app.schemas.reports import ReportVersionInfo
from app.users import current_active_user
from app.utils.rbac import check_search_space_access
logger = logging.getLogger(__name__)
router = APIRouter()
MAX_REPORT_LIST_LIMIT = 500
class ExportFormat(str, Enum):
PDF = "pdf"
DOCX = "docx"
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
_CODE_FENCE_RE = re.compile(r"^```(?:markdown|md)?\s*\n", re.MULTILINE)
def _strip_wrapping_code_fences(text: str) -> str:
"""Remove wrapping code fences (```markdown...```) that LLMs often add."""
stripped = text.strip()
m = _CODE_FENCE_RE.match(stripped)
if m and stripped.endswith("```"):
stripped = stripped[m.end() : -3].rstrip()
return stripped
async def _get_report_with_access(
report_id: int,
session: AsyncSession,
user: User,
) -> Report:
"""Fetch a report and verify the user belongs to its search space.
Raises HTTPException(404) if not found, HTTPException(403) if no access.
"""
result = await session.execute(select(Report).filter(Report.id == report_id))
report = result.scalars().first()
if not report:
raise HTTPException(status_code=404, detail="Report not found")
# Lightweight membership check - no granular RBAC, just "is the user a
# member of the search space this report belongs to?"
await check_search_space_access(session, user, report.search_space_id)
return report
async def _get_version_siblings(
session: AsyncSession,
report: Report,
) -> list[ReportVersionInfo]:
"""Get all versions in the same report group, ordered by created_at."""
if not report.report_group_id:
# Legacy report without group — it's the only version
return [ReportVersionInfo(id=report.id, created_at=report.created_at)]
result = await session.execute(
select(Report.id, Report.created_at)
.filter(Report.report_group_id == report.report_group_id)
.order_by(Report.created_at.asc())
)
rows = result.all()
return [ReportVersionInfo(id=row[0], created_at=row[1]) for row in rows]
# ---------------------------------------------------------------------------
# Routes
# ---------------------------------------------------------------------------
@router.get("/reports", response_model=list[ReportRead])
async def read_reports(
skip: int = Query(default=0, ge=0),
limit: int = Query(default=100, ge=1, le=MAX_REPORT_LIST_LIMIT),
search_space_id: int | None = None,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
List reports the user has access to.
Filters by search space membership.
"""
try:
if search_space_id is not None:
# Verify the caller is a member of the requested search space
await check_search_space_access(session, user, search_space_id)
result = await session.execute(
select(Report)
.filter(Report.search_space_id == search_space_id)
.order_by(Report.id.desc())
.offset(skip)
.limit(limit)
)
else:
result = await session.execute(
select(Report)
.join(SearchSpace)
.join(SearchSpaceMembership)
.filter(SearchSpaceMembership.user_id == user.id)
.order_by(Report.id.desc())
.offset(skip)
.limit(limit)
)
return result.scalars().all()
except HTTPException:
raise
except SQLAlchemyError:
raise HTTPException(
status_code=500, detail="Database error occurred while fetching reports"
) from None
@router.get("/reports/{report_id}", response_model=ReportRead)
async def read_report(
report_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
Get a specific report by ID (metadata only, no content).
"""
try:
return await _get_report_with_access(report_id, session, user)
except HTTPException:
raise
except SQLAlchemyError:
raise HTTPException(
status_code=500, detail="Database error occurred while fetching report"
) from None
@router.get("/reports/{report_id}/content", response_model=ReportContentRead)
async def read_report_content(
report_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
Get full Markdown content of a report, including version siblings.
"""
try:
report = await _get_report_with_access(report_id, session, user)
versions = await _get_version_siblings(session, report)
return ReportContentRead(
id=report.id,
title=report.title,
content=report.content,
report_metadata=report.report_metadata,
report_group_id=report.report_group_id,
versions=versions,
)
except HTTPException:
raise
except SQLAlchemyError:
raise HTTPException(
status_code=500,
detail="Database error occurred while fetching report content",
) from None
@router.get("/reports/{report_id}/export")
async def export_report(
report_id: int,
format: ExportFormat = Query(
ExportFormat.PDF, description="Export format: pdf or docx"
),
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
Export a report as PDF or DOCX.
"""
try:
report = await _get_report_with_access(report_id, session, user)
if not report.content:
raise HTTPException(
status_code=400, detail="Report has no content to export"
)
# Strip wrapping code fences that LLMs sometimes add around Markdown.
# Without this, pandoc treats the entire content as a code block.
markdown_content = _strip_wrapping_code_fences(report.content)
# Convert Markdown to the requested format.
#
# DOCX: pypandoc (pandoc) handles the full conversion directly.
#
# PDF: two-step pipeline — pypandoc converts Markdown → Typst markup,
# then the `typst` Python library compiles Typst → PDF. This avoids
# requiring the Typst CLI on the system PATH; the typst pip package
# bundles the compiler as a native extension. Typst produces
# professional styling for tables, headings, code blocks, etc.
#
# Use "gfm" as the input format because LLM output uses GFM-style
# pipe tables that pandoc's stricter default "markdown" may mangle.
def _convert_and_read() -> bytes:
"""Run all blocking I/O (tempfile, pandoc/typst, file read, cleanup) in a thread."""
if format == ExportFormat.PDF:
# Step 1: Markdown → Typst markup via pandoc.
# We must set mainfont / monofont so the generated template's
# `font` parameter is non-empty; without it pandoc emits
# `font: ()` which makes Typst error with
# "font fallback list must not be empty".
# We use fonts that ship embedded inside typst-py so this
# works even on systems with no fonts installed.
typst_markup: str = pypandoc.convert_text(
markdown_content,
"typst",
format="gfm",
extra_args=[
"--standalone",
"-V",
"mainfont:Libertinus Serif",
"-V",
"monofont:DejaVu Sans Mono",
],
)
# Step 2: Typst markup → PDF via typst Python library
pdf_bytes: bytes = typst.compile(typst_markup.encode("utf-8"))
return pdf_bytes
else:
# DOCX: let pandoc handle the full conversion
fd, tmp_path = tempfile.mkstemp(suffix=f".{format.value}")
os.close(fd)
try:
pypandoc.convert_text(
markdown_content,
format.value,
format="gfm",
extra_args=["--standalone"],
outputfile=tmp_path,
)
with open(tmp_path, "rb") as f:
return f.read()
finally:
os.unlink(tmp_path)
loop = asyncio.get_running_loop()
output = await loop.run_in_executor(None, _convert_and_read)
# Sanitize filename
safe_title = (
"".join(
c if c.isalnum() or c in " -_" else "_" for c in report.title
).strip()[:80]
or "report"
)
media_types = {
ExportFormat.PDF: "application/pdf",
ExportFormat.DOCX: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
}
return StreamingResponse(
io.BytesIO(output),
media_type=media_types[format],
headers={
"Content-Disposition": f'attachment; filename="{safe_title}.{format.value}"',
},
)
except HTTPException:
raise
except Exception as e:
logger.exception("Report export failed")
raise HTTPException(status_code=500, detail=f"Export failed: {e!s}") from e
@router.delete("/reports/{report_id}", response_model=dict)
async def delete_report(
report_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
Delete a report.
"""
try:
db_report = await _get_report_with_access(report_id, session, user)
await session.delete(db_report)
await session.commit()
return {"message": "Report deleted successfully"}
except HTTPException:
raise
except SQLAlchemyError:
await session.rollback()
raise HTTPException(
status_code=500, detail="Database error occurred while deleting report"
) from None

6
surfsense_backend/app/schemas/__init__.py
View file

@ -76,6 +76,7 @@ from .rbac_schemas import (
RoleUpdate,
UserSearchSpaceAccess,
)
from .reports import ReportBase, ReportContentRead, ReportRead, ReportVersionInfo
from .search_source_connector import (
MCPConnectorCreate,
MCPConnectorRead,
@ -185,6 +186,11 @@ __all__ = [
"PodcastUpdate",
"RefreshTokenRequest",
"RefreshTokenResponse",
# Report schemas
"ReportBase",
"ReportContentRead",
"ReportRead",
"ReportVersionInfo",
"RoleCreate",
"RoleRead",
"RoleUpdate",

53
surfsense_backend/app/schemas/reports.py Normal file
View file

@ -0,0 +1,53 @@
"""Report schemas for API responses."""
from datetime import datetime
from typing import Any
from pydantic import BaseModel
class ReportBase(BaseModel):
"""Base report schema."""
title: str
content: str | None = None
report_style: str | None = None
search_space_id: int
class ReportRead(BaseModel):
"""Schema for reading a report (list view, no content)."""
id: int
title: str
report_style: str | None = None
report_metadata: dict[str, Any] | None = None
report_group_id: int | None = None
created_at: datetime
class Config:
from_attributes = True
class ReportVersionInfo(BaseModel):
"""Lightweight version entry for the version switcher UI."""
id: int
created_at: datetime
class Config:
from_attributes = True
class ReportContentRead(BaseModel):
"""Schema for reading a report with full Markdown content."""
id: int
title: str
content: str | None = None
report_metadata: dict[str, Any] | None = None
report_group_id: int | None = None
versions: list[ReportVersionInfo] = []
class Config:
from_attributes = True

139
surfsense_backend/app/services/public_chat_service.py
View file

@ -29,6 +29,7 @@ from app.db import (
Podcast,
PodcastStatus,
PublicChatSnapshot,
Report,
SearchSpaceMembership,
User,
)
@ -38,6 +39,7 @@ UI_TOOLS = {
"display_image",
"link_preview",
"generate_podcast",
"generate_report",
"scrape_webpage",
"multi_link_preview",
}
@ -195,19 +197,22 @@ async def create_snapshot(
message_ids = []
podcasts_data = []
podcast_ids_seen: set[int] = set()
reports_data = []
report_ids_seen: set[int] = set()
for msg in sorted(thread.messages, key=lambda m: m.created_at):
author = await get_author_display(session, msg.author_id, user_cache)
sanitized_content = sanitize_content_for_public(msg.content)
# Extract podcast references and update status to "ready" for completed podcasts
# Extract podcast/report references and update status to "ready" for completed ones
if isinstance(sanitized_content, list):
for part in sanitized_content:
if (
isinstance(part, dict)
and part.get("type") == "tool-call"
and part.get("toolName") == "generate_podcast"
):
if not isinstance(part, dict) or part.get("type") != "tool-call":
continue
tool_name = part.get("toolName")
if tool_name == "generate_podcast":
result_data = part.get("result", {})
podcast_id = result_data.get("podcast_id")
if podcast_id and podcast_id not in podcast_ids_seen:
@ -220,6 +225,17 @@ async def create_snapshot(
# Update status to "ready" so frontend renders PodcastPlayer
part["result"] = {**result_data, "status": "ready"}
elif tool_name == "generate_report":
result_data = part.get("result", {})
report_id = result_data.get("report_id")
if report_id and report_id not in report_ids_seen:
report_info = await _get_report_for_snapshot(session, report_id)
if report_info:
reports_data.append(report_info)
report_ids_seen.add(report_id)
# Update status to "ready" so frontend renders ReportCard
part["result"] = {**result_data, "status": "ready"}
messages_data.append(
{
"id": msg.id,
@ -266,6 +282,7 @@ async def create_snapshot(
"author": thread_author,
"messages": messages_data,
"podcasts": podcasts_data,
"reports": reports_data,
}
# Create new snapshot
@ -309,6 +326,27 @@ async def _get_podcast_for_snapshot(
}
async def _get_report_for_snapshot(
session: AsyncSession,
report_id: int,
) -> dict | None:
"""Get report info for embedding in snapshot_data."""
result = await session.execute(select(Report).filter(Report.id == report_id))
report = result.scalars().first()
if not report:
return None
return {
"original_id": report.id,
"title": report.title,
"content": report.content,
"report_metadata": report.report_metadata,
"report_group_id": report.report_group_id,
"created_at": report.created_at.isoformat() if report.created_at else None,
}
# =============================================================================
# Snapshot Retrieval
# =============================================================================
@ -578,6 +616,7 @@ async def clone_from_snapshot(
data = snapshot.snapshot_data
messages_data = data.get("messages", [])
podcasts_lookup = {p.get("original_id"): p for p in data.get("podcasts", [])}
reports_lookup = {r.get("original_id"): r for r in data.get("reports", [])}
new_thread = NewChatThread(
title=data.get("title", "Cloned Chat"),
@ -594,6 +633,7 @@ async def clone_from_snapshot(
await session.flush()
podcast_id_mapping: dict[int, int] = {}
report_id_mapping: dict[int, int] = {}
# Check which authors from snapshot still exist in DB
author_ids_from_snapshot: set[UUID] = set()
@ -655,6 +695,37 @@ async def clone_from_snapshot(
"podcast_id": podcast_id_mapping[old_podcast_id],
}
if (
isinstance(part, dict)
and part.get("type") == "tool-call"
and part.get("toolName") == "generate_report"
):
result = part.get("result", {})
old_report_id = result.get("report_id")
if old_report_id and old_report_id not in report_id_mapping:
report_info = reports_lookup.get(old_report_id)
if report_info:
new_report = Report(
title=report_info.get("title", "Cloned Report"),
content=report_info.get("content"),
report_metadata=report_info.get("report_metadata"),
search_space_id=target_search_space_id,
thread_id=new_thread.id,
)
session.add(new_report)
await session.flush()
# For cloned reports, set report_group_id = own id
# (each cloned report starts as its own v1)
new_report.report_group_id = new_report.id
report_id_mapping[old_report_id] = new_report.id
if old_report_id and old_report_id in report_id_mapping:
part["result"] = {
**result,
"report_id": report_id_mapping[old_report_id],
}
new_message = NewChatMessage(
thread_id=new_thread.id,
role=role,
@ -696,3 +767,59 @@ async def get_snapshot_podcast(
return podcast
return None
async def get_snapshot_report(
session: AsyncSession,
share_token: str,
report_id: int,
) -> dict | None:
"""
Get report info from a snapshot by original report ID.
Used for displaying report content in public view.
Looks up the report by its original_id in the snapshot's reports array.
"""
snapshot = await get_snapshot_by_token(session, share_token)
if not snapshot:
return None
reports = snapshot.snapshot_data.get("reports", [])
# Find report by original_id
for report in reports:
if report.get("original_id") == report_id:
return report
return None
async def get_snapshot_report_versions(
session: AsyncSession,
share_token: str,
report_group_id: int | None,
) -> list[dict]:
"""
Get all report versions in the same group from a snapshot.
Returns a list of lightweight version entries (id + created_at)
for the version switcher UI, sorted by original_id (insertion order).
"""
if not report_group_id:
return []
snapshot = await get_snapshot_by_token(session, share_token)
if not snapshot:
return []
reports = snapshot.snapshot_data.get("reports", [])
siblings = [r for r in reports if r.get("report_group_id") == report_group_id]
# Sort by original_id (ascending = insertion order ≈ created_at order)
siblings.sort(key=lambda r: r.get("original_id", 0))
return [
{"id": r.get("original_id"), "created_at": r.get("created_at")}
for r in siblings
]

2
surfsense_backend/pyproject.toml
View file

@ -63,6 +63,8 @@ dependencies = [
"unstructured-client>=0.42.3",
"langchain-unstructured>=1.0.1",
"slowapi>=0.1.9",
"pypandoc_binary>=1.16.2",
"typst>=0.14.0",
]
[dependency-groups]

6437
surfsense_backend/uv.lock generated
View file

File diff suppressed because it is too large Load diff