apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-04-25 00:36:31 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

chore: cleanup

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-01-07 19:07:06 -08:00
parent 33ab74f698
commit 48fc70a08b
22 changed files with 8 additions and 1540 deletions
Download patch file Download diff file Expand all files Collapse all files

7
surfsense_backend/.env.example
View file

@ -55,10 +55,11 @@ DISCORD_CLIENT_SECRET=your_discord_client_secret_here
DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback
DISCORD_BOT_TOKEN=your_bot_token_from_developer_portal
# Jira OAuth Configuration
JIRA_CLIENT_ID=your_jira_client_id_here
JIRA_CLIENT_SECRET=your_jira_client_secret_here
# Atlassian OAuth Configuration
ATLASSIAN_CLIENT_ID=V4Axk5VLcsAKJxffMjRGSHtlh17uVswl
ATLASSIAN_CLIENT_SECRET=ATOAmjcoJ_wpyr98F5nF9BVZFDtXpLHs53YnK8TVQhjJh2LuRPYrnDirBwW5lV5cWRbK9B430F02
JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback
CONFLUENCE_REDIRECT_URI=http://localhost:8000/api/v1/auth/confluence/connector/callback
# Linear OAuth Configuration
LINEAR_CLIENT_ID=your_linear_client_id_here

36
surfsense_backend/app/services/docling_service.py
View file

@ -128,42 +128,6 @@ class DoclingService:
logger.error(f"❌ Docling initialization failed: {e}")
raise RuntimeError(f"Docling initialization failed: {e}") from e
def _configure_easyocr_local_models(self):
"""Configure EasyOCR to use pre-downloaded local models."""
try:
import os
import easyocr
# Set SSL environment for EasyOCR downloads
os.environ["CURL_CA_BUNDLE"] = ""
os.environ["REQUESTS_CA_BUNDLE"] = ""
# Try to use local models first, fallback to download if needed
try:
reader = easyocr.Reader(
["en"],
download_enabled=False,
model_storage_directory="/root/.EasyOCR/model",
)
logger.info("✅ EasyOCR configured for local models")
return reader
except Exception:
# If local models fail, allow download with SSL bypass
logger.info(
"🔄 Local models failed, attempting download with SSL bypass..."
)
reader = easyocr.Reader(
["en"],
download_enabled=True,
model_storage_directory="/root/.EasyOCR/model",
)
logger.info("✅ EasyOCR configured with downloaded models")
return reader
except Exception as e:
logger.warning(f"⚠️ EasyOCR configuration failed: {e}")
return None
async def process_document(
self, file_path: str, filename: str | None = None
) -> dict[str, Any]:

55
surfsense_backend/app/services/llm_service.py
View file

@ -342,40 +342,7 @@ async def get_document_summary_llm(
)
# Backward-compatible aliases (deprecated - will be removed in future versions)
async def get_user_llm_instance(
session: AsyncSession, user_id: str, search_space_id: int, role: str
) -> ChatLiteLLM | None:
"""
Deprecated: Use get_search_space_llm_instance instead.
LLM preferences are now stored at the search space level, not per-user.
"""
return await get_search_space_llm_instance(session, search_space_id, role)
# Legacy aliases for backward compatibility
async def get_long_context_llm(
session: AsyncSession, search_space_id: int
) -> ChatLiteLLM | None:
"""Deprecated: Use get_document_summary_llm instead."""
return await get_document_summary_llm(session, search_space_id)
async def get_fast_llm(
session: AsyncSession, search_space_id: int
) -> ChatLiteLLM | None:
"""Deprecated: Use get_agent_llm instead."""
return await get_agent_llm(session, search_space_id)
async def get_strategic_llm(
session: AsyncSession, search_space_id: int
) -> ChatLiteLLM | None:
"""Deprecated: Use get_document_summary_llm instead."""
return await get_document_summary_llm(session, search_space_id)
# User-based legacy aliases (LLM preferences are now per-search-space, not per-user)
# Backward-compatible alias (LLM preferences are now per-search-space, not per-user)
async def get_user_long_context_llm(
session: AsyncSession, user_id: str, search_space_id: int
) -> ChatLiteLLM | None:
@ -384,23 +351,3 @@ async def get_user_long_context_llm(
The user_id parameter is ignored as LLM preferences are now per-search-space.
"""
return await get_document_summary_llm(session, search_space_id)
async def get_user_fast_llm(
session: AsyncSession, user_id: str, search_space_id: int
) -> ChatLiteLLM | None:
"""
Deprecated: Use get_agent_llm instead.
The user_id parameter is ignored as LLM preferences are now per-search-space.
"""
return await get_agent_llm(session, search_space_id)
async def get_user_strategic_llm(
session: AsyncSession, user_id: str, search_space_id: int
) -> ChatLiteLLM | None:
"""
Deprecated: Use get_document_summary_llm instead.
The user_id parameter is ignored as LLM preferences are now per-search-space.
"""
return await get_document_summary_llm(session, search_space_id)

114
surfsense_backend/app/services/query_service.py
View file

@ -1,114 +0,0 @@
import datetime
from typing import Any
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
from sqlalchemy.ext.asyncio import AsyncSession
from app.services.llm_service import get_document_summary_llm
class QueryService:
"""
Service for query-related operations, including reformulation and processing.
"""
@staticmethod
async def reformulate_query_with_chat_history(
user_query: str,
session: AsyncSession,
search_space_id: int,
chat_history_str: str | None = None,
) -> str:
"""
Reformulate the user query using the search space's document summary LLM to make it more
effective for information retrieval and research purposes.
Args:
user_query: The original user query
session: Database session for accessing LLM configs
search_space_id: Search Space ID to get LLM preferences
chat_history_str: Optional chat history string
Returns:
str: The reformulated query
"""
if not user_query or not user_query.strip():
return user_query
try:
# Get the search space's document summary LLM instance
llm = await get_document_summary_llm(session, search_space_id)
if not llm:
print(
f"Warning: No document summary LLM configured for search space {search_space_id}. Using original query."
)
return user_query
# Create system message with instructions
system_message = SystemMessage(
content=f"""
Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")}
You are a highly skilled AI assistant specializing in query optimization for advanced research.
Your primary objective is to transform a user's initial query into a highly effective search query.
This reformulated query will be used to retrieve information from diverse data sources.
**Chat History Context:**
{chat_history_str if chat_history_str else "No prior conversation history is available."}
If chat history is provided, analyze it to understand the user's evolving information needs and the broader context of their request. Use this understanding to refine the current query, ensuring it builds upon or clarifies previous interactions.
**Query Reformulation Guidelines:**
Your reformulated query should:
1. **Enhance Specificity and Detail:** Add precision to narrow the search focus effectively, making the query less ambiguous and more targeted.
2. **Resolve Ambiguities:** Identify and clarify vague terms or phrases. If a term has multiple meanings, orient the query towards the most likely one given the context.
3. **Expand Key Concepts:** Incorporate relevant synonyms, related terms, and alternative phrasings for core concepts. This helps capture a wider range of relevant documents.
4. **Deconstruct Complex Questions:** If the original query is multifaceted, break it down into its core searchable components or rephrase it to address each aspect clearly. The final output must still be a single, coherent query string.
5. **Optimize for Comprehensiveness:** Ensure the query is structured to uncover all essential facets of the original request, aiming for thorough information retrieval suitable for research.
6. **Maintain User Intent:** The reformulated query must stay true to the original intent of the user's query. Do not introduce new topics or shift the focus significantly.
**Crucial Constraints:**
* **Conciseness and Effectiveness:** While aiming for comprehensiveness, the reformulated query MUST be as concise as possible. Eliminate all unnecessary verbosity. Focus on essential keywords, entities, and concepts that directly contribute to effective retrieval.
* **Single, Direct Output:** Return ONLY the reformulated query itself. Do NOT include any explanations, introductory phrases (e.g., "Reformulated query:", "Here is the optimized query:"), or any other surrounding text or markdown formatting.
Your output should be a single, optimized query string, ready for immediate use in a search system.
"""
)
# Create human message with the user query
human_message = HumanMessage(
content=f"Reformulate this query for better research results: {user_query}"
)
# Get the response from the LLM
response = await llm.agenerate(messages=[[system_message, human_message]])
# Extract the reformulated query from the response
reformulated_query = response.generations[0][0].text.strip()
# Return the original query if the reformulation is empty
if not reformulated_query:
return user_query
return reformulated_query
except Exception as e:
# Log the error and return the original query
print(f"Error reformulating query: {e}")
return user_query
@staticmethod
async def langchain_chat_history_to_str(chat_history: list[Any]) -> str:
"""
Convert a list of chat history messages to a string.
"""
chat_history_str = "<chat_history>\n"
for chat_message in chat_history:
if isinstance(chat_message, HumanMessage):
chat_history_str += f"<user>{chat_message.content}</user>\n"
elif isinstance(chat_message, AIMessage):
chat_history_str += f"<assistant>{chat_message.content}</assistant>\n"
elif isinstance(chat_message, SystemMessage):
chat_history_str += f"<system>{chat_message.content}</system>\n"
chat_history_str += "</chat_history>"
return chat_history_str

82
surfsense_backend/app/utils/document_converters.py
View file

@ -222,88 +222,6 @@ async def convert_document_to_markdown(elements):
return "".join(markdown_parts)
def convert_chunks_to_langchain_documents(chunks):
"""
Convert chunks from hybrid search results to LangChain Document objects.
Args:
chunks: List of chunk dictionaries from hybrid search results
Returns:
List of LangChain Document objects
"""
try:
from langchain_core.documents import Document as LangChainDocument
except ImportError:
raise ImportError(
"LangChain is not installed. Please install it with `pip install langchain langchain-core`"
) from None
langchain_docs = []
for chunk in chunks:
# Extract content from the chunk
content = chunk.get("content", "")
# Create metadata dictionary
metadata = {
"chunk_id": chunk.get("chunk_id"),
"score": chunk.get("score"),
"rank": chunk.get("rank") if "rank" in chunk else None,
}
# Add document information to metadata
if "document" in chunk:
doc = chunk["document"]
metadata.update(
{
"document_id": doc.get("id"),
"document_title": doc.get("title"),
"document_type": doc.get("document_type"),
}
)
# Add document metadata if available
if "metadata" in doc:
# Prefix document metadata keys to avoid conflicts
doc_metadata = {
f"doc_meta_{k}": v for k, v in doc.get("metadata", {}).items()
}
metadata.update(doc_metadata)
# Add source URL if available in metadata
if "url" in doc.get("metadata", {}):
metadata["source"] = doc["metadata"]["url"]
elif "sourceURL" in doc.get("metadata", {}):
metadata["source"] = doc["metadata"]["sourceURL"]
# Ensure source_id is set for citation purposes
# Use document_id as the source_id if available
if "document_id" in metadata:
metadata["source_id"] = metadata["document_id"]
# Update content for citation mode - format as XML with explicit source_id
new_content = f"""
<document>
<metadata>
<source_id>{metadata.get("source_id", metadata.get("document_id", "unknown"))}</source_id>
</metadata>
<content>
<text>
{content}
</text>
</content>
</document>
"""
# Create LangChain Document
langchain_doc = LangChainDocument(page_content=new_content, metadata=metadata)
langchain_docs.append(langchain_doc)
return langchain_docs
def generate_content_hash(content: str, search_space_id: int) -> str:
"""Generate SHA-256 hash for the given content combined with search space ID."""
combined_data = f"{search_space_id}:{content}"

38
surfsense_web/components/copy-button.tsx
View file

@ -1,38 +0,0 @@
"use client";
import { Copy, CopyCheck } from "lucide-react";
import type { RefObject } from "react";
import { useEffect, useRef, useState } from "react";
import { Button } from "./ui/button";
export default function CopyButton({ ref }: { ref: RefObject<HTMLDivElement | null> }) {
const [copy, setCopy] = useState(false);
const timeoutRef = useRef<NodeJS.Timeout | null>(null);
useEffect(() => {
return () => {
if (timeoutRef.current) {
clearTimeout(timeoutRef.current);
}
};
}, []);
const handleClick = () => {
if (ref.current) {
const text = ref.current.innerText;
navigator.clipboard.writeText(text);
setCopy(true);
timeoutRef.current = setTimeout(() => {
setCopy(false);
}, 2000);
}
};
return (
<div className="w-full flex justify-end">
<Button variant="ghost" onClick={handleClick}>
{copy ? <CopyCheck /> : <Copy />}
</Button>
</div>
);
}

22
surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx
View file

@ -1,22 +0,0 @@
"use client";
import { Card, CardContent, CardHeader } from "@/components/ui/card";
import { Skeleton } from "@/components/ui/skeleton";
export function EditConnectorLoadingSkeleton() {
return (
<div className="container mx-auto py-8 max-w-3xl">
<Skeleton className="h-8 w-48 mb-6" />
<Card className="border-2 border-border">
<CardHeader>
<Skeleton className="h-7 w-3/4 mb-2" />
<Skeleton className="h-4 w-full" />
</CardHeader>
<CardContent className="space-y-4">
<Skeleton className="h-10 w-full" />
<Skeleton className="h-20 w-full" />
</CardContent>
</Card>
</div>
);
}

28
surfsense_web/components/editConnector/EditConnectorNameForm.tsx
View file

@ -1,28 +0,0 @@
"use client";
import type { Control } from "react-hook-form";
import { FormControl, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form";
import { Input } from "@/components/ui/input";
// Assuming EditConnectorFormValues is defined elsewhere or passed as generic
interface EditConnectorNameFormProps {
control: Control<any>; // Use Control<EditConnectorFormValues> if type is available
}
export function EditConnectorNameForm({ control }: EditConnectorNameFormProps) {
return (
<FormField
control={control}
name="name"
render={({ field }) => (
<FormItem>
<FormLabel>Connector Name</FormLabel>
<FormControl>
<Input {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
);
}

189
surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx
View file

@ -1,189 +0,0 @@
import { CircleAlert, Edit, KeyRound, Loader2 } from "lucide-react";
import type React from "react";
import type { UseFormReturn } from "react-hook-form";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import {
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import { Skeleton } from "@/components/ui/skeleton";
// Types needed from parent
interface GithubRepo {
id: number;
name: string;
full_name: string;
private: boolean;
url: string;
description: string | null;
last_updated: string | null;
}
type GithubPatFormValues = { github_pat: string };
type EditMode = "viewing" | "editing_repos";
interface EditGitHubConnectorConfigProps {
// State from parent
editMode: EditMode;
originalPat: string;
currentSelectedRepos: string[];
fetchedRepos: GithubRepo[] | null;
newSelectedRepos: string[];
isFetchingRepos: boolean;
// Forms from parent
patForm: UseFormReturn<GithubPatFormValues>;
// Handlers from parent
setEditMode: (mode: EditMode) => void;
handleFetchRepositories: (values: GithubPatFormValues) => Promise<void>;
handleRepoSelectionChange: (repoFullName: string, checked: boolean) => void;
setNewSelectedRepos: React.Dispatch<React.SetStateAction<string[]>>;
setFetchedRepos: React.Dispatch<React.SetStateAction<GithubRepo[] | null>>;
}
export function EditGitHubConnectorConfig({
editMode,
originalPat,
currentSelectedRepos,
fetchedRepos,
newSelectedRepos,
isFetchingRepos,
patForm,
setEditMode,
handleFetchRepositories,
handleRepoSelectionChange,
setNewSelectedRepos,
setFetchedRepos,
}: EditGitHubConnectorConfigProps) {
return (
<div className="space-y-4">
<h4 className="font-medium text-muted-foreground">Repository Selection & Access</h4>
{/* Viewing Mode */}
{editMode === "viewing" && (
<div className="space-y-3 p-4 border rounded-md bg-muted/50">
<FormLabel>Currently Indexed Repositories:</FormLabel>
{currentSelectedRepos.length > 0 ? (
<ul className="list-disc pl-5 text-sm">
{currentSelectedRepos.map((repo) => (
<li key={repo}>{repo}</li>
))}
</ul>
) : (
<p className="text-sm text-muted-foreground">(No repositories currently selected)</p>
)}
<Button
type="button"
variant="outline"
size="sm"
onClick={() => setEditMode("editing_repos")}
>
<Edit className="mr-2 h-4 w-4" /> Change Selection / Update PAT
</Button>
<FormDescription>
To change repo selections or update the PAT, click above.
</FormDescription>
</div>
)}
{/* Editing Mode */}
{editMode === "editing_repos" && (
<div className="space-y-4 p-4 border rounded-md">
{/* PAT Input */}
<div className="flex items-end gap-4 p-4 border rounded-md bg-muted/90">
<FormField
control={patForm.control}
name="github_pat"
render={({ field }) => (
<FormItem className="flex-grow">
<FormLabel className="flex items-center gap-1">
<KeyRound className="h-4 w-4" /> GitHub PAT
</FormLabel>
<FormControl>
<Input type="password" placeholder="ghp_... or github_pat_..." {...field} />
</FormControl>
<FormDescription>
Enter PAT to fetch/update repos or if you need to update the stored token.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<Button
type="button"
disabled={isFetchingRepos}
size="sm"
onClick={async () => {
const isValid = await patForm.trigger("github_pat");
if (isValid) {
handleFetchRepositories(patForm.getValues());
}
}}
>
{isFetchingRepos ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : (
"Fetch Repositories"
)}
</Button>
</div>
{/* Repo List */}
{isFetchingRepos && <Skeleton className="h-40 w-full" />}
{!isFetchingRepos &&
fetchedRepos !== null &&
(fetchedRepos.length === 0 ? (
<Alert variant="destructive">
<CircleAlert className="h-4 w-4" />
<AlertTitle>No Repositories Found</AlertTitle>
<AlertDescription>Check PAT & permissions.</AlertDescription>
</Alert>
) : (
<div className="space-y-2">
<FormLabel>
Select Repositories to Index ({newSelectedRepos.length} selected):
</FormLabel>
<div className="h-64 w-full rounded-md border p-4 overflow-y-auto">
{fetchedRepos.map((repo) => (
<div key={repo.id} className="flex items-center space-x-2 mb-2 py-1">
<Checkbox
id={`repo-${repo.id}`}
checked={newSelectedRepos.includes(repo.full_name)}
onCheckedChange={(checked) =>
handleRepoSelectionChange(repo.full_name, !!checked)
}
/>
<label
htmlFor={`repo-${repo.id}`}
className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
>
{repo.full_name} {repo.private && "(Private)"}
</label>
</div>
))}
</div>
</div>
))}
<Button
type="button"
variant="ghost"
size="sm"
onClick={() => {
setEditMode("viewing");
setFetchedRepos(null);
setNewSelectedRepos(currentSelectedRepos);
patForm.reset({ github_pat: originalPat }); // Reset PAT form on cancel
}}
>
Cancel Repo Change
</Button>
</div>
)}
</div>
);
}

49
surfsense_web/components/editConnector/EditSimpleTokenForm.tsx
View file

@ -1,49 +0,0 @@
"use client";
import { KeyRound } from "lucide-react";
import type { Control } from "react-hook-form";
import {
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
// Assuming EditConnectorFormValues is defined elsewhere or passed as generic
interface EditSimpleTokenFormProps {
control: Control<any>;
fieldName: string; // e.g., "SLACK_BOT_TOKEN"
fieldLabel: string; // e.g., "Slack Bot Token"
fieldDescription: string;
placeholder?: string;
}
export function EditSimpleTokenForm({
control,
fieldName,
fieldLabel,
fieldDescription,
placeholder,
}: EditSimpleTokenFormProps) {
return (
<FormField
control={control}
name={fieldName}
render={({ field }) => (
<FormItem>
<FormLabel className="flex items-center gap-1">
<KeyRound className="h-4 w-4" /> {fieldLabel}
</FormLabel>
<FormControl>
<Input type="password" placeholder={placeholder} {...field} />
</FormControl>
<FormDescription>{fieldDescription}</FormDescription>
<FormMessage />
</FormItem>
)}
/>
);
}

59
surfsense_web/components/editConnector/types.ts
View file

@ -1,59 +0,0 @@
import * as z from "zod";
// Types
export interface GithubRepo {
id: number;
name: string;
full_name: string;
private: boolean;
url: string;
description: string | null;
last_updated: string | null;
}
export type EditMode = "viewing" | "editing_repos";
// Schemas
export const githubPatSchema = z.object({
github_pat: z
.string()
.min(20, { message: "GitHub Personal Access Token seems too short." })
.refine((pat) => pat.startsWith("ghp_") || pat.startsWith("github_pat_"), {
message: "GitHub PAT should start with 'ghp_' or 'github_pat_'",
}),
});
export type GithubPatFormValues = z.infer<typeof githubPatSchema>;
export const editConnectorSchema = z.object({
name: z.string().min(3, { message: "Connector name must be at least 3 characters." }),
SLACK_BOT_TOKEN: z.string().optional(),
NOTION_INTEGRATION_TOKEN: z.string().optional(),
TAVILY_API_KEY: z.string().optional(),
SEARXNG_HOST: z.string().optional(),
SEARXNG_API_KEY: z.string().optional(),
SEARXNG_ENGINES: z.string().optional(),
SEARXNG_CATEGORIES: z.string().optional(),
SEARXNG_LANGUAGE: z.string().optional(),
SEARXNG_SAFESEARCH: z.string().optional(),
SEARXNG_VERIFY_SSL: z.string().optional(),
LINKUP_API_KEY: z.string().optional(),
DISCORD_BOT_TOKEN: z.string().optional(),
CONFLUENCE_BASE_URL: z.string().optional(),
CONFLUENCE_EMAIL: z.string().optional(),
CONFLUENCE_API_TOKEN: z.string().optional(),
BOOKSTACK_BASE_URL: z.string().optional(),
BOOKSTACK_TOKEN_ID: z.string().optional(),
BOOKSTACK_TOKEN_SECRET: z.string().optional(),
JIRA_BASE_URL: z.string().optional(),
JIRA_EMAIL: z.string().optional(),
JIRA_API_TOKEN: z.string().optional(),
GOOGLE_CALENDAR_CLIENT_ID: z.string().optional(),
GOOGLE_CALENDAR_CLIENT_SECRET: z.string().optional(),
GOOGLE_CALENDAR_REFRESH_TOKEN: z.string().optional(),
GOOGLE_CALENDAR_CALENDAR_IDS: z.string().optional(),
LUMA_API_KEY: z.string().optional(),
ELASTICSEARCH_API_KEY: z.string().optional(),
FIRECRAWL_API_KEY: z.string().optional(),
INITIAL_URLS: z.string().optional(),
});
export type EditConnectorFormValues = z.infer<typeof editConnectorSchema>;

1
surfsense_web/components/inference-params-editor.tsx
View file

@ -151,3 +151,4 @@ export default function InferenceParamsEditor({ params, setParams }: InferencePa
</div>
);
}

16
surfsense_web/components/settings/llm-role-manager.tsx
View file

@ -45,7 +45,7 @@ const ROLE_DESCRIPTIONS = {
document_summary: {
icon: FileText,
title: "Document Summary LLM",
description: "Handles document summarization, long context analysis, and query reformulation",
description: "Handles document summarization",
color: "bg-purple-100 text-purple-800 border-purple-200",
examples: "Document analysis, podcasts, research synthesis",
characteristics: ["Large context window", "Deep reasoning", "Summarization"],
@ -74,7 +74,6 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
data: preferences = {},
isFetching: preferencesLoading,
error: preferencesError,
refetch: refreshPreferences,
} = useAtomValue(llmPreferencesAtom);
const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom);
@ -187,19 +186,6 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
<span className="hidden sm:inline">Refresh Configs</span>
<span className="sm:hidden">Configs</span>
</Button>
<Button
variant="outline"
size="sm"
onClick={() => refreshPreferences()}
disabled={isLoading}
className="flex items-center gap-2 text-xs md:text-sm h-8 md:h-9"
>
<RefreshCw
className={`h-3 w-3 md:h-4 md:w-4 ${preferencesLoading ? "animate-spin" : ""}`}
/>
<span className="hidden sm:inline">Refresh Preferences</span>
<span className="sm:hidden">Prefs</span>
</Button>
</div>
</div>

13
surfsense_web/components/sources/types.ts
View file

@ -1,13 +0,0 @@
export interface Connector {
id: string;
title: string;
description: string;
icon: React.ReactNode;
status: "available" | "coming-soon" | "connected";
}
export interface ConnectorCategory {
id: string;
title: string;
connectors: Connector[];
}

41
surfsense_web/components/tool-ui/shared/action-buttons.tsx
View file

@ -1,41 +0,0 @@
"use client";
import type { FC } from "react";
import { Button } from "@/components/ui/button";
import type { Action, ActionsConfig } from "./schema";
interface ActionButtonsProps {
actions?: Action[] | ActionsConfig;
onAction?: (actionId: string) => void;
disabled?: boolean;
}
export const ActionButtons: FC<ActionButtonsProps> = ({ actions, onAction, disabled }) => {
if (!actions) return null;
// Normalize actions to array format
const actionArray: Action[] = Array.isArray(actions)
? actions
: ([
actions.confirm && { ...actions.confirm, id: "confirm" },
actions.cancel && { ...actions.cancel, id: "cancel" },
].filter(Boolean) as Action[]);
if (actionArray.length === 0) return null;
return (
<div className="flex flex-wrap gap-2 pt-3">
{actionArray.map((action) => (
<Button
key={action.id}
variant={action.variant || "default"}
size="sm"
disabled={disabled || action.disabled}
onClick={() => onAction?.(action.id)}
>
{action.label}
</Button>
))}
</div>
);
};

2
surfsense_web/components/tool-ui/shared/index.ts
View file

@ -1,2 +0,0 @@
export * from "./action-buttons";
export * from "./schema";

23
surfsense_web/components/tool-ui/shared/schema.ts
View file

@ -1,23 +0,0 @@
import { z } from "zod";
/**
* Shared action schema for tool UI components
*/
export const ActionSchema = z.object({
id: z.string(),
label: z.string(),
variant: z.enum(["default", "secondary", "destructive", "outline", "ghost", "link"]).optional(),
disabled: z.boolean().optional(),
});
export type Action = z.infer<typeof ActionSchema>;
/**
* Actions configuration schema
*/
export const ActionsConfigSchema = z.object({
confirm: ActionSchema.optional(),
cancel: ActionSchema.optional(),
});
export type ActionsConfig = z.infer<typeof ActionsConfigSchema>;

43
surfsense_web/hooks/use-chat.ts
View file

@ -1,43 +0,0 @@
import { useEffect, useState } from "react";
import type { ResearchMode } from "@/components/chat";
import type { Document } from "@/contracts/types/document.types";
import { getBearerToken } from "@/lib/auth-utils";
interface UseChatStateProps {
search_space_id: string;
chat_id?: string;
}
export function useChatState({ chat_id }: UseChatStateProps) {
const [token, setToken] = useState<string | null>(null);
const [isLoading, setIsLoading] = useState(false);
const [currentChatId, setCurrentChatId] = useState<string | null>(chat_id || null);
// Chat configuration state
const [researchMode, setResearchMode] = useState<ResearchMode>("QNA");
const [selectedConnectors, setSelectedConnectors] = useState<string[]>([]);
const [selectedDocuments, setSelectedDocuments] = useState<Document[]>([]);
const [topK, setTopK] = useState<number>(5);
useEffect(() => {
const bearerToken = getBearerToken();
setToken(bearerToken);
}, []);
return {
token,
setToken,
isLoading,
setIsLoading,
currentChatId,
setCurrentChatId,
researchMode,
setResearchMode,
selectedConnectors,
setSelectedConnectors,
selectedDocuments,
setSelectedDocuments,
topK,
setTopK,
};
}

680
surfsense_web/hooks/use-connector-edit-page.ts
View file

@ -1,680 +0,0 @@
import { zodResolver } from "@hookform/resolvers/zod";
import { useAtomValue } from "jotai";
import { useRouter } from "next/navigation";
import { useCallback, useEffect, useState } from "react";
import { useForm } from "react-hook-form";
import { toast } from "sonner";
import { updateConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms";
import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
import {
type EditConnectorFormValues,
type EditMode,
editConnectorSchema,
type GithubPatFormValues,
type GithubRepo,
githubPatSchema,
} from "@/components/editConnector/types";
import type { EnumConnectorName } from "@/contracts/enums/connector";
import type { UpdateConnectorResponse } from "@/contracts/types/connector.types";
import type { SearchSourceConnector } from "@/hooks/use-search-source-connectors";
import { authenticatedFetch } from "@/lib/auth-utils";
const normalizeListInput = (value: unknown): string[] => {
if (Array.isArray(value)) {
return value.map((item) => String(item).trim()).filter((item) => item.length > 0);
}
if (typeof value === "string") {
return value
.split(",")
.map((item) => item.trim())
.filter((item) => item.length > 0);
}
return [];
};
const arraysEqual = (a: string[], b: string[]): boolean => {
if (a.length !== b.length) return false;
return a.every((value, index) => value === b[index]);
};
const normalizeBoolean = (value: unknown): boolean | null => {
if (typeof value === "boolean") return value;
if (typeof value === "string") {
const lowered = value.trim().toLowerCase();
if (["true", "1", "yes", "on"].includes(lowered)) return true;
if (["false", "0", "no", "off"].includes(lowered)) return false;
}
if (typeof value === "number") {
if (value === 1) return true;
if (value === 0) return false;
}
return null;
};
export function useConnectorEditPage(connectorId: number, searchSpaceId: string) {
const router = useRouter();
const { data: connectors = [], isLoading: connectorsLoading } = useAtomValue(connectorsAtom);
const { mutateAsync: updateConnector } = useAtomValue(updateConnectorMutationAtom);
// State managed by the hook
const [connector, setConnector] = useState<SearchSourceConnector | null>(null);
const [originalConfig, setOriginalConfig] = useState<Record<string, unknown> | null>(null);
const [isSaving, setIsSaving] = useState(false);
const [currentSelectedRepos, setCurrentSelectedRepos] = useState<string[]>([]);
const [originalPat, setOriginalPat] = useState<string>("");
const [editMode, setEditMode] = useState<EditMode>("viewing");
const [fetchedRepos, setFetchedRepos] = useState<GithubRepo[] | null>(null);
const [newSelectedRepos, setNewSelectedRepos] = useState<string[]>([]);
const [isFetchingRepos, setIsFetchingRepos] = useState(false);
// Forms managed by the hook
const patForm = useForm<GithubPatFormValues>({
resolver: zodResolver(githubPatSchema),
defaultValues: { github_pat: "" },
});
const editForm = useForm<EditConnectorFormValues>({
resolver: zodResolver(editConnectorSchema),
defaultValues: {
name: "",
SLACK_BOT_TOKEN: "",
NOTION_INTEGRATION_TOKEN: "",
TAVILY_API_KEY: "",
SEARXNG_HOST: "",
SEARXNG_API_KEY: "",
SEARXNG_ENGINES: "",
SEARXNG_CATEGORIES: "",
SEARXNG_LANGUAGE: "",
SEARXNG_SAFESEARCH: "",
SEARXNG_VERIFY_SSL: "",
DISCORD_BOT_TOKEN: "",
CONFLUENCE_BASE_URL: "",
CONFLUENCE_EMAIL: "",
CONFLUENCE_API_TOKEN: "",
BOOKSTACK_BASE_URL: "",
BOOKSTACK_TOKEN_ID: "",
BOOKSTACK_TOKEN_SECRET: "",
JIRA_BASE_URL: "",
JIRA_EMAIL: "",
JIRA_API_TOKEN: "",
LUMA_API_KEY: "",
ELASTICSEARCH_API_KEY: "",
FIRECRAWL_API_KEY: "",
INITIAL_URLS: "",
},
});
// Effect to load initial data
useEffect(() => {
if (!connectorsLoading && connectors.length > 0 && !connector) {
const currentConnector = connectors.find((c) => c.id === connectorId);
if (currentConnector) {
setConnector(currentConnector);
const config = currentConnector.config || {};
setOriginalConfig(config);
editForm.reset({
name: currentConnector.name,
SLACK_BOT_TOKEN: config.SLACK_BOT_TOKEN || "",
NOTION_INTEGRATION_TOKEN: config.NOTION_INTEGRATION_TOKEN || "",
TAVILY_API_KEY: config.TAVILY_API_KEY || "",
SEARXNG_HOST: config.SEARXNG_HOST || "",
SEARXNG_API_KEY: config.SEARXNG_API_KEY || "",
SEARXNG_ENGINES: Array.isArray(config.SEARXNG_ENGINES)
? config.SEARXNG_ENGINES.join(", ")
: config.SEARXNG_ENGINES || "",
SEARXNG_CATEGORIES: Array.isArray(config.SEARXNG_CATEGORIES)
? config.SEARXNG_CATEGORIES.join(", ")
: config.SEARXNG_CATEGORIES || "",
SEARXNG_LANGUAGE: config.SEARXNG_LANGUAGE || "",
SEARXNG_SAFESEARCH:
config.SEARXNG_SAFESEARCH !== undefined && config.SEARXNG_SAFESEARCH !== null
? String(config.SEARXNG_SAFESEARCH)
: "",
SEARXNG_VERIFY_SSL:
config.SEARXNG_VERIFY_SSL !== undefined && config.SEARXNG_VERIFY_SSL !== null
? String(config.SEARXNG_VERIFY_SSL)
: "",
LINKUP_API_KEY: config.LINKUP_API_KEY || "",
DISCORD_BOT_TOKEN: config.DISCORD_BOT_TOKEN || "",
CONFLUENCE_BASE_URL: config.CONFLUENCE_BASE_URL || "",
CONFLUENCE_EMAIL: config.CONFLUENCE_EMAIL || "",
CONFLUENCE_API_TOKEN: config.CONFLUENCE_API_TOKEN || "",
BOOKSTACK_BASE_URL: config.BOOKSTACK_BASE_URL || "",
BOOKSTACK_TOKEN_ID: config.BOOKSTACK_TOKEN_ID || "",
BOOKSTACK_TOKEN_SECRET: config.BOOKSTACK_TOKEN_SECRET || "",
JIRA_BASE_URL: config.JIRA_BASE_URL || "",
JIRA_EMAIL: config.JIRA_EMAIL || "",
JIRA_API_TOKEN: config.JIRA_API_TOKEN || "",
LUMA_API_KEY: config.LUMA_API_KEY || "",
ELASTICSEARCH_API_KEY: config.ELASTICSEARCH_API_KEY || "",
FIRECRAWL_API_KEY: config.FIRECRAWL_API_KEY || "",
INITIAL_URLS: config.INITIAL_URLS || "",
});
if (currentConnector.connector_type === "GITHUB_CONNECTOR") {
const savedRepos = config.repo_full_names || [];
const savedPat = config.GITHUB_PAT || "";
setCurrentSelectedRepos(savedRepos);
setNewSelectedRepos(savedRepos);
setOriginalPat(savedPat);
patForm.reset({ github_pat: savedPat });
setEditMode("viewing");
}
} else {
toast.error("Connector not found.");
router.push(`/dashboard/${searchSpaceId}`);
}
}
}, [
connectorId,
connectors,
connectorsLoading,
router,
searchSpaceId,
connector,
editForm.reset,
patForm.reset,
// Note: editForm and patForm are intentionally excluded from dependencies
// to prevent infinite loops. They are stable form objects from react-hook-form.
]);
// Handlers managed by the hook
const handleFetchRepositories = useCallback(
async (values: GithubPatFormValues) => {
setIsFetchingRepos(true);
setFetchedRepos(null);
try {
const response = await authenticatedFetch(
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/github/repositories`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ github_pat: values.github_pat }),
}
);
if (!response.ok) {
const err = await response.json();
throw new Error(err.detail || "Fetch failed");
}
const data: GithubRepo[] = await response.json();
setFetchedRepos(data);
setNewSelectedRepos(currentSelectedRepos);
toast.success(`Found ${data.length} repos.`);
} catch (error) {
console.error("Error fetching GitHub repositories:", error);
toast.error(error instanceof Error ? error.message : "Failed to fetch repositories.");
} finally {
setIsFetchingRepos(false);
}
},
[currentSelectedRepos]
); // Added dependency
const handleRepoSelectionChange = useCallback((repoFullName: string, checked: boolean) => {
setNewSelectedRepos((prev) =>
checked ? [...prev, repoFullName] : prev.filter((name) => name !== repoFullName)
);
}, []);
const handleSaveChanges = useCallback(
async (formData: EditConnectorFormValues) => {
if (!connector || !originalConfig) return;
setIsSaving(true);
const updatePayload: Partial<SearchSourceConnector> = {};
let configChanged = false;
let newConfig: Record<string, unknown> | null = null;
if (formData.name !== connector.name) {
updatePayload.name = formData.name;
}
switch (connector.connector_type) {
case "GITHUB_CONNECTOR": {
const currentPatInForm = patForm.getValues("github_pat");
const patChanged = currentPatInForm !== originalPat;
const initialRepoSet = new Set(currentSelectedRepos);
const newRepoSet = new Set(newSelectedRepos);
const reposChanged =
initialRepoSet.size !== newRepoSet.size ||
![...initialRepoSet].every((repo) => newRepoSet.has(repo));
if (
patChanged ||
(editMode === "editing_repos" && reposChanged && fetchedRepos !== null)
) {
if (
!currentPatInForm ||
!(currentPatInForm.startsWith("ghp_") || currentPatInForm.startsWith("github_pat_"))
) {
toast.error("Invalid GitHub PAT format. Cannot save.");
setIsSaving(false);
return;
}
newConfig = {
GITHUB_PAT: currentPatInForm,
repo_full_names: newSelectedRepos,
};
if (reposChanged && newSelectedRepos.length === 0) {
toast.warning("Warning: No repositories selected.");
}
}
break;
}
case "SLACK_CONNECTOR":
if (formData.SLACK_BOT_TOKEN !== originalConfig.SLACK_BOT_TOKEN) {
if (!formData.SLACK_BOT_TOKEN) {
toast.error("Slack Token empty.");
setIsSaving(false);
return;
}
newConfig = { SLACK_BOT_TOKEN: formData.SLACK_BOT_TOKEN };
}
break;
case "NOTION_CONNECTOR":
if (formData.NOTION_INTEGRATION_TOKEN !== originalConfig.NOTION_INTEGRATION_TOKEN) {
if (!formData.NOTION_INTEGRATION_TOKEN) {
toast.error("Notion Token empty.");
setIsSaving(false);
return;
}
newConfig = {
NOTION_INTEGRATION_TOKEN: formData.NOTION_INTEGRATION_TOKEN,
};
}
break;
case "TAVILY_API":
if (formData.TAVILY_API_KEY !== originalConfig.TAVILY_API_KEY) {
if (!formData.TAVILY_API_KEY) {
toast.error("Tavily Key empty.");
setIsSaving(false);
return;
}
newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY };
}
break;
case "SEARXNG_API": {
const host = (formData.SEARXNG_HOST || "").trim();
if (!host) {
toast.error("SearxNG host is required.");
setIsSaving(false);
return;
}
const candidateConfig: Record<string, unknown> = { SEARXNG_HOST: host };
const originalHost =
typeof originalConfig.SEARXNG_HOST === "string" ? originalConfig.SEARXNG_HOST : "";
let hasChanges = host !== originalHost.trim();
const apiKey = (formData.SEARXNG_API_KEY || "").trim();
const originalApiKey =
typeof originalConfig.SEARXNG_API_KEY === "string"
? originalConfig.SEARXNG_API_KEY
: "";
const originalApiKeyTrimmed = originalApiKey.trim();
if (apiKey !== originalApiKeyTrimmed) {
candidateConfig.SEARXNG_API_KEY = apiKey || null;
hasChanges = true;
}
const newEngines = normalizeListInput(formData.SEARXNG_ENGINES || "");
const originalEngines = normalizeListInput(originalConfig.SEARXNG_ENGINES);
if (!arraysEqual(newEngines, originalEngines)) {
candidateConfig.SEARXNG_ENGINES = newEngines;
hasChanges = true;
}
const newCategories = normalizeListInput(formData.SEARXNG_CATEGORIES || "");
const originalCategories = normalizeListInput(originalConfig.SEARXNG_CATEGORIES);
if (!arraysEqual(newCategories, originalCategories)) {
candidateConfig.SEARXNG_CATEGORIES = newCategories;
hasChanges = true;
}
const language = (formData.SEARXNG_LANGUAGE || "").trim();
const originalLanguage =
typeof originalConfig.SEARXNG_LANGUAGE === "string"
? originalConfig.SEARXNG_LANGUAGE
: "";
const originalLanguageTrimmed = originalLanguage.trim();
if (language !== originalLanguageTrimmed) {
candidateConfig.SEARXNG_LANGUAGE = language || null;
hasChanges = true;
}
const safesearchRaw = (formData.SEARXNG_SAFESEARCH || "").trim();
const originalSafesearch = originalConfig.SEARXNG_SAFESEARCH;
if (safesearchRaw) {
const parsed = Number(safesearchRaw);
if (Number.isNaN(parsed) || !Number.isInteger(parsed) || parsed < 0 || parsed > 2) {
toast.error("SearxNG SafeSearch must be 0, 1, or 2.");
setIsSaving(false);
return;
}
if (parsed !== Number(originalSafesearch)) {
candidateConfig.SEARXNG_SAFESEARCH = parsed;
hasChanges = true;
}
} else if (originalSafesearch !== undefined && originalSafesearch !== null) {
candidateConfig.SEARXNG_SAFESEARCH = null;
hasChanges = true;
}
const verifyRaw = (formData.SEARXNG_VERIFY_SSL || "").trim().toLowerCase();
const originalVerifyBool = normalizeBoolean(originalConfig.SEARXNG_VERIFY_SSL);
if (verifyRaw) {
let parsedBool: boolean | null = null;
if (["true", "1", "yes", "on"].includes(verifyRaw)) parsedBool = true;
else if (["false", "0", "no", "off"].includes(verifyRaw)) parsedBool = false;
if (parsedBool === null) {
toast.error("SearxNG SSL verification must be true or false.");
setIsSaving(false);
return;
}
if (parsedBool !== originalVerifyBool) {
candidateConfig.SEARXNG_VERIFY_SSL = parsedBool;
hasChanges = true;
}
} else if (originalVerifyBool !== null) {
candidateConfig.SEARXNG_VERIFY_SSL = null;
hasChanges = true;
}
if (hasChanges) {
newConfig = candidateConfig;
}
break;
}
case "LINKUP_API":
if (formData.LINKUP_API_KEY !== originalConfig.LINKUP_API_KEY) {
if (!formData.LINKUP_API_KEY) {
toast.error("Linkup API Key cannot be empty.");
setIsSaving(false);
return;
}
newConfig = { LINKUP_API_KEY: formData.LINKUP_API_KEY };
}
break;
case "DISCORD_CONNECTOR":
if (formData.DISCORD_BOT_TOKEN !== originalConfig.DISCORD_BOT_TOKEN) {
if (!formData.DISCORD_BOT_TOKEN) {
toast.error("Discord Bot Token cannot be empty.");
setIsSaving(false);
return;
}
newConfig = { DISCORD_BOT_TOKEN: formData.DISCORD_BOT_TOKEN };
}
break;
case "CONFLUENCE_CONNECTOR":
if (
formData.CONFLUENCE_BASE_URL !== originalConfig.CONFLUENCE_BASE_URL ||
formData.CONFLUENCE_EMAIL !== originalConfig.CONFLUENCE_EMAIL ||
formData.CONFLUENCE_API_TOKEN !== originalConfig.CONFLUENCE_API_TOKEN
) {
if (
!formData.CONFLUENCE_BASE_URL ||
!formData.CONFLUENCE_EMAIL ||
!formData.CONFLUENCE_API_TOKEN
) {
toast.error("All Confluence fields are required.");
setIsSaving(false);
return;
}
newConfig = {
CONFLUENCE_BASE_URL: formData.CONFLUENCE_BASE_URL,
CONFLUENCE_EMAIL: formData.CONFLUENCE_EMAIL,
CONFLUENCE_API_TOKEN: formData.CONFLUENCE_API_TOKEN,
};
}
break;
case "BOOKSTACK_CONNECTOR":
if (
formData.BOOKSTACK_BASE_URL !== originalConfig.BOOKSTACK_BASE_URL ||
formData.BOOKSTACK_TOKEN_ID !== originalConfig.BOOKSTACK_TOKEN_ID ||
formData.BOOKSTACK_TOKEN_SECRET !== originalConfig.BOOKSTACK_TOKEN_SECRET
) {
if (
!formData.BOOKSTACK_BASE_URL ||
!formData.BOOKSTACK_TOKEN_ID ||
!formData.BOOKSTACK_TOKEN_SECRET
) {
toast.error("All BookStack fields are required.");
setIsSaving(false);
return;
}
newConfig = {
BOOKSTACK_BASE_URL: formData.BOOKSTACK_BASE_URL,
BOOKSTACK_TOKEN_ID: formData.BOOKSTACK_TOKEN_ID,
BOOKSTACK_TOKEN_SECRET: formData.BOOKSTACK_TOKEN_SECRET,
};
}
break;
case "JIRA_CONNECTOR": {
// Check if this is an OAuth connector (has access_token or _token_encrypted flag)
const isJiraOAuth = !!(originalConfig.access_token || originalConfig._token_encrypted);
if (isJiraOAuth) {
// OAuth connectors don't allow editing credentials through the form
// Only allow name changes, which are handled separately
break;
}
// Legacy API token connector - allow editing credentials
if (
formData.JIRA_BASE_URL !== originalConfig.JIRA_BASE_URL ||
formData.JIRA_EMAIL !== originalConfig.JIRA_EMAIL ||
formData.JIRA_API_TOKEN !== originalConfig.JIRA_API_TOKEN
) {
if (!formData.JIRA_BASE_URL || !formData.JIRA_EMAIL || !formData.JIRA_API_TOKEN) {
toast.error("All Jira fields are required.");
setIsSaving(false);
return;
}
newConfig = {
JIRA_BASE_URL: formData.JIRA_BASE_URL,
JIRA_EMAIL: formData.JIRA_EMAIL,
JIRA_API_TOKEN: formData.JIRA_API_TOKEN,
};
}
break;
}
case "LUMA_CONNECTOR":
if (formData.LUMA_API_KEY !== originalConfig.LUMA_API_KEY) {
if (!formData.LUMA_API_KEY) {
toast.error("Luma API Key cannot be empty.");
setIsSaving(false);
return;
}
newConfig = { LUMA_API_KEY: formData.LUMA_API_KEY };
}
break;
case "ELASTICSEARCH_CONNECTOR":
if (formData.ELASTICSEARCH_API_KEY !== originalConfig.ELASTICSEARCH_API_KEY) {
if (!formData.ELASTICSEARCH_API_KEY) {
toast.error("Elasticsearch API Key cannot be empty.");
setIsSaving(false);
return;
}
newConfig = { ELASTICSEARCH_API_KEY: formData.ELASTICSEARCH_API_KEY };
}
break;
case "WEBCRAWLER_CONNECTOR":
if (
formData.FIRECRAWL_API_KEY !== originalConfig.FIRECRAWL_API_KEY ||
formData.INITIAL_URLS !== originalConfig.INITIAL_URLS
) {
newConfig = {};
if (formData.FIRECRAWL_API_KEY?.trim()) {
if (!formData.FIRECRAWL_API_KEY.startsWith("fc-")) {
toast.warning(
"Firecrawl API keys typically start with 'fc-'. Please verify your key."
);
}
newConfig.FIRECRAWL_API_KEY = formData.FIRECRAWL_API_KEY.trim();
} else if (originalConfig.FIRECRAWL_API_KEY) {
toast.info(
"Firecrawl API key removed. Web crawler will use AsyncChromiumLoader as fallback."
);
}
if (formData.INITIAL_URLS !== undefined) {
if (formData.INITIAL_URLS?.trim()) {
newConfig.INITIAL_URLS = formData.INITIAL_URLS.trim();
} else if (originalConfig.INITIAL_URLS) {
toast.info("URLs removed from crawler configuration.");
}
}
}
break;
}
if (newConfig !== null) {
updatePayload.config = newConfig;
configChanged = true;
}
if (Object.keys(updatePayload).length === 0) {
toast.info("No changes detected.");
setIsSaving(false);
if (connector.connector_type === "GITHUB_CONNECTOR") {
setEditMode("viewing");
patForm.reset({ github_pat: originalPat });
}
return;
}
try {
const updatedConnector = (await updateConnector({
id: connectorId,
data: {
...updatePayload,
connector_type: connector.connector_type as EnumConnectorName,
},
})) as UpdateConnectorResponse;
toast.success("Connector updated!");
// Use the response from the API which has the full merged config
const newlySavedConfig = updatedConnector.config || originalConfig;
setOriginalConfig(newlySavedConfig);
// Update connector state with the full updated connector from the API
setConnector(updatedConnector);
if (configChanged) {
if (connector.connector_type === "GITHUB_CONNECTOR") {
const savedGitHubConfig = newlySavedConfig as {
GITHUB_PAT?: string;
repo_full_names?: string[];
};
setCurrentSelectedRepos(savedGitHubConfig.repo_full_names || []);
setOriginalPat(savedGitHubConfig.GITHUB_PAT || "");
setNewSelectedRepos(savedGitHubConfig.repo_full_names || []);
patForm.reset({ github_pat: savedGitHubConfig.GITHUB_PAT || "" });
} else if (connector.connector_type === "SLACK_CONNECTOR") {
editForm.setValue("SLACK_BOT_TOKEN", newlySavedConfig.SLACK_BOT_TOKEN || "");
} else if (connector.connector_type === "NOTION_CONNECTOR") {
editForm.setValue(
"NOTION_INTEGRATION_TOKEN",
newlySavedConfig.NOTION_INTEGRATION_TOKEN || ""
);
} else if (connector.connector_type === "TAVILY_API") {
editForm.setValue("TAVILY_API_KEY", newlySavedConfig.TAVILY_API_KEY || "");
} else if (connector.connector_type === "SEARXNG_API") {
editForm.setValue("SEARXNG_HOST", newlySavedConfig.SEARXNG_HOST || "");
editForm.setValue("SEARXNG_API_KEY", newlySavedConfig.SEARXNG_API_KEY || "");
editForm.setValue(
"SEARXNG_ENGINES",
normalizeListInput(newlySavedConfig.SEARXNG_ENGINES).join(", ")
);
editForm.setValue(
"SEARXNG_CATEGORIES",
normalizeListInput(newlySavedConfig.SEARXNG_CATEGORIES).join(", ")
);
editForm.setValue("SEARXNG_LANGUAGE", newlySavedConfig.SEARXNG_LANGUAGE || "");
editForm.setValue(
"SEARXNG_SAFESEARCH",
newlySavedConfig.SEARXNG_SAFESEARCH === null ||
newlySavedConfig.SEARXNG_SAFESEARCH === undefined
? ""
: String(newlySavedConfig.SEARXNG_SAFESEARCH)
);
const verifyValue = normalizeBoolean(newlySavedConfig.SEARXNG_VERIFY_SSL);
editForm.setValue(
"SEARXNG_VERIFY_SSL",
verifyValue === null ? "" : String(verifyValue)
);
} else if (connector.connector_type === "LINKUP_API") {
editForm.setValue("LINKUP_API_KEY", newlySavedConfig.LINKUP_API_KEY || "");
} else if (connector.connector_type === "DISCORD_CONNECTOR") {
editForm.setValue("DISCORD_BOT_TOKEN", newlySavedConfig.DISCORD_BOT_TOKEN || "");
} else if (connector.connector_type === "CONFLUENCE_CONNECTOR") {
editForm.setValue("CONFLUENCE_BASE_URL", newlySavedConfig.CONFLUENCE_BASE_URL || "");
editForm.setValue("CONFLUENCE_EMAIL", newlySavedConfig.CONFLUENCE_EMAIL || "");
editForm.setValue("CONFLUENCE_API_TOKEN", newlySavedConfig.CONFLUENCE_API_TOKEN || "");
} else if (connector.connector_type === "BOOKSTACK_CONNECTOR") {
editForm.setValue("BOOKSTACK_BASE_URL", newlySavedConfig.BOOKSTACK_BASE_URL || "");
editForm.setValue("BOOKSTACK_TOKEN_ID", newlySavedConfig.BOOKSTACK_TOKEN_ID || "");
editForm.setValue(
"BOOKSTACK_TOKEN_SECRET",
newlySavedConfig.BOOKSTACK_TOKEN_SECRET || ""
);
} else if (connector.connector_type === "JIRA_CONNECTOR") {
editForm.setValue("JIRA_BASE_URL", newlySavedConfig.JIRA_BASE_URL || "");
editForm.setValue("JIRA_EMAIL", newlySavedConfig.JIRA_EMAIL || "");
editForm.setValue("JIRA_API_TOKEN", newlySavedConfig.JIRA_API_TOKEN || "");
} else if (connector.connector_type === "LUMA_CONNECTOR") {
editForm.setValue("LUMA_API_KEY", newlySavedConfig.LUMA_API_KEY || "");
} else if (connector.connector_type === "ELASTICSEARCH_CONNECTOR") {
editForm.setValue(
"ELASTICSEARCH_API_KEY",
newlySavedConfig.ELASTICSEARCH_API_KEY || ""
);
} else if (connector.connector_type === "WEBCRAWLER_CONNECTOR") {
editForm.setValue("FIRECRAWL_API_KEY", newlySavedConfig.FIRECRAWL_API_KEY || "");
editForm.setValue("INITIAL_URLS", newlySavedConfig.INITIAL_URLS || "");
}
}
if (connector.connector_type === "GITHUB_CONNECTOR") {
setEditMode("viewing");
setFetchedRepos(null);
}
// Resetting simple form values is handled by useEffect if connector state updates
} catch (error) {
console.error("Error updating connector:", error);
toast.error(error instanceof Error ? error.message : "Failed to update connector.");
} finally {
setIsSaving(false);
}
},
[
connector,
originalConfig,
updateConnector,
connectorId,
patForm,
originalPat,
currentSelectedRepos,
newSelectedRepos,
editMode,
fetchedRepos,
editForm,
]
); // Added editForm to dependencies
// Return values needed by the component
return {
connectorsLoading,
connector,
isSaving,
editForm,
patForm,
handleSaveChanges,
// GitHub specific props
editMode,
setEditMode,
originalPat,
currentSelectedRepos,
fetchedRepos,
setFetchedRepos,
newSelectedRepos,
setNewSelectedRepos,
isFetchingRepos,
handleFetchRepositories,
handleRepoSelectionChange,
};
}

41
surfsense_web/lib/auth-utils.ts
View file

@ -130,44 +130,3 @@ export async function authenticatedFetch(
return response;
}
/**
* Type for the result of a fetch operation with built-in error handling
*/
export type FetchResult<T> =
| { success: true; data: T; response: Response }
| { success: false; error: string; status?: number };
/**
* Authenticated fetch with JSON response handling
* Returns a result object instead of throwing on non-401 errors
*/
export async function authenticatedFetchJson<T = unknown>(
url: string,
options?: RequestInit & { skipAuthRedirect?: boolean }
): Promise<FetchResult<T>> {
try {
const response = await authenticatedFetch(url, options);
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
return {
success: false,
error: errorData.detail || `Request failed: ${response.status}`,
status: response.status,
};
}
const data = await response.json();
return { success: true, data, response };
} catch (err: any) {
// Re-throw if it's the unauthorized redirect
if (err.message?.includes("Unauthorized")) {
throw err;
}
return {
success: false,
error: err.message || "Request failed",
};
}
}

7
surfsense_web/lib/utils.ts
View file

@ -1,4 +1,3 @@
import type { Message } from "@ai-sdk/react";
import { type ClassValue, clsx } from "clsx";
import { twMerge } from "tailwind-merge";
@ -6,12 +5,6 @@ export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs));
}
export function getChatTitleFromMessages(messages: Message[]) {
const userMessages = messages.filter((msg) => msg.role === "user");
if (userMessages.length === 0) return "Untitled Chat";
return userMessages[0].content;
}
export const formatDate = (date: Date): string => {
return date.toLocaleDateString("en-US", {
year: "numeric",

2
surfsense_web/messages/en.json
View file

@ -265,7 +265,7 @@
"no_documents": "No documents found",
"type": "Type",
"content_summary": "Content Summary",
"view_full": "View Full Content",
"view_full": "View Summary",
"filter_placeholder": "Filter by title...",
"rows_per_page": "Rows per page",
"refresh": "Refresh",