mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat: docs and ui tweaks
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
64f2b4a6eb
commit
271a21aee6
103 changed files with 2161 additions and 2625 deletions
|
|
@ -14,6 +14,7 @@ import {
|
|||
modelRolesAtom,
|
||||
} from "@/atoms/model-connections/model-connections-query.atoms";
|
||||
import { activeWorkspaceIdAtom } from "@/atoms/workspaces/workspace-query.atoms";
|
||||
import { ConnectorIndicator } from "@/components/assistant-ui/connector-popup";
|
||||
import { DocumentUploadDialogProvider } from "@/components/assistant-ui/document-upload-popup";
|
||||
import { LayoutDataProvider } from "@/components/layout";
|
||||
import { OnboardingTour } from "@/components/onboarding-tour";
|
||||
|
|
@ -214,7 +215,10 @@ export function DashboardClientLayout({
|
|||
return (
|
||||
<DocumentUploadDialogProvider>
|
||||
<OnboardingTour />
|
||||
<LayoutDataProvider workspaceId={workspaceId}>{children}</LayoutDataProvider>
|
||||
<LayoutDataProvider workspaceId={workspaceId}>
|
||||
{children}
|
||||
<ConnectorIndicator showTrigger={false} />
|
||||
</LayoutDataProvider>
|
||||
</DocumentUploadDialogProvider>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,11 +2,22 @@
|
|||
|
||||
import { ArrowRight, History, KeyRound } from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { useMemo } from "react";
|
||||
import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities";
|
||||
import { PLAYGROUND_PLATFORMS } from "@/lib/playground/catalog";
|
||||
import { formatPricing } from "@/lib/playground/format";
|
||||
|
||||
export function PlaygroundIndex({ workspaceId }: { workspaceId: number }) {
|
||||
const base = `/dashboard/${workspaceId}/playground`;
|
||||
|
||||
// The grid renders from the static catalog immediately; pricing fills in
|
||||
// once the capabilities fetch lands (blank while loading, never blocking).
|
||||
const { data: capabilities } = useScraperCapabilities(workspaceId);
|
||||
const pricingByName = useMemo(
|
||||
() => new Map(capabilities?.map((c) => [c.name, formatPricing(c.pricing)])),
|
||||
[capabilities]
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
<div>
|
||||
|
|
@ -67,6 +78,11 @@ export function PlaygroundIndex({ workspaceId }: { workspaceId: number }) {
|
|||
<ArrowRight className="h-4 w-4 text-muted-foreground opacity-0 transition-opacity group-hover:opacity-100" />
|
||||
</div>
|
||||
<code className="mt-2 text-xs text-muted-foreground">{verb.name}</code>
|
||||
{pricingByName.has(verb.name) ? (
|
||||
<span className="mt-2 text-xs tabular-nums text-muted-foreground">
|
||||
{pricingByName.get(verb.name)}
|
||||
</span>
|
||||
) : null}
|
||||
</Link>
|
||||
))}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities";
|
|||
import { scrapersApiService } from "@/lib/apis/scrapers-api.service";
|
||||
import { AbortedError, AppError } from "@/lib/error";
|
||||
import { findVerb } from "@/lib/playground/catalog";
|
||||
import { formatCost, formatDuration } from "@/lib/playground/format";
|
||||
import { formatCost, formatDuration, formatPricing } from "@/lib/playground/format";
|
||||
import { buildPayload, initialFormValues, parseSchemaFields } from "@/lib/playground/json-schema";
|
||||
import { ApiReference } from "./api-reference";
|
||||
import { OutputViewer } from "./output-viewer";
|
||||
|
|
@ -198,6 +198,12 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn
|
|||
<code className="mt-2 inline-block rounded bg-muted/40 px-1.5 py-0.5 text-xs text-muted-foreground">
|
||||
POST /workspaces/{workspaceId}/scrapers/{platform}/{verb}
|
||||
</code>
|
||||
<div className="mt-2 flex items-center gap-1.5 text-xs text-muted-foreground">
|
||||
<Coins className="h-3.5 w-3.5" />
|
||||
<span className="font-medium tabular-nums text-foreground">
|
||||
{formatPricing(capability.pricing)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<SchemaForm
|
||||
|
|
|
|||
|
|
@ -2,3 +2,21 @@ import { atom } from "jotai";
|
|||
|
||||
// Atom to control the connector dialog open state from anywhere in the app
|
||||
export const connectorDialogOpenAtom = atom(false);
|
||||
|
||||
/**
|
||||
* Requests the connector dialog to start an import flow for a specific
|
||||
* connector type (Google Drive / Composio Drive / OneDrive / Dropbox), set by
|
||||
* the Documents sidebar "Import" menu. `useConnectorDialog` consumes and clears
|
||||
* it.
|
||||
*
|
||||
* - `mode: "auto"` routes by connected-account count: none -> OAuth connect,
|
||||
* one -> edit view, many -> accounts list. Used by "Connect" and "Manage".
|
||||
* - `mode: "connect"` always starts a fresh OAuth connect, even when an account
|
||||
* already exists. Used by "Add another account".
|
||||
*/
|
||||
export interface ImportConnectorRequest {
|
||||
connectorType: string;
|
||||
mode: "auto" | "connect";
|
||||
}
|
||||
|
||||
export const importConnectorRequestAtom = atom<ImportConnectorRequest | null>(null);
|
||||
|
|
|
|||
|
|
@ -189,7 +189,7 @@ export const ConnectorIndicator = forwardRef<ConnectorIndicatorHandle, Connector
|
|||
}}
|
||||
className="max-w-3xl w-[95vw] sm:w-full h-[75vh] sm:h-[85vh] flex flex-col p-0 gap-0 overflow-hidden ring-0 dark:ring-0 [&>button]:right-4 sm:[&>button]:right-12 [&>button]:top-6 sm:[&>button]:top-10 [&>button]:opacity-80 [&>button]:hover:opacity-100 [&>button]:hover:bg-accent [&>button]:hover:text-accent-foreground [&>button>svg]:size-5 select-none"
|
||||
>
|
||||
<DialogTitle className="sr-only">Manage Connectors</DialogTitle>
|
||||
<DialogTitle className="sr-only">Manage External MCP Connectors</DialogTitle>
|
||||
{/* YouTube Crawler View - shown when adding YouTube videos */}
|
||||
{isYouTubeView && workspaceId ? (
|
||||
<YouTubeCrawlerView workspaceId={workspaceId} onBack={handleBackFromYouTube} />
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ export const ConnectorDialogHeader: FC<ConnectorDialogHeaderProps> = ({
|
|||
>
|
||||
<DialogHeader>
|
||||
<DialogTitle className="text-xl sm:text-3xl font-semibold tracking-tight">
|
||||
Manage Connectors
|
||||
Manage External MCP Connectors
|
||||
</DialogTitle>
|
||||
<DialogDescription className="text-xs sm:text-base text-muted-foreground/80 mt-1 sm:mt-1.5">
|
||||
Connect Surfsense to your favorite tools and services.
|
||||
|
|
|
|||
|
|
@ -293,7 +293,7 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
|
|||
{/* Documentation Link */}
|
||||
<div>
|
||||
<Link
|
||||
href="/docs/connectors/github"
|
||||
href="/docs/connectors/external/github"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-xs sm:text-sm font-medium underline underline-offset-4 hover:text-primary transition-colors inline-flex items-center gap-1.5"
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
|||
import { connectorsApiService, type ObsidianStats } from "@/lib/apis/connectors-api.service";
|
||||
import type { ConnectorConfigProps } from "../index";
|
||||
|
||||
const OBSIDIAN_SETUP_DOCS_URL = "/docs/connectors/obsidian";
|
||||
const OBSIDIAN_SETUP_DOCS_URL = "/docs/connectors/external/obsidian";
|
||||
|
||||
function formatTimestamp(value: unknown): string {
|
||||
if (typeof value !== "string" || !value) return "—";
|
||||
|
|
|
|||
|
|
@ -18,6 +18,23 @@ export const DEPRECATED_CONNECTOR_TYPES = new Set<string>([
|
|||
EnumConnectorName.SEARXNG_API,
|
||||
EnumConnectorName.LINKUP_API,
|
||||
EnumConnectorName.BAIDU_SEARCH_API,
|
||||
// Legacy content crawlers/search retired in favor of the file Import menu and
|
||||
// hosted MCP tooling. Existing rows stay manageable; new connections refused.
|
||||
EnumConnectorName.YOUTUBE_CONNECTOR,
|
||||
EnumConnectorName.WEBCRAWLER_CONNECTOR,
|
||||
EnumConnectorName.ELASTICSEARCH_CONNECTOR,
|
||||
]);
|
||||
|
||||
/**
|
||||
* File-import connectors surfaced through the Documents sidebar "Import" menu
|
||||
* instead of the external-MCP connector catalog. They index remote files into
|
||||
* the knowledge base, so they belong with uploads, not with live MCP tools.
|
||||
*/
|
||||
export const IMPORT_CONNECTOR_TYPES = new Set<string>([
|
||||
EnumConnectorName.GOOGLE_DRIVE_CONNECTOR,
|
||||
EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR,
|
||||
EnumConnectorName.ONEDRIVE_CONNECTOR,
|
||||
EnumConnectorName.DROPBOX_CONNECTOR,
|
||||
]);
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -2,7 +2,10 @@ import { format } from "date-fns";
|
|||
import { useAtom, useAtomValue } from "jotai";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { connectorDialogOpenAtom } from "@/atoms/connector-dialog/connector-dialog.atoms";
|
||||
import {
|
||||
connectorDialogOpenAtom,
|
||||
importConnectorRequestAtom,
|
||||
} from "@/atoms/connector-dialog/connector-dialog.atoms";
|
||||
import {
|
||||
createConnectorMutationAtom,
|
||||
deleteConnectorMutationAtom,
|
||||
|
|
@ -67,6 +70,8 @@ export const useConnectorDialog = () => {
|
|||
|
||||
// Use global atom for dialog open state so it can be controlled from anywhere
|
||||
const [isOpen, setIsOpen] = useAtom(connectorDialogOpenAtom);
|
||||
// Import-flow request from the Documents sidebar "Import" menu
|
||||
const [importConnectorRequest, setImportConnectorRequest] = useAtom(importConnectorRequestAtom);
|
||||
const [activeTab, setActiveTab] = useState("all");
|
||||
const [connectingId, setConnectingId] = useState<string | null>(null);
|
||||
const [isScrolled, setIsScrolled] = useState(false);
|
||||
|
|
@ -1372,6 +1377,47 @@ export const useConnectorDialog = () => {
|
|||
[isStartingIndexing, isDisconnecting, isSaving, isCreatingConnector, setIsOpen]
|
||||
);
|
||||
|
||||
// Consume an import request from the Documents sidebar "Import" menu.
|
||||
// mode "connect" -> always OAuth connect (add account). mode "auto" routes by
|
||||
// account count: none -> connect, one -> edit view, many -> accounts list.
|
||||
useEffect(() => {
|
||||
if (!importConnectorRequest || !workspaceId) return;
|
||||
|
||||
const { connectorType, mode } = importConnectorRequest;
|
||||
setImportConnectorRequest(null);
|
||||
|
||||
const connectorDef =
|
||||
OAUTH_CONNECTORS.find((c) => c.connectorType === connectorType) ||
|
||||
COMPOSIO_CONNECTORS.find((c) => c.connectorType === connectorType);
|
||||
|
||||
const existing = (allConnectors || []).filter(
|
||||
(c: SearchSourceConnector) => c.connector_type === connectorType
|
||||
);
|
||||
|
||||
if (mode === "connect" || existing.length === 0) {
|
||||
if (connectorDef) {
|
||||
handleConnectOAuth(connectorDef);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
setIsOpen(true);
|
||||
if (existing.length === 1) {
|
||||
handleStartEdit(existing[0]);
|
||||
} else {
|
||||
handleViewAccountsList(connectorType, connectorDef?.title);
|
||||
}
|
||||
}, [
|
||||
importConnectorRequest,
|
||||
workspaceId,
|
||||
allConnectors,
|
||||
setImportConnectorRequest,
|
||||
handleConnectOAuth,
|
||||
handleStartEdit,
|
||||
handleViewAccountsList,
|
||||
setIsOpen,
|
||||
]);
|
||||
|
||||
const handleTabChange = useCallback((value: string) => {
|
||||
setActiveTab(value);
|
||||
}, []);
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels";
|
|||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
COMPOSIO_CONNECTORS,
|
||||
IMPORT_CONNECTOR_TYPES,
|
||||
LIVE_CONNECTOR_TYPES,
|
||||
OAUTH_CONNECTORS,
|
||||
} from "../constants/connector-constants";
|
||||
|
|
@ -33,12 +34,17 @@ export const ActiveConnectorsTab: FC<ActiveConnectorsTabProps> = ({
|
|||
searchQuery,
|
||||
hasSources,
|
||||
activeDocumentTypes,
|
||||
connectors,
|
||||
connectors: allActiveConnectors,
|
||||
indexingConnectorIds,
|
||||
onTabChange: _onTabChange,
|
||||
onManage,
|
||||
onViewAccountsList,
|
||||
}) => {
|
||||
// Import connectors (Drive/OneDrive/Dropbox) are managed via the Documents
|
||||
// sidebar "Import" menu, so they are excluded from the MCP connector list.
|
||||
const connectors = allActiveConnectors.filter(
|
||||
(c) => !IMPORT_CONNECTOR_TYPES.has(c.connector_type)
|
||||
);
|
||||
// Convert activeDocumentTypes array to Record for utility function
|
||||
const documentTypeCounts = activeDocumentTypes.reduce(
|
||||
(acc, [docType, count]) => {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
CRAWLERS,
|
||||
DEPRECATED_CONNECTOR_TYPES,
|
||||
getConnectorCategory,
|
||||
IMPORT_CONNECTOR_TYPES,
|
||||
OAUTH_CONNECTORS,
|
||||
OTHER_CONNECTORS,
|
||||
} from "../constants/connector-constants";
|
||||
|
|
@ -82,30 +83,41 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
|||
const passesDeploymentFilter = (c: DeploymentFilterableConnector) =>
|
||||
(!c.selfHostedOnly || selfHosted) && (!c.desktopOnly || isDesktop);
|
||||
|
||||
// Filter connectors based on search and deployment mode
|
||||
// Import connectors (Drive/OneDrive/Dropbox) are managed via the Documents
|
||||
// sidebar "Import" menu, so they never appear in the MCP connector catalog.
|
||||
const notImport = (c: { connectorType?: string }) =>
|
||||
!c.connectorType || !IMPORT_CONNECTOR_TYPES.has(c.connectorType);
|
||||
|
||||
// Filter connectors based on search, deployment mode, and import exclusion
|
||||
const filteredOAuth = OAUTH_CONNECTORS.filter(
|
||||
(c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c)
|
||||
(c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c) && notImport(c)
|
||||
);
|
||||
|
||||
const filteredCrawlers = CRAWLERS.filter(
|
||||
(c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c)
|
||||
(c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c) && notImport(c)
|
||||
);
|
||||
|
||||
const filteredOther = OTHER_CONNECTORS.filter(
|
||||
(c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c)
|
||||
(c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c) && notImport(c)
|
||||
);
|
||||
|
||||
// Filter Composio connectors
|
||||
const filteredComposio = COMPOSIO_CONNECTORS.filter(
|
||||
(c) =>
|
||||
c.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
|
||||
c.description.toLowerCase().includes(searchQuery.toLowerCase())
|
||||
(c.title.toLowerCase().includes(searchQuery.toLowerCase()) ||
|
||||
c.description.toLowerCase().includes(searchQuery.toLowerCase())) &&
|
||||
notImport(c)
|
||||
);
|
||||
|
||||
const isDeprecated = (c: { connectorType?: string }) =>
|
||||
!!c.connectorType && DEPRECATED_CONNECTOR_TYPES.has(c.connectorType);
|
||||
|
||||
const inCategory =
|
||||
(category: ConnectorCategory) =>
|
||||
<T extends { connectorType?: string }>(connector: T): boolean =>
|
||||
!!connector.connectorType && getConnectorCategory(connector.connectorType) === category;
|
||||
!isDeprecated(connector) &&
|
||||
!!connector.connectorType &&
|
||||
getConnectorCategory(connector.connectorType) === category;
|
||||
|
||||
const knowledgeBase = {
|
||||
oauth: filteredOAuth.filter(inCategory("knowledge_base")),
|
||||
|
|
@ -120,6 +132,16 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
|||
crawlers: filteredCrawlers.filter(inCategory("tools_live")),
|
||||
};
|
||||
|
||||
// Deprecated cards render in a single section at the very bottom. Discord and
|
||||
// Teams come from OAUTH_CONNECTORS; Luma, Tavily, Linkup, Baidu, Elasticsearch
|
||||
// from OTHER_CONNECTORS (SearXNG has no catalog card); YouTube and Web Pages
|
||||
// from CRAWLERS.
|
||||
const deprecated = {
|
||||
oauth: filteredOAuth.filter(isDeprecated),
|
||||
other: filteredOther.filter(isDeprecated),
|
||||
crawlers: filteredCrawlers.filter(isDeprecated),
|
||||
};
|
||||
|
||||
const renderOAuthCard = (connector: OAuthConnector | ComposioConnector) => {
|
||||
const isConnected = connectedTypes.has(connector.connectorType);
|
||||
const isConnecting = connectingId === connector.id;
|
||||
|
|
@ -245,6 +267,9 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
|||
isConnecting={isConnecting}
|
||||
documentCount={documentCount}
|
||||
isIndexing={isIndexing}
|
||||
deprecated={
|
||||
!!crawler.connectorType && DEPRECATED_CONNECTOR_TYPES.has(crawler.connectorType)
|
||||
}
|
||||
onConnect={handleConnect}
|
||||
onManage={actualConnector && onManage ? () => onManage(actualConnector) : undefined}
|
||||
/>
|
||||
|
|
@ -261,8 +286,10 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
|||
toolsLive.composio.length > 0 ||
|
||||
toolsLive.other.length > 0 ||
|
||||
toolsLive.crawlers.length > 0;
|
||||
const hasDeprecated =
|
||||
deprecated.oauth.length > 0 || deprecated.other.length > 0 || deprecated.crawlers.length > 0;
|
||||
|
||||
const hasAnyResults = hasKnowledgeBase || hasToolsLive;
|
||||
const hasAnyResults = hasKnowledgeBase || hasToolsLive || hasDeprecated;
|
||||
|
||||
if (!hasAnyResults && searchQuery) {
|
||||
return (
|
||||
|
|
@ -307,6 +334,19 @@ export const AllConnectorsTab: FC<AllConnectorsTabProps> = ({
|
|||
</div>
|
||||
</section>
|
||||
)}
|
||||
|
||||
{hasDeprecated && (
|
||||
<section>
|
||||
<div className="flex items-center gap-2 mb-4">
|
||||
<h3 className="text-sm font-semibold text-muted-foreground">Deprecated</h3>
|
||||
</div>
|
||||
<div className="grid grid-cols-1 sm:grid-cols-2 gap-3">
|
||||
{deprecated.oauth.map(renderOAuthCard)}
|
||||
{deprecated.crawlers.map(renderCrawlerCard)}
|
||||
{deprecated.other.map(renderOtherCard)}
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import {
|
|||
Wrench,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { AnimatePresence, motion } from "motion/react";
|
||||
import { AnimatePresence, motion, useReducedMotion } from "motion/react";
|
||||
import Image from "next/image";
|
||||
import { useParams } from "next/navigation";
|
||||
import { type FC, useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
|
@ -56,7 +56,6 @@ import { currentUserAtom } from "@/atoms/user/user-query.atoms";
|
|||
import { AssistantMessage } from "@/components/assistant-ui/assistant-message";
|
||||
import { ChatSessionStatus } from "@/components/assistant-ui/chat-session-status";
|
||||
import { ChatViewport } from "@/components/assistant-ui/chat-viewport";
|
||||
import { ConnectorIndicator } from "@/components/assistant-ui/connector-popup";
|
||||
import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup";
|
||||
import {
|
||||
InlineMentionEditor,
|
||||
|
|
@ -94,6 +93,7 @@ import {
|
|||
import { Popover, PopoverAnchor } from "@/components/ui/popover";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||
import {
|
||||
CONNECTOR_ICON_TO_TYPES,
|
||||
|
|
@ -105,9 +105,11 @@ import { useBatchCommentsPreload } from "@/hooks/use-comments";
|
|||
import { useCommentsSync } from "@/hooks/use-comments-sync";
|
||||
import { useMediaQuery } from "@/hooks/use-media-query";
|
||||
import { useElectronAPI } from "@/hooks/use-platform";
|
||||
import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities";
|
||||
import { captureDisplayToPngDataUrl } from "@/lib/chat/display-media-capture";
|
||||
import { getMentionDocKey } from "@/lib/chat/mention-doc-key";
|
||||
import { slideoutOpenedTickAtom } from "@/lib/layout-events";
|
||||
import { findPlatform, type PlaygroundPlatform } from "@/lib/playground/catalog";
|
||||
import { getWorkspaceIdNumber } from "@/lib/route-params";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
|
|
@ -965,7 +967,6 @@ const Composer: FC = () => {
|
|||
workspaceId={workspaceId ?? 0}
|
||||
onChatModelSelected={handleChatModelSelected}
|
||||
/>
|
||||
<ConnectorIndicator showTrigger={false} />
|
||||
</div>
|
||||
<ConnectToolsBanner
|
||||
isThreadEmpty={isThreadEmpty}
|
||||
|
|
@ -981,6 +982,66 @@ const Composer: FC = () => {
|
|||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Full-color brand marks for the platform-native scraper APIs (web, Google
|
||||
* Search, Google Maps, Reddit, YouTube) available in this workspace, shown beside the
|
||||
* composer "+" so the user can see these native endpoints are connected. Laid
|
||||
* out as a clean row (not stacked) after a hairline divider that separates them
|
||||
* from the composer actions. The capability registry is the source of truth;
|
||||
* icons are display-only with a status tooltip. One-time staggered entrance,
|
||||
* reduced-motion aware.
|
||||
*/
|
||||
const ConnectedScraperIcons: FC<{ workspaceId: number }> = ({ workspaceId }) => {
|
||||
const { data: capabilities } = useScraperCapabilities(workspaceId);
|
||||
const reduceMotion = useReducedMotion();
|
||||
|
||||
const platforms = useMemo<PlaygroundPlatform[]>(() => {
|
||||
if (!capabilities?.length) return [];
|
||||
const seen = new Set<string>();
|
||||
const result: PlaygroundPlatform[] = [];
|
||||
for (const cap of capabilities) {
|
||||
const platformId = cap.name.split(".")[0];
|
||||
if (seen.has(platformId)) continue;
|
||||
seen.add(platformId);
|
||||
const platform = findPlatform(platformId);
|
||||
if (platform) result.push(platform);
|
||||
}
|
||||
return result;
|
||||
}, [capabilities]);
|
||||
|
||||
if (platforms.length === 0) return null;
|
||||
|
||||
return (
|
||||
<div className="hidden items-center gap-1 sm:flex">
|
||||
<div aria-hidden className="h-5 w-px shrink-0 bg-border" />
|
||||
<div className="flex items-center gap-0.5" aria-label="Connected data sources">
|
||||
{platforms.map((platform, i) => {
|
||||
const Icon = platform.icon;
|
||||
return (
|
||||
<Tooltip key={platform.id}>
|
||||
<TooltipTrigger asChild>
|
||||
<motion.span
|
||||
initial={{ opacity: 0, y: reduceMotion ? 0 : 4 }}
|
||||
animate={{ opacity: 1, y: 0 }}
|
||||
transition={{
|
||||
duration: 0.2,
|
||||
ease: [0.23, 1, 0.32, 1],
|
||||
delay: reduceMotion ? 0 : i * 0.04,
|
||||
}}
|
||||
className="flex size-5 items-center justify-center"
|
||||
>
|
||||
<Icon className="size-3.5" />
|
||||
</motion.span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom">{platform.label} · Connected</TooltipContent>
|
||||
</Tooltip>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
interface ComposerActionProps {
|
||||
isBlockedByOtherUser?: boolean;
|
||||
workspaceId: number;
|
||||
|
|
@ -1142,7 +1203,7 @@ const ComposerAction: FC<ComposerActionProps> = ({
|
|||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onSelect={() => setConnectorDialogOpen(true)}>
|
||||
<Unplug className="size-4" />
|
||||
Manage Connectors
|
||||
Manage External MCP Connectors
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onSelect={() => setToolsPopoverOpen(true)}>
|
||||
<Settings2 className="size-4" />
|
||||
|
|
@ -1362,7 +1423,7 @@ const ComposerAction: FC<ComposerActionProps> = ({
|
|||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onSelect={() => setConnectorDialogOpen(true)}>
|
||||
<Unplug className="h-4 w-4" />
|
||||
Manage Connectors
|
||||
Manage External MCP Connectors
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuSub
|
||||
open={toolsPopoverOpen}
|
||||
|
|
@ -1556,6 +1617,7 @@ const ComposerAction: FC<ComposerActionProps> = ({
|
|||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
)}
|
||||
<ConnectedScraperIcons workspaceId={workspaceId} />
|
||||
</div>
|
||||
{!hasModelConfigured && (
|
||||
<div className="flex items-center gap-1.5 text-amber-600 dark:text-amber-400 text-xs">
|
||||
|
|
|
|||
|
|
@ -313,7 +313,7 @@ export function FolderTreeView({
|
|||
<div className="flex flex-1 flex-col items-center justify-center gap-1 px-4 py-12 text-muted-foreground select-none">
|
||||
<p className="text-sm font-medium">No documents found</p>
|
||||
<p className="text-xs text-muted-foreground/70">
|
||||
Use the plus menu to upload files or manage connectors
|
||||
Use the Import button above to add files, or the plus menu to manage connectors
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -2,16 +2,29 @@
|
|||
|
||||
import { useQuery } from "@rocicorp/zero/react";
|
||||
import { useAtom, useAtomValue, useSetAtom } from "jotai";
|
||||
import { FolderPlus, ListFilter, SlidersVertical, Trash2 } from "lucide-react";
|
||||
import {
|
||||
FolderInput,
|
||||
FolderPlus,
|
||||
ListFilter,
|
||||
Plus,
|
||||
Settings2,
|
||||
SlidersVertical,
|
||||
Trash2,
|
||||
Upload,
|
||||
} from "lucide-react";
|
||||
import { useParams } from "next/navigation";
|
||||
import { useTranslations } from "next-intl";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { makeFolderMention, mentionedDocumentsAtom } from "@/atoms/chat/mentioned-documents.atom";
|
||||
import { importConnectorRequestAtom } from "@/atoms/connector-dialog/connector-dialog.atoms";
|
||||
import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
|
||||
import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
|
||||
import { expandedFolderIdsAtom } from "@/atoms/documents/folder.atoms";
|
||||
import { agentCreatedDocumentsAtom } from "@/atoms/documents/ui.atoms";
|
||||
import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom";
|
||||
import { useConnectorStatus } from "@/components/assistant-ui/connector-popup/hooks/use-connector-status";
|
||||
import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup";
|
||||
import { CreateFolderDialog } from "@/components/documents/CreateFolderDialog";
|
||||
import type { DocumentNodeDoc } from "@/components/documents/DocumentNode";
|
||||
import { getDocumentTypeIcon } from "@/components/documents/DocumentTypeIcon";
|
||||
|
|
@ -19,7 +32,7 @@ import type { FolderDisplay } from "@/components/documents/FolderNode";
|
|||
import { FolderPickerDialog } from "@/components/documents/FolderPickerDialog";
|
||||
import { FolderTreeView } from "@/components/documents/FolderTreeView";
|
||||
import { VersionHistoryDialog } from "@/components/documents/version-history";
|
||||
import { useRuntimeConfig } from "@/components/providers/runtime-config";
|
||||
import { useIsSelfHosted, useRuntimeConfig } from "@/components/providers/runtime-config";
|
||||
import { EXPORT_FILE_EXTENSIONS } from "@/components/shared/ExportMenuItems";
|
||||
import { DEFAULT_EXCLUDE_PATTERNS } from "@/components/sources/FolderWatchDialog";
|
||||
import {
|
||||
|
|
@ -38,6 +51,7 @@ import {
|
|||
DropdownMenuCheckboxItem,
|
||||
DropdownMenuContent,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuSeparator,
|
||||
DropdownMenuSub,
|
||||
DropdownMenuSubContent,
|
||||
DropdownMenuSubTrigger,
|
||||
|
|
@ -47,6 +61,9 @@ import { Skeleton } from "@/components/ui/skeleton";
|
|||
import { Spinner } from "@/components/ui/spinner";
|
||||
import { useAnonymousMode, useIsAnonymous } from "@/contexts/anonymous-mode";
|
||||
import { useLoginGate } from "@/contexts/login-gate";
|
||||
import { EnumConnectorName } from "@/contracts/enums/connector";
|
||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
|
||||
import type { DocumentTypeEnum } from "@/contracts/types/document.types";
|
||||
import { useElectronAPI, usePlatform } from "@/hooks/use-platform";
|
||||
import { documentsApiService } from "@/lib/apis/documents-api.service";
|
||||
|
|
@ -163,6 +180,127 @@ export function EmbeddedDocumentsMenu({
|
|||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Import menu: local file upload plus the cloud-drive import connectors
|
||||
* (Google Drive / OneDrive / Dropbox). Drive/OneDrive/Dropbox set
|
||||
* `importConnectorRequestAtom`, which the connector dialog consumes to run
|
||||
* OAuth or open the existing account's config. In anonymous mode, `gate`
|
||||
* intercepts every item to trigger the login flow.
|
||||
*/
|
||||
export function EmbeddedImportMenu({ gate }: { gate?: (feature: string) => void }) {
|
||||
const { openDialog } = useDocumentUploadDialog();
|
||||
const setImportRequest = useSetAtom(importConnectorRequestAtom);
|
||||
const selfHosted = useIsSelfHosted();
|
||||
const { isConnectorEnabled, getConnectorStatusMessage } = useConnectorStatus();
|
||||
const { data: connectors } = useAtomValue(connectorsAtom);
|
||||
|
||||
// Native Google Drive connector self-hosted only; hosted deployments use Composio.
|
||||
const driveType = selfHosted
|
||||
? EnumConnectorName.GOOGLE_DRIVE_CONNECTOR
|
||||
: EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR;
|
||||
|
||||
const cloudItems = [
|
||||
{ type: driveType, label: "Google Drive" },
|
||||
{ type: EnumConnectorName.ONEDRIVE_CONNECTOR, label: "OneDrive" },
|
||||
{ type: EnumConnectorName.DROPBOX_CONNECTOR, label: "Dropbox" },
|
||||
];
|
||||
|
||||
return (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-7 w-7 text-muted-foreground hover:bg-accent hover:text-accent-foreground"
|
||||
aria-label="Import documents"
|
||||
>
|
||||
<FolderInput className="h-3.5 w-3.5" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" className="w-56">
|
||||
<DropdownMenuItem onSelect={() => (gate ? gate("upload files") : openDialog())}>
|
||||
<Upload className="h-4 w-4" />
|
||||
Upload Files
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuSeparator />
|
||||
{cloudItems.map((item) => {
|
||||
const enabled = gate ? true : isConnectorEnabled(item.type);
|
||||
const statusMessage = enabled ? null : getConnectorStatusMessage(item.type);
|
||||
const icon = getConnectorIcon(item.type, "h-4 w-4");
|
||||
// gate = anonymous mode; treat every connector as unconnected so items
|
||||
// route through the login gate rather than reading workspace connectors.
|
||||
const accountCount = gate
|
||||
? 0
|
||||
: (connectors ?? []).filter(
|
||||
(c: SearchSourceConnector) => c.connector_type === item.type
|
||||
).length;
|
||||
|
||||
// Unavailable (e.g. maintenance): non-actionable item explaining why.
|
||||
if (!enabled) {
|
||||
return (
|
||||
<DropdownMenuItem key={item.type} disabled title={statusMessage ?? undefined}>
|
||||
{icon}
|
||||
{item.label}
|
||||
</DropdownMenuItem>
|
||||
);
|
||||
}
|
||||
|
||||
// Connected: manage the existing account(s) or add another.
|
||||
if (accountCount > 0) {
|
||||
return (
|
||||
<DropdownMenuSub key={item.type}>
|
||||
<DropdownMenuSubTrigger>
|
||||
{icon}
|
||||
<span className="min-w-0 flex-1 truncate">{item.label}</span>
|
||||
<span className="ml-auto text-xs text-muted-foreground">{accountCount}</span>
|
||||
</DropdownMenuSubTrigger>
|
||||
<DropdownMenuSubContent className="w-48">
|
||||
<DropdownMenuItem
|
||||
onSelect={() =>
|
||||
gate
|
||||
? gate("manage import connectors")
|
||||
: setImportRequest({ connectorType: item.type, mode: "auto" })
|
||||
}
|
||||
>
|
||||
<Settings2 className="h-4 w-4" />
|
||||
{accountCount > 1 ? "Manage accounts" : "Manage"}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem
|
||||
onSelect={() =>
|
||||
gate
|
||||
? gate("import from cloud storage")
|
||||
: setImportRequest({ connectorType: item.type, mode: "connect" })
|
||||
}
|
||||
>
|
||||
<Plus className="h-4 w-4" />
|
||||
Add another account
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuSubContent>
|
||||
</DropdownMenuSub>
|
||||
);
|
||||
}
|
||||
|
||||
// Not connected: single click starts the first OAuth connect.
|
||||
return (
|
||||
<DropdownMenuItem
|
||||
key={item.type}
|
||||
onSelect={() =>
|
||||
gate
|
||||
? gate("import from cloud storage")
|
||||
: setImportRequest({ connectorType: item.type, mode: "auto" })
|
||||
}
|
||||
>
|
||||
{icon}
|
||||
{item.label}
|
||||
</DropdownMenuItem>
|
||||
);
|
||||
})}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
);
|
||||
}
|
||||
|
||||
function CloudDocumentsSkeleton() {
|
||||
const rows = [
|
||||
{ id: "row-1", widthClass: "w-44" },
|
||||
|
|
@ -1041,12 +1179,15 @@ function AuthenticatedDocumentsSidebarBase({
|
|||
defaultOpen={true}
|
||||
contentClassName="px-0"
|
||||
persistentAction={
|
||||
<EmbeddedDocumentsMenu
|
||||
typeCounts={typeCounts}
|
||||
activeTypes={activeTypes}
|
||||
onToggleType={onToggleType}
|
||||
onCreateFolder={() => handleCreateFolder(null)}
|
||||
/>
|
||||
<div className="flex items-center gap-0.5">
|
||||
<EmbeddedImportMenu />
|
||||
<EmbeddedDocumentsMenu
|
||||
typeCounts={typeCounts}
|
||||
activeTypes={activeTypes}
|
||||
onToggleType={onToggleType}
|
||||
onCreateFolder={() => handleCreateFolder(null)}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
>
|
||||
{renderDocumentTree()}
|
||||
|
|
@ -1214,12 +1355,15 @@ function AnonymousDocumentsSidebar({ embedded = false }: DocumentsSidebarProps)
|
|||
defaultOpen={true}
|
||||
contentClassName="px-0"
|
||||
persistentAction={
|
||||
<EmbeddedDocumentsMenu
|
||||
typeCounts={hasDoc ? { FILE: 1 } : {}}
|
||||
activeTypes={[]}
|
||||
onToggleType={() => {}}
|
||||
onCreateFolder={() => gate("create folders")}
|
||||
/>
|
||||
<div className="flex items-center gap-0.5">
|
||||
<EmbeddedImportMenu gate={gate} />
|
||||
<EmbeddedDocumentsMenu
|
||||
typeCounts={hasDoc ? { FILE: 1 } : {}}
|
||||
activeTypes={[]}
|
||||
onToggleType={() => {}}
|
||||
onCreateFolder={() => gate("create folders")}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
>
|
||||
<FolderTreeView
|
||||
|
|
|
|||
|
|
@ -1,103 +0,0 @@
|
|||
---
|
||||
title: Airtable
|
||||
description: Connect your Airtable bases to SurfSense
|
||||
---
|
||||
|
||||
# Airtable OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up an Airtable OAuth integration for SurfSense.
|
||||
|
||||
## Step 1: Access Airtable OAuth Integrations
|
||||
|
||||
1. Navigate to [airtable.com/create/oauth](https://airtable.com/create/oauth)
|
||||
2. In the **Builder Hub**, under **Developers**, click **"OAuth integrations"**
|
||||
3. Click **"Register an OAuth integration"**
|
||||
|
||||
|
||||
|
||||
## Step 2: Register an Integration
|
||||
|
||||
Fill in the basic integration details:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Name** | `SurfSense` |
|
||||
| **OAuth redirect URL** | `http://localhost:8000/api/v1/auth/airtable/connector/callback` |
|
||||
|
||||
Click **"Register integration"**
|
||||
|
||||
|
||||
|
||||
## Step 3: Configure Scopes
|
||||
|
||||
After registration, configure the required scopes (permissions) for your integration:
|
||||
|
||||
### Record data and comments
|
||||
|
||||
| Scope | Description |
|
||||
|-------|-------------|
|
||||
| `data.recordComments:read` | See comments in records |
|
||||
| `data.records:read` | See the data in records |
|
||||
|
||||
### Base schema
|
||||
|
||||
| Scope | Description |
|
||||
|-------|-------------|
|
||||
| `schema.bases:read` | See the structure of a base, like table names or field types |
|
||||
|
||||
### User metadata
|
||||
|
||||
| Scope | Description |
|
||||
|-------|-------------|
|
||||
| `user.email:read` | See the user's email address |
|
||||
|
||||
|
||||
|
||||
## Step 4: Configure Support Information
|
||||
|
||||
Scroll down to configure the support information and authorization preview:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Support email** | Your support email address |
|
||||
| **Privacy policy URL** | Your privacy policy URL |
|
||||
| **Terms of service URL** | Your terms of service URL |
|
||||
|
||||
The preview shows what users will see when authorizing SurfSense:
|
||||
- The data in your records
|
||||
- Comments in your records
|
||||
- The structure of your base, like table names or field types
|
||||
- Your email address
|
||||
|
||||
Click **"Save changes"**
|
||||
|
||||
|
||||
|
||||
## Step 5: Get OAuth Credentials
|
||||
|
||||
After saving, you'll find your OAuth credentials on the integration page:
|
||||
|
||||
1. Copy your **Client ID**
|
||||
2. Copy your **Client Secret**
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Airtable Connector
|
||||
|
||||
Add the Airtable credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
AIRTABLE_CLIENT_ID=your_airtable_client_id
|
||||
AIRTABLE_CLIENT_SECRET=your_airtable_client_secret
|
||||
AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
---
|
||||
title: ClickUp
|
||||
description: Connect your ClickUp workspace to SurfSense
|
||||
---
|
||||
|
||||
# ClickUp OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up a ClickUp OAuth integration for SurfSense.
|
||||
|
||||
## Step 1: Access ClickUp API Settings
|
||||
|
||||
1. Open your ClickUp workspace
|
||||
2. Navigate to **Settings** (gear icon) → **ClickUp API**
|
||||
3. You'll see the **ClickUp API Settings** page
|
||||
|
||||
|
||||
|
||||
## Step 2: Create an App
|
||||
|
||||
1. Click **"+ Create an App"** in the top-right corner
|
||||
2. Fill in the app details:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **App Name** | `SurfSense` |
|
||||
| **Redirect URL(s)** | `localhost:8000` |
|
||||
|
||||
3. Click **"Save"** to create the app
|
||||
|
||||
|
||||
|
||||
## Step 3: Get OAuth Credentials
|
||||
|
||||
After creating the app, you'll see your credentials:
|
||||
|
||||
1. Copy your **Client ID**
|
||||
2. Copy your **Client Secret** (click "Show" to reveal, or "Regenerate" if needed)
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with ClickUp Connector
|
||||
|
||||
Add the ClickUp credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
CLICKUP_CLIENT_ID=your_clickup_client_id
|
||||
CLICKUP_CLIENT_SECRET=your_clickup_client_secret
|
||||
CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
|
@ -1,114 +0,0 @@
|
|||
---
|
||||
title: Confluence
|
||||
description: Connect your Confluence spaces to SurfSense
|
||||
---
|
||||
|
||||
# Confluence OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up an Atlassian OAuth 2.0 (3LO) integration for SurfSense to connect your Confluence spaces.
|
||||
|
||||
## Step 1: Access the Developer Console
|
||||
|
||||
1. Navigate to [developer.atlassian.com](https://developer.atlassian.com)
|
||||
2. Click your profile icon in the top-right corner
|
||||
3. Select **"Developer console"** from the dropdown
|
||||
|
||||
|
||||
|
||||
## Step 2: Create a New OAuth 2.0 Integration
|
||||
|
||||
1. In the Developer Console, under **My apps**, click the **"Create"** button
|
||||
2. Select **"OAuth 2.0 integration"** from the dropdown
|
||||
|
||||
|
||||
|
||||
## Step 3: Name Your Integration
|
||||
|
||||
1. Enter **Name**: `SurfSense`
|
||||
2. Check the box to agree to Atlassian's developer terms
|
||||
3. Click **"Create"**
|
||||
|
||||
<Callout type="info">
|
||||
New OAuth 2.0 integrations use rotating refresh tokens, which improve security by limiting token validity and enabling automatic detection of token reuse.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
## Step 4: Configure Callback URL
|
||||
|
||||
1. In the left sidebar, click **"Authorization"**
|
||||
2. Under **Callback URLs**, enter the redirect URI:
|
||||
```
|
||||
http://localhost:8000/api/v1/auth/confluence/connector/callback
|
||||
```
|
||||
3. Click **"Save changes"**
|
||||
|
||||
<Callout type="info">
|
||||
You can enter up to 10 redirect URIs, one per line.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
## Step 5: Configure API Permissions
|
||||
|
||||
1. In the left sidebar, click **"Permissions"**
|
||||
2. You'll see a list of available APIs including Confluence API
|
||||
|
||||
|
||||
|
||||
## Step 6: Configure Confluence API Scopes
|
||||
|
||||
1. Click **"Configure"** next to **Confluence API**
|
||||
|
||||
### Classic Scopes
|
||||
|
||||
Select the **"Classic scopes"** tab and enable:
|
||||
|
||||
| Scope Name | Code | Description |
|
||||
|------------|------|-------------|
|
||||
| Read user | `read:confluence-user` | View user information in Confluence that you have access to, including usernames, email addresses and profile pictures |
|
||||
|
||||
|
||||
|
||||
### Granular Scopes
|
||||
|
||||
Select the **"Granular scopes"** tab and enable:
|
||||
|
||||
| Scope Name | Code | Description |
|
||||
|------------|------|-------------|
|
||||
| View pages | `read:page:confluence` | View page content |
|
||||
| View comments | `read:comment:confluence` | View comments on pages or blogposts |
|
||||
| View spaces | `read:space:confluence` | View space details |
|
||||
| Write pages | `write:page:confluence` | Create and update page content |
|
||||
| Delete pages | `delete:page:confluence` | Delete pages |
|
||||
|
||||
4. Click **"Save"**
|
||||
|
||||
|
||||
|
||||
## Step 7: Get OAuth Credentials
|
||||
|
||||
1. In the left sidebar, click **"Settings"**
|
||||
2. Copy your **Client ID** and **Client Secret**
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Confluence Connector
|
||||
|
||||
Add the Atlassian credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
ATLASSIAN_CLIENT_ID=your_atlassian_client_id
|
||||
ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret
|
||||
CONFLUENCE_REDIRECT_URI=http://localhost:8000/api/v1/auth/confluence/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
|
@ -5,6 +5,10 @@ description: Connect your Discord servers to SurfSense
|
|||
|
||||
# Discord OAuth Integration Setup Guide
|
||||
|
||||
<Callout type="warn" title="Deprecated">
|
||||
The Discord connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Existing connections remain manageable. The guide below is retained for reference only.
|
||||
</Callout>
|
||||
|
||||
This guide walks you through setting up a Discord OAuth integration for SurfSense.
|
||||
|
||||
## Step 1: Create a New Discord Application
|
||||
|
|
@ -64,7 +68,7 @@ You'll also see your **Application ID** and **Public Key** on this page.
|
|||
|
||||
## Running SurfSense with Discord Connector
|
||||
|
||||
Add the Discord credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
Add the Discord credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
|
||||
|
||||
```bash
|
||||
DISCORD_CLIENT_ID=your_discord_client_id
|
||||
|
|
@ -5,6 +5,10 @@ description: Connect your Elasticsearch cluster to SurfSense
|
|||
|
||||
# Elasticsearch Integration Setup Guide
|
||||
|
||||
<Callout type="warn" title="Deprecated">
|
||||
The Elasticsearch connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Existing connections remain manageable. The guide below is retained for reference only.
|
||||
</Callout>
|
||||
|
||||
This guide walks you through connecting your Elasticsearch cluster to SurfSense.
|
||||
|
||||
## How it works
|
||||
|
|
@ -5,6 +5,10 @@ description: Connect your Luma events to SurfSense
|
|||
|
||||
# Luma Integration Setup Guide
|
||||
|
||||
<Callout type="warn" title="Deprecated">
|
||||
The Luma connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Existing connections remain manageable. The guide below is retained for reference only.
|
||||
</Callout>
|
||||
|
||||
This guide walks you through connecting your Luma events to SurfSense for event search and AI-powered insights.
|
||||
|
||||
## How it works
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"title": "Deprecated",
|
||||
"pages": ["discord", "microsoft-teams", "luma", "baidu-search", "elasticsearch", "web-crawler"],
|
||||
"defaultOpen": false
|
||||
}
|
||||
|
|
@ -7,8 +7,12 @@ description: Connect your Microsoft Teams to SurfSense
|
|||
|
||||
This guide walks you through setting up a Microsoft Teams OAuth integration for SurfSense using Azure App Registration.
|
||||
|
||||
<Callout type="warn" title="Deprecated">
|
||||
The Microsoft Teams connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Existing connections remain manageable. The guide below is retained for reference only.
|
||||
</Callout>
|
||||
|
||||
<Callout type="info">
|
||||
Microsoft Teams and [Microsoft OneDrive](/docs/connectors/microsoft-onedrive) share the same Azure App Registration. If you have already created an app for OneDrive, you can reuse the same Client ID and Client Secret. Just make sure both redirect URIs are added (see Step 3).
|
||||
Microsoft Teams and [Microsoft OneDrive](/docs/connectors/external/onedrive) share the same Azure App Registration. If you have already created an app for OneDrive, you can reuse the same Client ID and Client Secret. Just make sure both redirect URIs are added (see Step 3).
|
||||
</Callout>
|
||||
|
||||
## Step 1: Access Azure App Registrations
|
||||
|
|
@ -101,7 +105,7 @@ After registration, you will be taken to the app's **Overview** page. Here you w
|
|||
|
||||
## Running SurfSense with Microsoft Teams Connector
|
||||
|
||||
Add the Microsoft OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
Add the Microsoft OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)):
|
||||
|
||||
```bash
|
||||
MICROSOFT_CLIENT_ID=your_microsoft_client_id
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
---
|
||||
title: Web Crawler
|
||||
description: The web crawler connector is deprecated
|
||||
---
|
||||
|
||||
# Web Crawler
|
||||
|
||||
<Callout type="warn" title="Deprecated">
|
||||
The web crawler connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Crawling is now built into the platform as the [Web Crawl native scraper](/docs/connectors/native/web-crawl). To save individual webpages, you can also use the [SurfSense browser extension](https://github.com/MODSetter/SurfSense/tree/main/surfsense_browser_extension), which captures pages behind authentication.
|
||||
</Callout>
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
---
|
||||
title: Dropbox
|
||||
description: Connect your Dropbox to SurfSense
|
||||
---
|
||||
|
||||
# Dropbox OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up a Dropbox OAuth integration for SurfSense using the Dropbox App Console.
|
||||
|
||||
## Step 1: Access the Dropbox App Console
|
||||
|
||||
1. Navigate to [dropbox.com/developers/apps](https://www.dropbox.com/developers/apps)
|
||||
2. Sign in with your Dropbox account
|
||||
|
||||
## Step 2: Create a New App
|
||||
|
||||
1. Click **"Create app"**
|
||||
2. Fill in the app details:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Choose an API** | Select **"Scoped access"** |
|
||||
| **Choose the type of access** | Select **"Full Dropbox"** |
|
||||
| **Name your app** | `SurfSense` (or any unique name) |
|
||||
|
||||
3. Click **"Create app"**
|
||||
|
||||
## Step 3: Configure Redirect URI
|
||||
|
||||
1. On your app's **Settings** page, scroll to the **OAuth 2** section
|
||||
2. Under **Redirect URIs**, add: `http://localhost:8000/api/v1/auth/dropbox/connector/callback`
|
||||
3. Click **"Add"**
|
||||
|
||||
## Step 4: Get App Key and Secret
|
||||
|
||||
On the same **Settings** page, you will find:
|
||||
|
||||
1. **App key** - this is your `DROPBOX_APP_KEY`
|
||||
2. **App secret** - click **"Show"** to reveal, then copy. This is your `DROPBOX_APP_SECRET`
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your app secret publicly or include it in code repositories.
|
||||
</Callout>
|
||||
|
||||
## Step 5: Configure Permissions
|
||||
|
||||
1. Go to the **Permissions** tab of your app
|
||||
2. Enable the following scopes:
|
||||
|
||||
| Permission | Description |
|
||||
|------------|-------------|
|
||||
| `files.metadata.read` | View information about files and folders |
|
||||
| `files.content.read` | View and download file content |
|
||||
| `files.content.write` | Create, modify, and delete files |
|
||||
| `account_info.read` | View basic account information |
|
||||
|
||||
3. Click **"Submit"** to save the permissions
|
||||
|
||||
<Callout type="warn">
|
||||
All four permissions listed above are required. The connector will not authenticate successfully if any are missing.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Dropbox Connector
|
||||
|
||||
Add the Dropbox OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
DROPBOX_APP_KEY=your_dropbox_app_key
|
||||
DROPBOX_APP_SECRET=your_dropbox_app_secret
|
||||
DROPBOX_REDIRECT_URI=http://localhost:8000/api/v1/auth/dropbox/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
69
surfsense_web/content/docs/connectors/external/airtable.mdx
vendored
Normal file
69
surfsense_web/content/docs/connectors/external/airtable.mdx
vendored
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
---
|
||||
title: Airtable
|
||||
description: Let the SurfSense agent browse your Airtable bases, tables, and records
|
||||
---
|
||||
|
||||
The Airtable connector connects SurfSense to Airtable's hosted MCP server, giving the agent live tools to list your bases, browse tables, and read records. Nothing is indexed in the background.
|
||||
|
||||
<Callout type="info">
|
||||
This setup is only needed on **self-hosted** deployments. On SurfSense Cloud, just click **Connect**.
|
||||
</Callout>
|
||||
|
||||
## Step 1: Register an OAuth Integration
|
||||
|
||||
1. Navigate to [airtable.com/create/oauth](https://airtable.com/create/oauth)
|
||||
2. In the **Builder Hub**, under **Developers**, click **"OAuth integrations"**
|
||||
3. Click **"Register an OAuth integration"**
|
||||
|
||||
|
||||
|
||||
Fill in the details:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Name** | `SurfSense` |
|
||||
| **OAuth redirect URL** | `http://localhost:3929/api/v1/auth/mcp/airtable/connector/callback` |
|
||||
|
||||
<Callout type="info">
|
||||
`http://localhost:3929` is the default public URL of a Docker install. If you run SurfSense [manually](/docs/manual-installation), use your backend URL (`http://localhost:8000`); in production, use your public domain.
|
||||
</Callout>
|
||||
|
||||
Click **"Register integration"**
|
||||
|
||||
|
||||
|
||||
## Step 2: Configure Scopes
|
||||
|
||||
Enable the scopes SurfSense requests:
|
||||
|
||||
| Scope | Description |
|
||||
|-------|-------------|
|
||||
| `data.records:read` | See the data in records |
|
||||
| `schema.bases:read` | See the structure of a base, like table names or field types |
|
||||
|
||||
|
||||
|
||||
## Step 3: Fill in Support Information
|
||||
|
||||
Add a support email (plus privacy policy and terms URLs if you have them) and click **"Save changes"**.
|
||||
|
||||
|
||||
|
||||
## Step 4: Add the Credentials to SurfSense
|
||||
|
||||
Copy the **Client ID** and **Client Secret** from the integration page and add them to your `.env` file:
|
||||
|
||||
```bash
|
||||
AIRTABLE_CLIENT_ID=your_airtable_client_id
|
||||
AIRTABLE_CLIENT_SECRET=your_airtable_client_secret
|
||||
```
|
||||
|
||||
Restart SurfSense, open the **Connectors** dialog, and click **Connect** on Airtable:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly.
|
||||
</Callout>
|
||||
|
|
@ -5,6 +5,10 @@ description: Connect your Bookstack instance to SurfSense
|
|||
|
||||
# BookStack Integration Setup Guide
|
||||
|
||||
<Callout type="warn" title="Indexing deprecated">
|
||||
The BookStack indexing pipeline is currently deprecated — the knowledge base now stores files, notes, and uploads only. Existing BookStack connectors remain manageable, but new indexing runs are refused by the backend.
|
||||
</Callout>
|
||||
|
||||
This guide walks you through connecting your BookStack instance to SurfSense.
|
||||
|
||||
## How it works
|
||||
71
surfsense_web/content/docs/connectors/external/dropbox.mdx
vendored
Normal file
71
surfsense_web/content/docs/connectors/external/dropbox.mdx
vendored
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
---
|
||||
title: Dropbox
|
||||
description: Import your Dropbox files into the SurfSense knowledge base
|
||||
---
|
||||
|
||||
The Dropbox connector lets you pick files from Dropbox and index them into your knowledge base. It lives in the Documents sidebar's **Import** menu, not the connector catalog.
|
||||
|
||||
<Callout type="info">
|
||||
This setup is only needed on **self-hosted** deployments. On SurfSense Cloud, just use **Import → Dropbox**.
|
||||
</Callout>
|
||||
|
||||
## Step 1: Create a Dropbox App
|
||||
|
||||
1. Navigate to [dropbox.com/developers/apps](https://www.dropbox.com/developers/apps) and sign in
|
||||
2. Click **"Create app"** and fill in:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Choose an API** | **"Scoped access"** |
|
||||
| **Choose the type of access** | **"Full Dropbox"** |
|
||||
| **Name your app** | `SurfSense` (or any unique name) |
|
||||
|
||||
3. Click **"Create app"**
|
||||
|
||||
## Step 2: Configure the Redirect URI
|
||||
|
||||
1. On your app's **Settings** page, scroll to the **OAuth 2** section
|
||||
2. Under **Redirect URIs**, add:
|
||||
|
||||
```
|
||||
http://localhost:3929/api/v1/auth/dropbox/connector/callback
|
||||
```
|
||||
|
||||
<Callout type="info">
|
||||
`http://localhost:3929` is the default public URL of a Docker install. If you run SurfSense [manually](/docs/manual-installation), use your backend URL (`http://localhost:8000`); in production, use your public domain.
|
||||
</Callout>
|
||||
|
||||
## Step 3: Configure Permissions
|
||||
|
||||
1. Go to the **Permissions** tab of your app
|
||||
2. Enable all of the following scopes and click **"Submit"**:
|
||||
|
||||
| Permission | Description |
|
||||
|------------|-------------|
|
||||
| `files.metadata.read` | View information about files and folders |
|
||||
| `files.content.read` | View and download file content |
|
||||
| `files.content.write` | Create, modify, and delete files |
|
||||
| `account_info.read` | View basic account information |
|
||||
|
||||
<Callout type="warn">
|
||||
All four permissions are required. The connector will not authenticate successfully if any are missing.
|
||||
</Callout>
|
||||
|
||||
## Step 4: Add the Credentials to SurfSense
|
||||
|
||||
On the **Settings** page, copy the **App key** and **App secret** (click "Show" to reveal), and add them to your `.env` file:
|
||||
|
||||
```bash
|
||||
DROPBOX_APP_KEY=your_dropbox_app_key
|
||||
DROPBOX_APP_SECRET=your_dropbox_app_secret
|
||||
```
|
||||
|
||||
Restart SurfSense, open **Documents → Import → Dropbox**, and authorize:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your app secret publicly or include it in code repositories.
|
||||
</Callout>
|
||||
|
|
@ -5,6 +5,10 @@ description: Connect your GitHub repositories to SurfSense
|
|||
|
||||
# GitHub Integration Setup Guide
|
||||
|
||||
<Callout type="warn" title="Indexing deprecated">
|
||||
The GitHub indexing pipeline is currently deprecated — the knowledge base now stores files, notes, and uploads only. Existing GitHub connectors remain manageable, but new indexing runs are refused by the backend.
|
||||
</Callout>
|
||||
|
||||
This guide walks you through connecting your GitHub repositories to SurfSense for code search and AI-powered insights.
|
||||
|
||||
## How it works
|
||||
97
surfsense_web/content/docs/connectors/external/google.mdx
vendored
Normal file
97
surfsense_web/content/docs/connectors/external/google.mdx
vendored
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
---
|
||||
title: Google (Drive, Gmail, Calendar)
|
||||
description: Set up one Google OAuth app for the Drive, Gmail, and Calendar connectors
|
||||
---
|
||||
|
||||
One Google Cloud OAuth app powers Google login and all three Google connectors:
|
||||
|
||||
- **Google Drive** — import Drive files into your knowledge base via the Documents sidebar's **Import** menu
|
||||
- **Gmail** — the agent can search, read, draft, and send emails
|
||||
- **Google Calendar** — the agent can search and manage your events
|
||||
|
||||
<Callout type="info">
|
||||
This setup is only needed on **self-hosted** deployments. On SurfSense Cloud, just click **Connect**.
|
||||
</Callout>
|
||||
|
||||
<Callout type="info">
|
||||
The redirect URIs below use `http://localhost:3929` — the default public URL of a Docker install. If you run SurfSense [manually](/docs/manual-installation), use your backend URL (`http://localhost:8000`); in production, use your public domain.
|
||||
</Callout>
|
||||
|
||||
## Step 1: Create a Google Cloud Project
|
||||
|
||||
1. Navigate to the [Google Cloud Console](https://console.cloud.google.com/)
|
||||
2. Select an existing project or create a new one
|
||||
|
||||
## Step 2: Enable the APIs
|
||||
|
||||
1. Go to **APIs & Services** > **Library**
|
||||
2. Search for and enable:
|
||||
- **People API** (required for Google OAuth itself)
|
||||
- **Google Drive API** (for the Drive connector)
|
||||
- **Gmail API** (for the Gmail connector)
|
||||
- **Google Calendar API** (for the Calendar connector)
|
||||
|
||||
You only need to enable the APIs for the connectors you plan to use — the People API is always required.
|
||||
|
||||
|
||||
|
||||
## Step 3: Configure the OAuth Consent Screen
|
||||
|
||||
1. Go to **APIs & Services** > **OAuth consent screen**
|
||||
2. Select **External** user type (or Internal if using Google Workspace)
|
||||
3. Fill in the required information:
|
||||
- **App name**: `SurfSense`
|
||||
- **User support email**: Your email address
|
||||
- **Developer contact information**: Your email address
|
||||
4. Click **Save and Continue**
|
||||
|
||||
|
||||
|
||||
### Add Scopes
|
||||
|
||||
Click **Add or Remove Scopes** and add:
|
||||
|
||||
| Scope | Used by |
|
||||
|-------|---------|
|
||||
| `openid`, `.../auth/userinfo.email`, `.../auth/userinfo.profile` | All (basic OAuth) |
|
||||
| `https://www.googleapis.com/auth/drive` | Drive connector |
|
||||
| `https://www.googleapis.com/auth/gmail.modify` | Gmail connector |
|
||||
| `https://www.googleapis.com/auth/calendar.events` | Calendar connector |
|
||||
|
||||
<Callout type="info">
|
||||
The write-capable scopes (`drive`, `gmail.modify`, `calendar.events`) are what let the agent create files, send emails, and manage events — always with your confirmation in chat. If you only want read access, use `drive.readonly`, `gmail.readonly`, and `calendar.readonly` instead; the write tools then won't work.
|
||||
</Callout>
|
||||
|
||||
## Step 4: Create the OAuth Client
|
||||
|
||||
1. Go to **APIs & Services** > **Credentials**
|
||||
2. Click **Create Credentials** > **OAuth client ID**
|
||||
3. Select **Web application** as the application type
|
||||
4. Enter **Name**: `SurfSense`
|
||||
5. Under **Authorized redirect URIs**, add one URI per connector you plan to use:
|
||||
|
||||
```
|
||||
http://localhost:3929/api/v1/auth/google/drive/connector/callback
|
||||
http://localhost:3929/api/v1/auth/google/gmail/connector/callback
|
||||
http://localhost:3929/api/v1/auth/google/calendar/connector/callback
|
||||
```
|
||||
|
||||
6. Click **Create**
|
||||
|
||||
|
||||
|
||||
## Step 5: Add the Credentials to SurfSense
|
||||
|
||||
Copy the **Client ID** and **Client Secret** from the confirmation dialog, then add them to your `.env` file (see the Google section of `.env.example` for the exact variable names — the same client ID and secret are shared by Google login and all three connectors, plus one redirect URI variable per connector).
|
||||
|
||||
|
||||
|
||||
Restart SurfSense and the Google connectors are ready to connect:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly.
|
||||
</Callout>
|
||||
101
surfsense_web/content/docs/connectors/external/index.mdx
vendored
Normal file
101
surfsense_web/content/docs/connectors/external/index.mdx
vendored
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
---
|
||||
title: External Connectors
|
||||
description: Connect third-party services like Notion, Slack, Google, GitHub, and Obsidian
|
||||
---
|
||||
|
||||
import { Card, Cards } from 'fumadocs-ui/components/card';
|
||||
|
||||
External connectors link SurfSense to third-party services. Open the **Connectors** dialog in your workspace, pick a service, and connect — via OAuth, an API token, a webhook, or a companion plugin, depending on the service. You can connect multiple accounts of the same service.
|
||||
|
||||
Most of them act as **live tools for the AI agent** — when you chat, the agent can search your Notion pages, read Slack threads, or create Jira issues in real time. File sources (Google Drive, OneDrive, Dropbox) instead show up in the Documents sidebar's **Import** menu, where you pick files to index into your knowledge base.
|
||||
|
||||
## One-click connectors (no setup)
|
||||
|
||||
These work out of the box on every deployment — SurfSense connects to the service's hosted MCP server and the service issues credentials automatically. Just click **Connect**:
|
||||
|
||||
| Connector | What the agent can do |
|
||||
|-----------|----------------------|
|
||||
| **Notion** | Search, read, create, and update pages |
|
||||
| **Linear** | Search, read, and manage issues and projects |
|
||||
| **Jira** | Search issues with JQL, browse projects, create and edit issues |
|
||||
| **Confluence** | Search and read spaces, create and update pages |
|
||||
| **ClickUp** | Search and read tasks |
|
||||
|
||||
Write actions (creating a page, editing an issue) always ask for your confirmation in chat before they run.
|
||||
|
||||
## Connectors that need OAuth credentials (self-hosted)
|
||||
|
||||
For these, a self-hosted deployment needs its own OAuth app registered with the provider. Create the app once, put the credentials in your `.env` (each variable is documented in `.env.example`), restart, and the **Connect** button works for everyone on your instance:
|
||||
|
||||
<Cards>
|
||||
<Card
|
||||
title="Google (Drive, Gmail, Calendar)"
|
||||
description="One Google Cloud OAuth app powers all three connectors"
|
||||
href="/docs/connectors/external/google"
|
||||
/>
|
||||
<Card
|
||||
title="Slack"
|
||||
description="Search and read channels and threads"
|
||||
href="/docs/connectors/external/slack"
|
||||
/>
|
||||
<Card
|
||||
title="Airtable"
|
||||
description="Browse bases, tables, and records"
|
||||
href="/docs/connectors/external/airtable"
|
||||
/>
|
||||
<Card
|
||||
title="Microsoft OneDrive"
|
||||
description="Import your OneDrive files into the knowledge base"
|
||||
href="/docs/connectors/external/onedrive"
|
||||
/>
|
||||
<Card
|
||||
title="Dropbox"
|
||||
description="Import your Dropbox files into the knowledge base"
|
||||
href="/docs/connectors/external/dropbox"
|
||||
/>
|
||||
</Cards>
|
||||
|
||||
<Callout type="info">
|
||||
On SurfSense Cloud these are already configured — just click **Connect**. The setup guides above only apply to self-hosted deployments.
|
||||
</Callout>
|
||||
|
||||
## Token, webhook, and plugin connectors
|
||||
|
||||
These you configure yourself with an API token, a webhook, or a companion plugin:
|
||||
|
||||
<Cards>
|
||||
<Card
|
||||
title="GitHub"
|
||||
description="Index repositories into your knowledge base"
|
||||
href="/docs/connectors/external/github"
|
||||
/>
|
||||
<Card
|
||||
title="BookStack"
|
||||
description="Index your BookStack documentation"
|
||||
href="/docs/connectors/external/bookstack"
|
||||
/>
|
||||
<Card
|
||||
title="Circleback"
|
||||
description="Receive meeting notes and transcripts via webhook"
|
||||
href="/docs/connectors/external/circleback"
|
||||
/>
|
||||
<Card
|
||||
title="Obsidian"
|
||||
description="Sync your vault with the SurfSense Obsidian plugin"
|
||||
href="/docs/connectors/external/obsidian"
|
||||
/>
|
||||
</Cards>
|
||||
|
||||
There's also a generic **Custom MCP** connector in the catalog that lets you plug any MCP server into the agent — including API-key-based services like Tavily or Linkup.
|
||||
|
||||
## Importing files from Drive, OneDrive, and Dropbox
|
||||
|
||||
Google Drive, OneDrive, and Dropbox are file sources, so they don't appear in the connector catalog. Instead, open the **Documents** sidebar, click **Import**, and pick the service. After authorizing, browse and select the files you want — they're indexed into your knowledge base and kept searchable alongside your uploads and notes.
|
||||
|
||||
## Managing connections
|
||||
|
||||
Everything happens in the **Connectors** dialog:
|
||||
|
||||
- **Active** tab — see connected accounts, live status, and document counts.
|
||||
- **Reconnect** — if a token expires, the connector card prompts you to re-authorize.
|
||||
- **Disconnect** — removes the connection. For knowledge-base connectors this also removes the documents it indexed.
|
||||
15
surfsense_web/content/docs/connectors/external/meta.json
vendored
Normal file
15
surfsense_web/content/docs/connectors/external/meta.json
vendored
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"title": "External Connectors",
|
||||
"pages": [
|
||||
"google",
|
||||
"slack",
|
||||
"airtable",
|
||||
"onedrive",
|
||||
"dropbox",
|
||||
"github",
|
||||
"bookstack",
|
||||
"circleback",
|
||||
"obsidian"
|
||||
],
|
||||
"defaultOpen": false
|
||||
}
|
||||
78
surfsense_web/content/docs/connectors/external/onedrive.mdx
vendored
Normal file
78
surfsense_web/content/docs/connectors/external/onedrive.mdx
vendored
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
---
|
||||
title: Microsoft OneDrive
|
||||
description: Import your OneDrive files into the SurfSense knowledge base
|
||||
---
|
||||
|
||||
The OneDrive connector lets you pick files from OneDrive and index them into your knowledge base. It lives in the Documents sidebar's **Import** menu, not the connector catalog.
|
||||
|
||||
<Callout type="info">
|
||||
This setup is only needed on **self-hosted** deployments. On SurfSense Cloud, just use **Import → OneDrive**.
|
||||
</Callout>
|
||||
|
||||
## Step 1: Create an Azure App Registration
|
||||
|
||||
1. Navigate to [portal.azure.com](https://portal.azure.com)
|
||||
2. Search for **"App registrations"** and open it
|
||||
3. Click **"+ New registration"** and fill in:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Name** | `SurfSense` |
|
||||
| **Supported account types** | **"Accounts in any organizational directory (Any Microsoft Entra ID tenant - Multitenant) and personal Microsoft accounts"** |
|
||||
| **Redirect URI** | Platform: `Web`, URI: `http://localhost:3929/api/v1/auth/onedrive/connector/callback` |
|
||||
|
||||
<Callout type="info">
|
||||
`http://localhost:3929` is the default public URL of a Docker install. If you run SurfSense [manually](/docs/manual-installation), use your backend URL (`http://localhost:8000`); in production, use your public domain.
|
||||
</Callout>
|
||||
|
||||
4. Click **"Register"**
|
||||
|
||||
## Step 2: Get the Application (Client) ID
|
||||
|
||||
On the app's **Overview** page, copy the **Application (client) ID** — this is your Client ID.
|
||||
|
||||
## Step 3: Create a Client Secret
|
||||
|
||||
1. Under **Manage**, click **"Certificates & secrets"**
|
||||
2. On the **"Client secrets"** tab, click **"+ New client secret"**
|
||||
3. Enter a description (e.g., `SurfSense`) and pick an expiration period
|
||||
4. Click **"Add"**
|
||||
5. **Important**: Copy the secret **Value** immediately — it will not be shown again!
|
||||
|
||||
## Step 4: Configure API Permissions
|
||||
|
||||
1. Under **Manage**, click **"API permissions"** > **"+ Add a permission"**
|
||||
2. Select **"Microsoft Graph"** > **"Delegated permissions"**
|
||||
3. Add all of the following:
|
||||
|
||||
| Permission | Description |
|
||||
|------------|-------------|
|
||||
| `Files.Read.All` | Read all files the user can access |
|
||||
| `Files.ReadWrite.All` | Read and write all files the user can access |
|
||||
| `offline_access` | Maintain access to granted data |
|
||||
| `User.Read` | Sign in and read user profile |
|
||||
|
||||
4. Click **"Add permissions"**
|
||||
|
||||
<Callout type="warn">
|
||||
All four permissions are required. The connector will not authenticate successfully if any are missing.
|
||||
</Callout>
|
||||
|
||||
## Step 5: Add the Credentials to SurfSense
|
||||
|
||||
Add the credentials to your `.env` file:
|
||||
|
||||
```bash
|
||||
MICROSOFT_CLIENT_ID=your_microsoft_client_id
|
||||
MICROSOFT_CLIENT_SECRET=your_microsoft_client_secret
|
||||
```
|
||||
|
||||
Restart SurfSense, open **Documents → Import → OneDrive**, and authorize:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly or include it in code repositories.
|
||||
</Callout>
|
||||
98
surfsense_web/content/docs/connectors/external/slack.mdx
vendored
Normal file
98
surfsense_web/content/docs/connectors/external/slack.mdx
vendored
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
---
|
||||
title: Slack
|
||||
description: Let the SurfSense agent search and read your Slack workspace
|
||||
---
|
||||
|
||||
The Slack connector connects SurfSense to Slack's hosted MCP server, giving the agent live tools to search channels and read threads on your behalf. Nothing is indexed in the background — the agent queries Slack in real time when you ask.
|
||||
|
||||
<Callout type="info">
|
||||
This setup is only needed on **self-hosted** deployments. On SurfSense Cloud, just click **Connect**.
|
||||
</Callout>
|
||||
|
||||
## Step 1: Create a New Slack App
|
||||
|
||||
1. Navigate to [api.slack.com/apps](https://api.slack.com/apps)
|
||||
2. Click **"Create New App"**
|
||||
3. Select **"From scratch"**
|
||||
|
||||
|
||||
|
||||
## Step 2: Name App & Choose Workspace
|
||||
|
||||
1. Enter **App Name**: `SurfSense`
|
||||
2. Select the workspace to develop your app in
|
||||
3. Click **"Create App"**
|
||||
|
||||
<Callout type="warn">
|
||||
You won't be able to change the workspace later. The workspace will control the app even if you leave it.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
## Step 3: Get App Credentials
|
||||
|
||||
On the **Basic Information** page:
|
||||
|
||||
1. Copy your **Client ID**
|
||||
2. Copy your **Client Secret** (click Show to reveal)
|
||||
|
||||
|
||||
|
||||
## Step 4: Configure the Redirect URL
|
||||
|
||||
1. In the left sidebar, click **"OAuth & Permissions"**
|
||||
2. Under **Redirect URLs**, click **"Add New Redirect URL"**
|
||||
3. Enter your SurfSense callback URL:
|
||||
|
||||
```
|
||||
http://localhost:3929/api/v1/auth/mcp/slack/connector/callback
|
||||
```
|
||||
|
||||
<Callout type="info">
|
||||
`http://localhost:3929` is the default public URL of a Docker install. If you run SurfSense [manually](/docs/manual-installation), use your backend URL (`http://localhost:8000`); in production, use your public domain. Note that Slack requires HTTPS redirect URLs for production apps.
|
||||
</Callout>
|
||||
|
||||
4. Click **"Add"**, then **"Save URLs"**
|
||||
|
||||
|
||||
|
||||
## Step 5: Configure User Token Scopes
|
||||
|
||||
On the same **OAuth & Permissions** page, scroll to **Scopes**. The connector acts on behalf of the connecting user, so add these under **User Token Scopes**:
|
||||
|
||||
| OAuth Scope | Description |
|
||||
|-------------|-------------|
|
||||
| `search:read.public` | Search public channels |
|
||||
| `search:read.private` | Search private channels the user is in |
|
||||
| `search:read.mpim` | Search group direct messages |
|
||||
| `search:read.im` | Search direct messages |
|
||||
| `channels:history` | Read messages in public channels |
|
||||
| `groups:history` | Read messages in private channels |
|
||||
| `mpim:history` | Read group direct messages |
|
||||
| `im:history` | Read direct messages |
|
||||
|
||||
## Step 6: Enable Public Distribution
|
||||
|
||||
1. In the left sidebar, click **"Manage Distribution"**
|
||||
2. Under **Share Your App with Other Workspaces**, enable distribution so users can authorize the app in their own workspaces
|
||||
|
||||
|
||||
|
||||
## Step 7: Add the Credentials to SurfSense
|
||||
|
||||
Add the Client ID and Client Secret to your `.env` file:
|
||||
|
||||
```bash
|
||||
SLACK_CLIENT_ID=your_slack_client_id
|
||||
SLACK_CLIENT_SECRET=your_slack_client_secret
|
||||
```
|
||||
|
||||
Restart SurfSense, open the **Connectors** dialog, and click **Connect** on Slack:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your app credentials publicly.
|
||||
</Callout>
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
---
|
||||
title: Gmail
|
||||
description: Connect your Gmail to SurfSense
|
||||
---
|
||||
|
||||
# Gmail OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up a Google OAuth 2.0 integration for SurfSense to connect your Gmail account.
|
||||
|
||||
## Step 1: Access the Google Cloud Console
|
||||
|
||||
1. Navigate to [Google Cloud Console](https://console.cloud.google.com/)
|
||||
2. Select an existing project or create a new one
|
||||
|
||||
## Step 2: Enable Required APIs
|
||||
|
||||
1. Go to **APIs & Services** > **Library**
|
||||
2. Search for and enable the following APIs:
|
||||
- **People API** (required for Google OAuth)
|
||||
- **Gmail API** (required for Gmail connector)
|
||||
|
||||
|
||||
|
||||
## Step 3: Configure OAuth Consent Screen
|
||||
|
||||
1. Go to **APIs & Services** > **OAuth consent screen**
|
||||
2. Select **External** user type (or Internal if using Google Workspace)
|
||||
3. Fill in the required information:
|
||||
- **App name**: `SurfSense`
|
||||
- **User support email**: Your email address
|
||||
- **Developer contact information**: Your email address
|
||||
4. Click **Save and Continue**
|
||||
|
||||
|
||||
|
||||
### Add Scopes
|
||||
|
||||
1. Click **Add or Remove Scopes**
|
||||
2. Add the following scopes:
|
||||
- `https://www.googleapis.com/auth/gmail.modify` - Read, compose, send, and permanently delete Gmail messages
|
||||
- `https://www.googleapis.com/auth/userinfo.email` - View user email address
|
||||
- `https://www.googleapis.com/auth/userinfo.profile` - View user profile info
|
||||
- `openid` - OpenID Connect authentication
|
||||
3. Click **Update** and then **Save and Continue**
|
||||
|
||||
<Callout type="info">
|
||||
The `gmail.modify` scope is required for HITL (Human-in-the-Loop) tools like sending emails, creating drafts, and trashing messages. If you only need read access, you can use `gmail.readonly` instead, but HITL tools will not work.
|
||||
</Callout>
|
||||
|
||||
## Step 4: Create OAuth Client ID
|
||||
|
||||
1. Go to **APIs & Services** > **Credentials**
|
||||
2. Click **Create Credentials** > **OAuth client ID**
|
||||
3. Select **Web application** as the application type
|
||||
4. Enter **Name**: `SurfSense`
|
||||
5. Under **Authorized redirect URIs**, add:
|
||||
```
|
||||
http://localhost:8000/api/v1/auth/google/gmail/connector/callback
|
||||
```
|
||||
6. Click **Create**
|
||||
|
||||
|
||||
|
||||
## Step 5: Get OAuth Credentials
|
||||
|
||||
1. After creating the OAuth client, you'll see a dialog with your credentials
|
||||
2. Copy your **Client ID** and **Client Secret**
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Gmail Connector
|
||||
|
||||
Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
GOOGLE_OAUTH_CLIENT_ID=your_google_client_id
|
||||
GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret
|
||||
GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
---
|
||||
title: Google Calendar
|
||||
description: Connect your Google Calendar to SurfSense
|
||||
---
|
||||
|
||||
# Google Calendar OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up a Google OAuth 2.0 integration for SurfSense to connect your Google Calendar.
|
||||
|
||||
## Step 1: Access the Google Cloud Console
|
||||
|
||||
1. Navigate to [Google Cloud Console](https://console.cloud.google.com/)
|
||||
2. Select an existing project or create a new one
|
||||
|
||||
## Step 2: Enable Required APIs
|
||||
|
||||
1. Go to **APIs & Services** > **Library**
|
||||
2. Search for and enable the following APIs:
|
||||
- **People API** (required for Google OAuth)
|
||||
- **Google Calendar API** (required for Calendar connector)
|
||||
|
||||
|
||||
|
||||
## Step 3: Configure OAuth Consent Screen
|
||||
|
||||
1. Go to **APIs & Services** > **OAuth consent screen**
|
||||
2. Select **External** user type (or Internal if using Google Workspace)
|
||||
3. Fill in the required information:
|
||||
- **App name**: `SurfSense`
|
||||
- **User support email**: Your email address
|
||||
- **Developer contact information**: Your email address
|
||||
4. Click **Save and Continue**
|
||||
|
||||
|
||||
|
||||
### Add Scopes
|
||||
|
||||
1. Click **Add or Remove Scopes**
|
||||
2. Add the following scope:
|
||||
- `https://www.googleapis.com/auth/calendar.events` - Read, create, update, and delete Google Calendar events
|
||||
3. Click **Update** and then **Save and Continue**
|
||||
|
||||
<Callout type="info">
|
||||
The `calendar.events` scope is required for HITL (Human-in-the-Loop) tools like creating, updating, and deleting calendar events. If you only need read access, you can use `calendar.readonly` instead, but HITL tools will not work.
|
||||
</Callout>
|
||||
|
||||
## Step 4: Create OAuth Client ID
|
||||
|
||||
1. Go to **APIs & Services** > **Credentials**
|
||||
2. Click **Create Credentials** > **OAuth client ID**
|
||||
3. Select **Web application** as the application type
|
||||
4. Enter **Name**: `SurfSense`
|
||||
5. Under **Authorized redirect URIs**, add:
|
||||
```
|
||||
http://localhost:8000/api/v1/auth/google/calendar/connector/callback
|
||||
```
|
||||
6. Click **Create**
|
||||
|
||||
|
||||
|
||||
## Step 5: Get OAuth Credentials
|
||||
|
||||
1. After creating the OAuth client, you'll see a dialog with your credentials
|
||||
2. Copy your **Client ID** and **Client Secret**
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Google Calendar Connector
|
||||
|
||||
Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
GOOGLE_OAUTH_CLIENT_ID=your_google_client_id
|
||||
GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret
|
||||
GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
---
|
||||
title: Google Drive
|
||||
description: Connect your Google Drive to SurfSense
|
||||
---
|
||||
|
||||
# Google Drive OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up a Google OAuth 2.0 integration for SurfSense to connect your Google Drive.
|
||||
|
||||
## Step 1: Access the Google Cloud Console
|
||||
|
||||
1. Navigate to [Google Cloud Console](https://console.cloud.google.com/)
|
||||
2. Select an existing project or create a new one
|
||||
|
||||
## Step 2: Enable Required APIs
|
||||
|
||||
1. Go to **APIs & Services** > **Library**
|
||||
2. Search for and enable the following APIs:
|
||||
- **People API** (required for Google OAuth)
|
||||
- **Google Drive API** (required for Drive connector)
|
||||
|
||||
|
||||
|
||||
## Step 3: Configure OAuth Consent Screen
|
||||
|
||||
1. Go to **APIs & Services** > **OAuth consent screen**
|
||||
2. Select **External** user type (or Internal if using Google Workspace)
|
||||
3. Fill in the required information:
|
||||
- **App name**: `SurfSense`
|
||||
- **User support email**: Your email address
|
||||
- **Developer contact information**: Your email address
|
||||
4. Click **Save and Continue**
|
||||
|
||||
|
||||
|
||||
### Add Scopes
|
||||
|
||||
1. Click **Add or Remove Scopes**
|
||||
2. Add the following scopes:
|
||||
- `https://www.googleapis.com/auth/drive` - Full access to Google Drive (read, create, and trash files)
|
||||
- `https://www.googleapis.com/auth/userinfo.email` - View user email address
|
||||
- `https://www.googleapis.com/auth/userinfo.profile` - View user profile info
|
||||
- `openid` - OpenID Connect authentication
|
||||
3. Click **Update** and then **Save and Continue**
|
||||
|
||||
<Callout type="info">
|
||||
The `drive` scope is required for HITL (Human-in-the-Loop) tools like creating and trashing files. If you only need read access, you can use `drive.readonly` instead, but HITL tools will not work.
|
||||
</Callout>
|
||||
|
||||
## Step 4: Create OAuth Client ID
|
||||
|
||||
1. Go to **APIs & Services** > **Credentials**
|
||||
2. Click **Create Credentials** > **OAuth client ID**
|
||||
3. Select **Web application** as the application type
|
||||
4. Enter **Name**: `SurfSense`
|
||||
5. Under **Authorized redirect URIs**, add:
|
||||
```
|
||||
http://localhost:8000/api/v1/auth/google/drive/connector/callback
|
||||
```
|
||||
6. Click **Create**
|
||||
|
||||
|
||||
|
||||
## Step 5: Get OAuth Credentials
|
||||
|
||||
1. After creating the OAuth client, you'll see a dialog with your credentials
|
||||
2. Copy your **Client ID** and **Client Secret**
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Google Drive Connector
|
||||
|
||||
Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
GOOGLE_OAUTH_CLIENT_ID=your_google_client_id
|
||||
GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret
|
||||
GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
|
@ -1,121 +1,36 @@
|
|||
---
|
||||
title: Connectors
|
||||
description: Integrate with third-party services
|
||||
description: Connect SurfSense to your tools and services
|
||||
---
|
||||
|
||||
import { Card, Cards } from 'fumadocs-ui/components/card';
|
||||
import { Zap, Cable } from 'lucide-react';
|
||||
|
||||
Connect SurfSense to your favorite tools and services. Browse the available integrations below to sync data from productivity apps, communication platforms, knowledge bases, and more.
|
||||
Connectors bring data into SurfSense — either as searchable knowledge or as live tools the AI agent can use during chat. There are two kinds:
|
||||
|
||||
<Cards>
|
||||
<Card
|
||||
title="Google Drive"
|
||||
description="Connect your Google Drive to SurfSense"
|
||||
href="/docs/connectors/google-drive"
|
||||
icon={<Zap />}
|
||||
title="Native Connectors"
|
||||
description="SurfSense's built-in scraper APIs: Reddit, YouTube, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST"
|
||||
href="/docs/connectors/native"
|
||||
/>
|
||||
<Card
|
||||
title="Gmail"
|
||||
description="Connect your Gmail to SurfSense"
|
||||
href="/docs/connectors/gmail"
|
||||
/>
|
||||
<Card
|
||||
title="Google Calendar"
|
||||
description="Connect your Google Calendar to SurfSense"
|
||||
href="/docs/connectors/google-calendar"
|
||||
/>
|
||||
<Card
|
||||
title="Dropbox"
|
||||
description="Connect your Dropbox to SurfSense"
|
||||
href="/docs/connectors/dropbox"
|
||||
/>
|
||||
<Card
|
||||
title="Notion"
|
||||
description="Connect your Notion workspaces to SurfSense"
|
||||
href="/docs/connectors/notion"
|
||||
/>
|
||||
<Card
|
||||
title="Slack"
|
||||
description="Connect your Slack workspace to SurfSense"
|
||||
href="/docs/connectors/slack"
|
||||
/>
|
||||
<Card
|
||||
title="Discord"
|
||||
description="Connect your Discord servers to SurfSense"
|
||||
href="/docs/connectors/discord"
|
||||
/>
|
||||
<Card
|
||||
title="Jira"
|
||||
description="Connect your Jira projects to SurfSense"
|
||||
href="/docs/connectors/jira"
|
||||
/>
|
||||
<Card
|
||||
title="Linear"
|
||||
description="Connect your Linear workspace to SurfSense"
|
||||
href="/docs/connectors/linear"
|
||||
/>
|
||||
<Card
|
||||
title="Microsoft Teams"
|
||||
description="Connect your Microsoft Teams to SurfSense"
|
||||
href="/docs/connectors/microsoft-teams"
|
||||
/>
|
||||
<Card
|
||||
title="Microsoft OneDrive"
|
||||
description="Connect your Microsoft OneDrive to SurfSense"
|
||||
href="/docs/connectors/microsoft-onedrive"
|
||||
/>
|
||||
<Card
|
||||
title="Confluence"
|
||||
description="Connect your Confluence spaces to SurfSense"
|
||||
href="/docs/connectors/confluence"
|
||||
/>
|
||||
<Card
|
||||
title="Airtable"
|
||||
description="Connect your Airtable bases to SurfSense"
|
||||
href="/docs/connectors/airtable"
|
||||
/>
|
||||
<Card
|
||||
title="ClickUp"
|
||||
description="Connect your ClickUp workspace to SurfSense"
|
||||
href="/docs/connectors/clickup"
|
||||
/>
|
||||
<Card
|
||||
title="GitHub"
|
||||
description="Connect your GitHub repositories to SurfSense"
|
||||
href="/docs/connectors/github"
|
||||
/>
|
||||
<Card
|
||||
title="Baidu Search"
|
||||
description="Search the Chinese web with Baidu AI Search"
|
||||
href="/docs/connectors/baidu-search"
|
||||
/>
|
||||
<Card
|
||||
title="Luma"
|
||||
description="Connect your Luma events to SurfSense"
|
||||
href="/docs/connectors/luma"
|
||||
/>
|
||||
<Card
|
||||
title="Circleback"
|
||||
description="Connect your Circleback meetings to SurfSense"
|
||||
href="/docs/connectors/circleback"
|
||||
/>
|
||||
<Card
|
||||
title="Elasticsearch"
|
||||
description="Connect your Elasticsearch cluster to SurfSense"
|
||||
href="/docs/connectors/elasticsearch"
|
||||
/>
|
||||
<Card
|
||||
title="Bookstack"
|
||||
description="Connect your Bookstack instance to SurfSense"
|
||||
href="/docs/connectors/bookstack"
|
||||
/>
|
||||
<Card
|
||||
title="Obsidian"
|
||||
description="Sync your Obsidian vault using the SurfSense plugin"
|
||||
href="/docs/connectors/obsidian"
|
||||
/>
|
||||
<Card
|
||||
title="Web Crawler"
|
||||
description="Crawl and index any website into SurfSense"
|
||||
href="/docs/connectors/web-crawler"
|
||||
icon={<Cable />}
|
||||
title="External Connectors"
|
||||
description="Third-party integrations: Notion, Slack, Linear, Jira, Google, GitHub, Obsidian, and more"
|
||||
href="/docs/connectors/external"
|
||||
/>
|
||||
</Cards>
|
||||
|
||||
## How connectors surface in SurfSense
|
||||
|
||||
- **Native scrapers** — the AI agent uses them as tools automatically in chat, and you can run them yourself from the **API Playground** or your own code via the REST API.
|
||||
- **Tools & Live Sources** — external connectors like Notion, Slack, Jira, or Linear give the agent real-time tools. Ask a question in chat and the agent searches or acts on the connected service directly; nothing is copied into SurfSense in the background.
|
||||
- **Knowledge Base** — file sources like Google Drive, OneDrive, and Dropbox are imported through the Documents sidebar's **Import** menu and indexed into your knowledge base alongside uploads and notes.
|
||||
|
||||
To manage external connectors, open the **Connectors** dialog inside your workspace. Connected accounts, sync status, and disconnect options all live there.
|
||||
|
||||
## Deprecated connectors
|
||||
|
||||
Some older connectors (Discord, Microsoft Teams, Luma, Elasticsearch, the old web crawler, and standalone search APIs) have been retired. Existing connections remain manageable, but new ones can't be added. Their docs are kept for reference under [Deprecated](/docs/connectors/deprecated/discord). Public web search and crawling are now built in as [native scrapers](/docs/connectors/native), and other search APIs can still be wired up through a custom MCP server.
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
---
|
||||
title: Jira
|
||||
description: Connect your Jira projects to SurfSense
|
||||
---
|
||||
|
||||
# Jira OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up an Atlassian OAuth 2.0 (3LO) integration for SurfSense to connect your Jira projects.
|
||||
|
||||
## Step 1: Access the Developer Console
|
||||
|
||||
1. Navigate to [developer.atlassian.com](https://developer.atlassian.com)
|
||||
2. Click your profile icon in the top-right corner
|
||||
3. Select **"Developer console"** from the dropdown
|
||||
|
||||
|
||||
|
||||
## Step 2: Create a New OAuth 2.0 Integration
|
||||
|
||||
1. In the Developer Console, under **My apps**, click the **"Create"** button
|
||||
2. Select **"OAuth 2.0 integration"** from the dropdown
|
||||
|
||||
|
||||
|
||||
## Step 3: Name Your Integration
|
||||
|
||||
1. Enter **Name**: `SurfSense`
|
||||
2. Check the box to agree to Atlassian's developer terms
|
||||
3. Click **"Create"**
|
||||
|
||||
<Callout type="info">
|
||||
New OAuth 2.0 integrations use rotating refresh tokens, which improve security by limiting token validity and enabling automatic detection of token reuse.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
## Step 4: Configure Callback URL
|
||||
|
||||
1. In the left sidebar, click **"Authorization"**
|
||||
2. Under **Callback URLs**, enter the redirect URI:
|
||||
```
|
||||
http://localhost:8000/api/v1/auth/jira/connector/callback
|
||||
```
|
||||
3. Click **"Save changes"**
|
||||
|
||||
<Callout type="info">
|
||||
You can enter up to 10 redirect URIs, one per line.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
## Step 5: Configure API Permissions
|
||||
|
||||
1. In the left sidebar, click **"Permissions"**
|
||||
2. You'll see a list of available APIs including Jira API
|
||||
|
||||
|
||||
|
||||
## Step 6: Configure Jira API Scopes
|
||||
|
||||
1. Click **"Configure"** next to **Jira API**
|
||||
2. Select the **"Classic scopes"** tab
|
||||
3. Under **Jira platform REST API**, select the following scopes:
|
||||
|
||||
| Scope Name | Code | Description |
|
||||
|------------|------|-------------|
|
||||
| View Jira issue data | `read:jira-work` | Read Jira project and issue data, search for issues, and objects associated with issues like attachments and worklogs |
|
||||
| View user profiles | `read:jira-user` | View user information in Jira that the user has access to, including usernames, email addresses, and avatars |
|
||||
| Create and manage issues | `write:jira-work` | Create, edit, and delete issues, comments, worklogs, and other Jira data the user has access to |
|
||||
|
||||
4. Click **"Save"**
|
||||
|
||||
|
||||
|
||||
## Step 7: Get OAuth Credentials
|
||||
|
||||
1. In the left sidebar, click **"Settings"**
|
||||
2. Copy your **Client ID** and **Client Secret**
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Jira Connector
|
||||
|
||||
Add the Atlassian credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
ATLASSIAN_CLIENT_ID=your_atlassian_client_id
|
||||
ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret
|
||||
JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
---
|
||||
title: Linear
|
||||
description: Connect your Linear workspace to SurfSense
|
||||
---
|
||||
|
||||
# Linear OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up a Linear OAuth integration for SurfSense.
|
||||
|
||||
## Step 1: Access Linear API Settings
|
||||
|
||||
1. Navigate to your workspace's API settings at `linear.app/<your-workspace>/settings/api`
|
||||
2. Under **OAuth Applications**, click **"+ New OAuth application"**
|
||||
|
||||
|
||||
|
||||
## Step 2: Create New Application
|
||||
|
||||
Fill in the application details:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Application icon** | Upload an icon (at least 256x256px) |
|
||||
| **Application name** | `SurfSense` |
|
||||
| **Developer name** | `SurfSense` |
|
||||
| **Developer URL** | `https://www.surfsense.com/` |
|
||||
| **Description** | Connect any LLM to your internal knowledge sources and chat with it in real time alongside your team. |
|
||||
| **Callback URLs** | `http://localhost:8000/api/v1/auth/linear/connector/callback` |
|
||||
| **GitHub username** | Your GitHub username (optional) |
|
||||
|
||||
### Settings
|
||||
|
||||
- **Public** - Enable this to allow the application to be installed by other workspaces
|
||||
|
||||
Click **Create** to create the application.
|
||||
|
||||
|
||||
|
||||
## Step 3: Get OAuth Credentials
|
||||
|
||||
After creating the application, you'll see your OAuth credentials:
|
||||
|
||||
1. Copy your **Client ID**
|
||||
2. Copy your **Client Secret**
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Linear Connector
|
||||
|
||||
Add the Linear credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
LINEAR_CLIENT_ID=your_linear_client_id
|
||||
LINEAR_CLIENT_SECRET=your_linear_client_secret
|
||||
LINEAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/linear/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
|
|
@ -1,28 +1,6 @@
|
|||
{
|
||||
"title": "Connectors",
|
||||
"icon": "Unplug",
|
||||
"pages": [
|
||||
"google-drive",
|
||||
"gmail",
|
||||
"google-calendar",
|
||||
"dropbox",
|
||||
"notion",
|
||||
"slack",
|
||||
"discord",
|
||||
"jira",
|
||||
"linear",
|
||||
"microsoft-teams",
|
||||
"microsoft-onedrive",
|
||||
"confluence",
|
||||
"airtable",
|
||||
"clickup",
|
||||
"github",
|
||||
"baidu-search",
|
||||
"luma",
|
||||
"circleback",
|
||||
"elasticsearch",
|
||||
"bookstack",
|
||||
"obsidian"
|
||||
],
|
||||
"pages": ["native", "external", "deprecated"],
|
||||
"defaultOpen": false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,104 +0,0 @@
|
|||
---
|
||||
title: Microsoft OneDrive
|
||||
description: Connect your Microsoft OneDrive to SurfSense
|
||||
---
|
||||
|
||||
# Microsoft OneDrive OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up a Microsoft OneDrive OAuth integration for SurfSense using Azure App Registration.
|
||||
|
||||
<Callout type="info">
|
||||
Microsoft OneDrive and [Microsoft Teams](/docs/connectors/microsoft-teams) share the same Azure App Registration. If you have already created an app for Teams, you can reuse the same Client ID and Client Secret. Just make sure both redirect URIs are added (see Step 3).
|
||||
</Callout>
|
||||
|
||||
## Step 1: Access Azure App Registrations
|
||||
|
||||
1. Navigate to [portal.azure.com](https://portal.azure.com)
|
||||
2. In the search bar, type **"app reg"**
|
||||
3. Select **"App registrations"** from the Services results
|
||||
|
||||
## Step 2: Create New Registration
|
||||
|
||||
1. On the **App registrations** page, click **"+ New registration"**
|
||||
|
||||
## Step 3: Register the Application
|
||||
|
||||
Fill in the application details:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Name** | `SurfSense` |
|
||||
| **Supported account types** | Select **"Accounts in any organizational directory (Any Microsoft Entra ID tenant - Multitenant) and personal Microsoft accounts"** |
|
||||
| **Redirect URI** | Platform: `Web`, URI: `http://localhost:8000/api/v1/auth/onedrive/connector/callback` |
|
||||
|
||||
Click **"Register"**
|
||||
|
||||
After registration, add the Teams redirect URI as well (if you plan to use the Teams connector):
|
||||
|
||||
1. Go to **Authentication** in the left sidebar
|
||||
2. Under **Platform configurations** > **Web** > **Redirect URIs**, click **Add URI**
|
||||
3. Add: `http://localhost:8000/api/v1/auth/teams/connector/callback`
|
||||
4. Click **Save**
|
||||
|
||||
## Step 4: Get Application (Client) ID
|
||||
|
||||
After registration, you will be taken to the app's **Overview** page. Here you will find:
|
||||
|
||||
1. Copy the **Application (client) ID** - this is your Client ID
|
||||
2. Note the **Directory (tenant) ID** if needed
|
||||
|
||||
## Step 5: Create Client Secret
|
||||
|
||||
1. In the left sidebar under **Manage**, click **"Certificates & secrets"**
|
||||
2. Select the **"Client secrets"** tab
|
||||
3. Click **"+ New client secret"**
|
||||
4. Enter a description (e.g., `SurfSense`) and select an expiration period
|
||||
5. Click **"Add"**
|
||||
6. **Important**: Copy the secret **Value** immediately. It will not be shown again!
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your client secret publicly or include it in code repositories.
|
||||
</Callout>
|
||||
|
||||
## Step 6: Configure API Permissions
|
||||
|
||||
1. In the left sidebar under **Manage**, click **"API permissions"**
|
||||
2. Click **"+ Add a permission"**
|
||||
3. Select **"Microsoft Graph"**
|
||||
4. Select **"Delegated permissions"**
|
||||
5. Add the following permissions:
|
||||
|
||||
| Permission | Type | Description | Admin Consent |
|
||||
|------------|------|-------------|---------------|
|
||||
| `Files.Read.All` | Delegated | Read all files the user can access | No |
|
||||
| `Files.ReadWrite.All` | Delegated | Read and write all files the user can access | No |
|
||||
| `offline_access` | Delegated | Maintain access to data you have given it access to | No |
|
||||
| `User.Read` | Delegated | Sign in and read user profile | No |
|
||||
|
||||
6. Click **"Add permissions"**
|
||||
|
||||
<Callout type="warn">
|
||||
All four permissions listed above are required. The connector will not authenticate successfully if any are missing.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Microsoft OneDrive Connector
|
||||
|
||||
Add the Microsoft OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
MICROSOFT_CLIENT_ID=your_microsoft_client_id
|
||||
MICROSOFT_CLIENT_SECRET=your_microsoft_client_secret
|
||||
ONEDRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/onedrive/connector/callback
|
||||
```
|
||||
|
||||
<Callout type="info">
|
||||
The `MICROSOFT_CLIENT_ID` and `MICROSOFT_CLIENT_SECRET` are shared between the OneDrive and Teams connectors. You only need to set them once.
|
||||
</Callout>
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
64
surfsense_web/content/docs/connectors/native/google-maps.mdx
Normal file
64
surfsense_web/content/docs/connectors/native/google-maps.mdx
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
---
|
||||
title: Google Maps
|
||||
description: Scrape public Google Maps places and reviews
|
||||
---
|
||||
|
||||
The Google Maps scraper has two verbs: **scrape** for place data and **reviews** for a place's review feed.
|
||||
|
||||
## Scrape places
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/google_maps/scrape
|
||||
```
|
||||
|
||||
Give it search queries (optionally scoped by location), Google Maps URLs, or place IDs. Returns structured place items — name, address, category, phone, website, rating, review count, coordinates, and opening hours.
|
||||
|
||||
At least one of `search_queries`, `urls`, or `place_ids` is required.
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `search_queries` | — | Search terms, e.g. `"coffee shops"` (max 20 sources per call, combined with URLs and place IDs) |
|
||||
| `urls` | — | Place page (`/maps/place/...`) or search-results URLs |
|
||||
| `place_ids` | — | Known Google place IDs (`ChIJ...`) |
|
||||
| `location` | — | Scope for search queries, e.g. `"New York, USA"` |
|
||||
| `max_places` | `10` | Max places per search query (max 1000) |
|
||||
| `include_details` | `false` | Also fetch each place's detail page: opening hours, popular times, extra contact info (slower) |
|
||||
| `max_reviews` | `0` | Reviews to attach per place |
|
||||
| `max_images` | `0` | Images to attach per place |
|
||||
| `language` | `en` | Result language code |
|
||||
|
||||
```bash
|
||||
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/google_maps/scrape" \
|
||||
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"search_queries": ["specialty coffee"],
|
||||
"location": "Amsterdam, Netherlands",
|
||||
"max_places": 20,
|
||||
"include_details": true
|
||||
}'
|
||||
```
|
||||
|
||||
This verb is dual-metered: billed per returned place, plus per attached review when `max_reviews > 0`.
|
||||
|
||||
## Fetch reviews
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/google_maps/reviews
|
||||
```
|
||||
|
||||
Give it place URLs or place IDs; returns review items with author, text, star rating, like count, owner response, and timestamps.
|
||||
|
||||
At least one of `urls` or `place_ids` is required.
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `urls` / `place_ids` | — | Up to 20 places per call |
|
||||
| `max_reviews` | `20` | Max reviews per place (max 100,000) |
|
||||
| `sort_by` | `newest` | `newest`, `mostRelevant`, `highestRanking`, or `lowestRanking` |
|
||||
| `start_date` | — | Only reviews on/after this ISO date |
|
||||
| `language` | `en` | Review language code |
|
||||
|
||||
Billing is per returned review.
|
||||
|
||||
For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Google Maps** in your workspace.
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
---
|
||||
title: Google Search
|
||||
description: Search Google and get structured SERP results
|
||||
---
|
||||
|
||||
The Google Search scraper runs searches and returns structured SERP data: organic results (title, URL, description), related queries, people-also-ask questions, and any AI overview.
|
||||
|
||||
## Endpoint
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/google_search/scrape
|
||||
```
|
||||
|
||||
## Inputs
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `queries` | required | 1–20 search terms or full Google Search URLs — terms are searched, URLs are scraped as-is |
|
||||
| `max_pages_per_query` | `1` | Result pages to fetch per query (max 10) |
|
||||
| `country_code` | — | Two-letter country to search from, e.g. `us`, `fr` |
|
||||
| `language_code` | — | Result language code (blank = Google default) |
|
||||
| `site` | — | Restrict results to a single domain, e.g. `example.com` |
|
||||
|
||||
## Example
|
||||
|
||||
```bash
|
||||
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/google_search/scrape" \
|
||||
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"queries": ["open source notebooklm alternative"],
|
||||
"max_pages_per_query": 2,
|
||||
"country_code": "us"
|
||||
}'
|
||||
```
|
||||
|
||||
The response is `{ "items": [...] }` — one item per fetched SERP page. Billing is per SERP page.
|
||||
|
||||
For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Google Search → Scrape** in your workspace.
|
||||
91
surfsense_web/content/docs/connectors/native/index.mdx
Normal file
91
surfsense_web/content/docs/connectors/native/index.mdx
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
---
|
||||
title: Native Connectors
|
||||
description: SurfSense's built-in scraper APIs for Reddit, YouTube, Google Maps, Google Search, and the web
|
||||
---
|
||||
|
||||
import { Card, Cards } from 'fumadocs-ui/components/card';
|
||||
|
||||
Native connectors are SurfSense's own scraper APIs — built into the platform, no third-party account or OAuth app required. They pull structured, public data from the platforms below and return clean JSON.
|
||||
|
||||
<Cards>
|
||||
<Card
|
||||
title="Reddit"
|
||||
description="Posts, comments, subreddits, and users — by URL or search"
|
||||
href="/docs/connectors/native/reddit"
|
||||
/>
|
||||
<Card
|
||||
title="YouTube"
|
||||
description="Videos, channels, playlists, subtitles, and comments"
|
||||
href="/docs/connectors/native/youtube"
|
||||
/>
|
||||
<Card
|
||||
title="Google Maps"
|
||||
description="Places with details, ratings, photos, and reviews"
|
||||
href="/docs/connectors/native/google-maps"
|
||||
/>
|
||||
<Card
|
||||
title="Google Search"
|
||||
description="Structured SERPs: organic results, people-also-ask, AI overviews"
|
||||
href="/docs/connectors/native/google-search"
|
||||
/>
|
||||
<Card
|
||||
title="Web Crawl"
|
||||
description="Scrape any page or spider a whole site into clean markdown"
|
||||
href="/docs/connectors/native/web-crawl"
|
||||
/>
|
||||
</Cards>
|
||||
|
||||
## Three ways to use them
|
||||
|
||||
Every scraper is available through the same three doors:
|
||||
|
||||
1. **In chat** — the AI agent uses these scrapers as tools automatically. Ask "what is r/selfhosted saying about SurfSense?" and the agent runs the Reddit scraper for you.
|
||||
2. **API Playground** — open **API Playground** in your workspace sidebar, pick a scraper, fill in the form, and run it interactively. Great for exploring what a scraper returns before writing code.
|
||||
3. **REST API** — call the scrapers from your own code. Each one is a single `POST`:
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/{platform}/{verb}
|
||||
Authorization: Bearer <your-api-key>
|
||||
```
|
||||
|
||||
The playground's **API reference** section on every scraper page generates ready-to-paste snippets (cURL, Python, JavaScript, Go, and more) with your workspace ID already filled in, plus the full input and output JSON schemas.
|
||||
|
||||
## API keys
|
||||
|
||||
To call the REST API you need two things, both under **API Playground → API Keys**:
|
||||
|
||||
1. Toggle **API key access** on for the workspace.
|
||||
2. Create a personal API key and send it as an `Authorization: Bearer` header.
|
||||
|
||||
## Sync and async runs
|
||||
|
||||
By default a `POST` blocks until the scrape finishes and returns the results. For long scrapes, append `?mode=async` — you get a `202` with a `run_id` immediately, then:
|
||||
|
||||
- **Stream progress**: `GET .../scrapers/runs/{run_id}/events` (Server-Sent Events, ends with a `run.finished` event)
|
||||
- **Fetch the result**: `GET .../scrapers/runs/{run_id}`
|
||||
- **Cancel**: `POST .../scrapers/runs/{run_id}/cancel`
|
||||
|
||||
## Avoiding blocks with a proxy (self-hosted)
|
||||
|
||||
Scrapers make real requests to the target platforms, and heavy use from a single server IP will eventually get rate-limited or blocked. If you self-host, we recommend routing scraper traffic through a proxy — any HTTP proxy or rotating residential/datacenter gateway you already use works; SurfSense doesn't require a specific vendor.
|
||||
|
||||
Set it up in your backend `.env` and restart:
|
||||
|
||||
```bash
|
||||
# A single endpoint (user:pass@host:port), used for all scraper traffic
|
||||
PROXY_URL=http://username:password@proxy.example.com:8080
|
||||
|
||||
# Or a comma-separated pool that SurfSense rotates through per request.
|
||||
# Gateways that rotate server-side just need the single PROXY_URL above.
|
||||
PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port
|
||||
```
|
||||
|
||||
Leave both unset and requests go out directly from your server's IP — fine for light use. See the proxy section of `.env.example` for the full details, including `PROXY_PROVIDER` if your vendor has a built-in integration.
|
||||
|
||||
<Callout type="info">
|
||||
On SurfSense Cloud, proxying is already handled — nothing to configure.
|
||||
</Callout>
|
||||
|
||||
## Runs and pricing
|
||||
|
||||
Every run — from chat, the playground, or the API — is recorded under **API Playground → Runs** with its input, output, duration, and cost. Scrapers are metered per item returned (per post, video, comment, place, review, SERP page, or crawled page); the current rate is shown on each scraper's playground card, and pricing is returned by the capabilities API (`GET .../scrapers/capabilities`).
|
||||
5
surfsense_web/content/docs/connectors/native/meta.json
Normal file
5
surfsense_web/content/docs/connectors/native/meta.json
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"title": "Native Connectors",
|
||||
"pages": ["reddit", "youtube", "google-maps", "google-search", "web-crawl"],
|
||||
"defaultOpen": false
|
||||
}
|
||||
49
surfsense_web/content/docs/connectors/native/reddit.mdx
Normal file
49
surfsense_web/content/docs/connectors/native/reddit.mdx
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
---
|
||||
title: Reddit
|
||||
description: Scrape public Reddit posts, comments, subreddits, and users
|
||||
---
|
||||
|
||||
The Reddit scraper pulls structured public data from Reddit. Give it URLs (a post, a subreddit, a user page, or a search URL) and/or search terms, and it returns posts (title, body, score, comment count, subreddit, author), their comment trees, and community/user metadata.
|
||||
|
||||
## Endpoint
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/reddit/scrape
|
||||
```
|
||||
|
||||
## Inputs
|
||||
|
||||
At least one of `urls`, `search_queries`, or `community` is required.
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `urls` | — | Reddit URLs: a post, `/r/<subreddit>`, `/user/<name>`, or a search URL (max 20 sources per call, combined with queries) |
|
||||
| `search_queries` | — | Search terms to run on Reddit |
|
||||
| `community` | — | Subreddit name (without `r/`) to scope searches to; with no queries, its listing is scraped |
|
||||
| `sort` | `new` | `relevance`, `hot`, `top`, `new`, `rising`, or `comments` |
|
||||
| `time_filter` | — | Window for `top`/`controversial`: `hour`, `day`, `week`, `month`, `year`, `all` |
|
||||
| `max_items` | `10` | Max total items returned across all sources (hard cap 100) |
|
||||
| `max_posts` | `10` | Max posts per subreddit/user/search target |
|
||||
| `max_comments` | `10` | Max comments per post (`0` = none) |
|
||||
| `skip_comments` | `false` | Skip comment trees entirely (faster) |
|
||||
| `include_nsfw` | `true` | Include over-18 posts |
|
||||
| `post_date_limit` / `comment_date_limit` | — | ISO dates for incremental scrapes: only return newer content |
|
||||
|
||||
## Example
|
||||
|
||||
```bash
|
||||
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/reddit/scrape" \
|
||||
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"search_queries": ["self-hosted AI research agent"],
|
||||
"community": "selfhosted",
|
||||
"sort": "top",
|
||||
"time_filter": "month",
|
||||
"max_items": 25
|
||||
}'
|
||||
```
|
||||
|
||||
The response is `{ "items": [...] }` — one item per post, comment, community, or user. Billing is per returned item.
|
||||
|
||||
For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Reddit → Scrape** in your workspace.
|
||||
58
surfsense_web/content/docs/connectors/native/web-crawl.mdx
Normal file
58
surfsense_web/content/docs/connectors/native/web-crawl.mdx
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
---
|
||||
title: Web Crawl
|
||||
description: Scrape any page or crawl a whole site into clean markdown
|
||||
---
|
||||
|
||||
The web crawler fetches one or more pages — or spiders a whole site — and returns each page as clean markdown, plus metadata, every link with its anchor text, and harvested contact signals (emails, phone numbers, social profiles). JS-rendered pages are loaded in a real browser and auto-scrolled, so lazy-loaded listings are captured too.
|
||||
|
||||
## Endpoint
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/web/crawl
|
||||
```
|
||||
|
||||
## Inputs
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `startUrls` | required | 1–20 seed URLs |
|
||||
| `maxCrawlDepth` | `0` | Link-hops to follow from each seed. `0` = fetch only the seeds; `1` = also the pages they link to; up to 5. The spider stays on the seed's site |
|
||||
| `maxCrawlPages` | `10` | Total pages fetched per call, seeds included (max 200) |
|
||||
| `maxLength` | `50000` | Max characters of markdown kept per page |
|
||||
| `includeUrlPatterns` | — | Regexes a discovered link must match to be followed (empty = follow every same-site link) |
|
||||
| `excludeUrlPatterns` | — | Regexes that exclude links from being followed (wins over include) |
|
||||
|
||||
## Example
|
||||
|
||||
Scrape a single page:
|
||||
|
||||
```bash
|
||||
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/web/crawl" \
|
||||
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{ "startUrls": ["https://example.com/pricing"] }'
|
||||
```
|
||||
|
||||
Crawl a site's blog, two hops deep:
|
||||
|
||||
```bash
|
||||
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/web/crawl" \
|
||||
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"startUrls": ["https://example.com/blog"],
|
||||
"maxCrawlDepth": 2,
|
||||
"maxCrawlPages": 50,
|
||||
"includeUrlPatterns": ["/blog/"]
|
||||
}'
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
One item per fetched page (in crawl order) with its markdown, metadata (title, description), crawl provenance (depth, referrer), links classified as internal/external/social/email/tel, and page-level contacts. The response also includes a site-wide `contacts` summary that deduplicates every email, phone, and social profile found — `siteWide: true` marks header/footer values (the company's own contacts) versus page-local finds like individual team members.
|
||||
|
||||
Contact details often live on about/contact/privacy pages, so crawl with `maxCrawlDepth >= 1` to surface them — useful for lead generation and competitive intelligence.
|
||||
|
||||
Billing is per successfully fetched page.
|
||||
|
||||
For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Web → Crawl** in your workspace.
|
||||
54
surfsense_web/content/docs/connectors/native/youtube.mdx
Normal file
54
surfsense_web/content/docs/connectors/native/youtube.mdx
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
---
|
||||
title: YouTube
|
||||
description: Scrape public YouTube videos, channels, playlists, and comments
|
||||
---
|
||||
|
||||
The YouTube scraper has two verbs: **scrape** for video metadata (and optionally subtitles) and **comments** for a video's comment threads.
|
||||
|
||||
## Scrape videos
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/youtube/scrape
|
||||
```
|
||||
|
||||
Give it YouTube URLs (video, channel `/@handle`, playlist, shorts, or hashtag pages) and/or search queries. Returns structured video items — title, views, likes, publish date, channel info, description, and optionally subtitles.
|
||||
|
||||
At least one of `urls` or `search_queries` is required.
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `urls` | — | Video, channel, playlist, shorts, or hashtag URLs (max 20 sources per call, combined with queries) |
|
||||
| `search_queries` | — | Search terms; each returns up to `max_results` videos |
|
||||
| `max_results` | `10` | Max items per source and per content type (a channel's videos, shorts, and streams are capped independently; max 1000) |
|
||||
| `download_subtitles` | `false` | Also fetch each video's subtitle track (slower) |
|
||||
| `subtitles_language` | `en` | Subtitle language code |
|
||||
|
||||
```bash
|
||||
curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/youtube/scrape" \
|
||||
-H "Authorization: Bearer $SURFSENSE_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"urls": ["https://www.youtube.com/@surfsense"],
|
||||
"max_results": 20
|
||||
}'
|
||||
```
|
||||
|
||||
Billing is per returned video/short/stream.
|
||||
|
||||
## Fetch comments
|
||||
|
||||
```bash
|
||||
POST /api/v1/workspaces/{workspace_id}/scrapers/youtube/comments
|
||||
```
|
||||
|
||||
Give it video URLs; returns comment items with author, text, like count, reply relationships, and timestamps — useful for gauging sentiment on specific videos.
|
||||
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `urls` | required | 1–20 video URLs |
|
||||
| `max_comments` | `20` | Max items per video, counting top-level comments and replies (max 100,000) |
|
||||
| `sort_by` | `NEWEST_FIRST` | `TOP_COMMENTS` or `NEWEST_FIRST` |
|
||||
|
||||
Billing is per returned comment or reply.
|
||||
|
||||
For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → YouTube** in your workspace.
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
---
|
||||
title: Notion
|
||||
description: Connect your Notion workspaces to SurfSense
|
||||
---
|
||||
|
||||
# Notion OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up a Notion OAuth integration for SurfSense.
|
||||
|
||||
## Step 1: Access Notion Integrations
|
||||
|
||||
1. Navigate to [notion.so/profile/integrations](https://notion.so/profile/integrations)
|
||||
2. Click the **"New integration"** button
|
||||
|
||||
|
||||
|
||||
## Step 2: Configure New Integration
|
||||
|
||||
Fill in the integration details:
|
||||
|
||||
| Field | Value |
|
||||
|-------|-------|
|
||||
| **Integration Name** | `SurfSense` |
|
||||
| **Associated workspace** | Select your workspace |
|
||||
| **Type** | `Public` |
|
||||
| **Company name** | Your company name |
|
||||
| **Website** | Your website URL |
|
||||
| **Tagline** | Brief description |
|
||||
| **Privacy Policy URL** | Your privacy policy URL |
|
||||
| **Terms of Use URL** | Your terms of use URL |
|
||||
| **Email** | Your developer email |
|
||||
| **Logo** | Upload a 512x512 logo |
|
||||
|
||||
### OAuth Redirect URI
|
||||
|
||||
Under **OAuth domains & URIs**, set the **Redirect URI** to:
|
||||
|
||||
```
|
||||
http://localhost:8000/api/v1/auth/notion/connector/callback
|
||||
```
|
||||
|
||||
Click **Save** to create the integration.
|
||||
|
||||
|
||||
|
||||
## Step 3: Get OAuth Credentials & Configure Capabilities
|
||||
|
||||
After creating the integration, you'll see the configuration page with your credentials:
|
||||
|
||||
1. Copy your **OAuth Client ID**
|
||||
2. Copy your **OAuth Client Secret** (click Refresh if needed)
|
||||
|
||||
### Set Required Capabilities
|
||||
|
||||
Enable the following capabilities:
|
||||
|
||||
| Capability Type | Required Setting |
|
||||
|----------------|------------------|
|
||||
| **Content Capabilities** | Read content |
|
||||
| **Comment Capabilities** | Read comments |
|
||||
| **User Capabilities** | Read user information including email addresses |
|
||||
|
||||
Click **Save** to apply the capabilities.
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Limitations & Unsupported Content
|
||||
|
||||
Notion's API has limitations on certain block types that cannot be retrieved. SurfSense will automatically skip these unsupported blocks and continue syncing all other content.
|
||||
|
||||
### Unsupported Block Types
|
||||
|
||||
The following Notion features are **not accessible via the Notion API** and will be skipped during sync:
|
||||
|
||||
- **Transcription blocks** - Audio/video transcriptions from Notion AI
|
||||
- **AI blocks** - AI-generated content blocks
|
||||
|
||||
### Learn More
|
||||
|
||||
The Notion API only supports specific block types for retrieval. The official list of **supported block types** is documented in Notion's Block reference:
|
||||
|
||||
- **[Block Object Reference](https://developers.notion.com/reference/block)** - Official documentation listing all supported block types. Any block type not listed here (such as `transcription` and `ai_block`) is not accessible via the Notion API.
|
||||
|
||||
For additional information:
|
||||
- [Working with Page Content](https://developers.notion.com/docs/working-with-page-content) - Guide on how the Notion API handles page content
|
||||
- [Notion API Reference](https://developers.notion.com/reference) - Complete API documentation
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Notion Connector
|
||||
|
||||
Add the Notion credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
NOTION_OAUTH_CLIENT_ID=your_notion_client_id
|
||||
NOTION_OAUTH_CLIENT_SECRET=your_notion_client_secret
|
||||
NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
---
|
||||
title: Slack
|
||||
description: Connect your Slack workspace to SurfSense
|
||||
---
|
||||
|
||||
# Slack OAuth Integration Setup Guide
|
||||
|
||||
This guide walks you through setting up a Slack OAuth integration for SurfSense.
|
||||
|
||||
## Step 1: Create a New Slack App
|
||||
|
||||
1. Navigate to [api.slack.com/apps](https://api.slack.com/apps)
|
||||
2. Click **"Create New App"**
|
||||
3. Select **"From scratch"** to manually configure your app
|
||||
|
||||
|
||||
|
||||
## Step 2: Name App & Choose Workspace
|
||||
|
||||
1. Enter **App Name**: `SurfSense`
|
||||
2. Select the workspace to develop your app in
|
||||
3. Click **"Create App"**
|
||||
|
||||
<Callout type="warn">
|
||||
You won't be able to change the workspace later. The workspace will control the app even if you leave it.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
## Step 3: Get App Credentials
|
||||
|
||||
After creating the app, you'll be taken to the **Basic Information** page. Here you'll find your credentials:
|
||||
|
||||
1. Copy your **Client ID**
|
||||
2. Copy your **Client Secret** (click Show to reveal)
|
||||
|
||||
<Callout type="warn">
|
||||
Never share your app credentials publicly.
|
||||
</Callout>
|
||||
|
||||
|
||||
|
||||
## Step 4: Configure Redirect URLs
|
||||
|
||||
1. In the left sidebar, click **"OAuth & Permissions"**
|
||||
2. Scroll down to **Redirect URLs**
|
||||
3. Click **"Add New Redirect URL"**
|
||||
4. Enter: `https://localhost:8000/api/v1/auth/slack/connector/callback`
|
||||
5. Click **"Add"**, then **"Save URLs"**
|
||||
|
||||
|
||||
|
||||
## Step 5: Configure Bot Token Scopes
|
||||
|
||||
On the same **OAuth & Permissions** page, scroll to **Scopes** and add the following **Bot Token Scopes**:
|
||||
|
||||
| OAuth Scope | Description |
|
||||
|-------------|-------------|
|
||||
| `channels:history` | View messages and other content in public channels |
|
||||
| `channels:read` | View basic information about public channels |
|
||||
| `groups:history` | View messages and other content in private channels |
|
||||
| `groups:read` | View basic information about private channels |
|
||||
| `im:history` | View messages and other content in direct messages |
|
||||
| `mpim:history` | View messages and other content in group direct messages |
|
||||
| `users:read` | View people in a workspace |
|
||||
|
||||
Click **"Add an OAuth Scope"** to add each scope.
|
||||
|
||||
|
||||
|
||||
## Step 6: Enable Public Distribution
|
||||
|
||||
1. In the left sidebar, click **"Manage Distribution"**
|
||||
2. Under **Share Your App with Other Workspaces**, ensure distribution is enabled
|
||||
3. You can use the **"Add to Slack"** button or **Sharable URL** to install the app
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Running SurfSense with Slack Connector
|
||||
|
||||
Add the Slack credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)):
|
||||
|
||||
```bash
|
||||
SLACK_CLIENT_ID=your_slack_client_id
|
||||
SLACK_CLIENT_SECRET=your_slack_client_secret
|
||||
SLACK_REDIRECT_URI=http://localhost:8000/api/v1/auth/slack/connector/callback
|
||||
```
|
||||
|
||||
Then restart the services:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
---
|
||||
title: Web Crawler
|
||||
description: Crawl and index websites with SurfSense
|
||||
---
|
||||
|
||||
# Documentation in progress
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
---
|
||||
title: Docker Compose Development
|
||||
description: Building SurfSense from source using docker-compose.dev.yml
|
||||
---
|
||||
|
||||
If you're contributing to SurfSense and want to build from source, use `docker-compose.dev.yml` instead:
|
||||
|
||||
```bash
|
||||
cd SurfSense/docker
|
||||
docker compose -f docker-compose.dev.yml up --build
|
||||
```
|
||||
|
||||
This file builds the backend and frontend from your local source code (instead
|
||||
of pulling prebuilt images) and includes pgAdmin for database inspection at
|
||||
[http://localhost:5050](http://localhost:5050). It intentionally keeps raw
|
||||
frontend, backend, and zero-cache ports published for debugging. Use the
|
||||
production `docker-compose.yml` for the default Caddy single-origin setup.
|
||||
|
||||
## Dev-Only Environment Variables
|
||||
|
||||
The following `.env` variables are **only used by the dev compose file** (they have no effect on the production `docker-compose.yml`):
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `PGADMIN_PORT` | pgAdmin web UI port | `5050` |
|
||||
| `PGADMIN_DEFAULT_EMAIL` | pgAdmin login email | `admin@surfsense.com` |
|
||||
| `PGADMIN_DEFAULT_PASSWORD` | pgAdmin login password | `surfsense` |
|
||||
| `REDIS_PORT` | Exposed Redis port (internal-only in prod) | `6379` |
|
||||
| `AUTH_TYPE` | Runtime auth mode | `LOCAL` |
|
||||
| `ETL_SERVICE` | Runtime document parsing service | `DOCLING` |
|
||||
| `DEPLOYMENT_MODE` | Runtime deployment mode | `self-hosted` |
|
||||
| `ZERO_CACHE_PORT` | Exposed zero-cache port for debugging | `4848` |
|
||||
|
||||
In the production compose file, the frontend reads `AUTH_TYPE`, `ETL_SERVICE`,
|
||||
and `DEPLOYMENT_MODE` at request time. Browser API and Zero traffic are
|
||||
same-origin relative through bundled Caddy.
|
||||
Production Docker exposes only the bundled Caddy proxy by default; dev compose
|
||||
keeps direct service ports so contributors can inspect and restart individual
|
||||
services without going through the proxy.
|
||||
|
|
@ -1,419 +0,0 @@
|
|||
---
|
||||
title: Docker Compose
|
||||
description: Manual Docker Compose setup for SurfSense
|
||||
---
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
git clone https://github.com/MODSetter/SurfSense.git
|
||||
cd SurfSense/docker
|
||||
cp .env.example .env
|
||||
# Edit .env, at minimum set SECRET_KEY
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
After starting, access SurfSense at:
|
||||
|
||||
- **SurfSense**: [http://localhost:3929](http://localhost:3929)
|
||||
- **Backend API**: [http://localhost:3929/api/v1](http://localhost:3929/api/v1)
|
||||
- **Zero sync**: `ws://localhost:3929/zero`
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
All configuration lives in a single `docker/.env` file (or `surfsense/.env` if you used the install script). Copy `.env.example` to `.env` and edit the values you need.
|
||||
|
||||
### Required
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `SECRET_KEY` | JWT secret key. Generate with: `openssl rand -base64 32`. Auto-generated by the install script. |
|
||||
|
||||
### Core Settings
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `SURFSENSE_VERSION` | Image tag to deploy. Use `latest`, a clean version (e.g. `0.0.14`), or a specific build (e.g. `0.0.14.1`) | `latest` |
|
||||
| `SURFSENSE_VARIANT` | Backend image variant. Leave empty for CPU, set `cuda` for CUDA 12.8, or `cuda126` for CUDA 12.6. | *(empty)* |
|
||||
| `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` |
|
||||
| `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` |
|
||||
| `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` |
|
||||
| `TTS_SERVICE` | Text-to-speech provider for podcasts | `local/kokoro` |
|
||||
| `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` |
|
||||
| `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` |
|
||||
|
||||
### Image Variants
|
||||
|
||||
SurfSense publishes CPU and CUDA backend image variants. The frontend image is not variant-specific.
|
||||
|
||||
| Backend tag | Use case | `SURFSENSE_VARIANT` |
|
||||
|-------------|----------|---------------------|
|
||||
| `:latest` | CPU-only default | *(empty)* |
|
||||
| `:latest-cuda` | NVIDIA CUDA 12.8 backend image | `cuda` |
|
||||
| `:latest-cuda126` | NVIDIA CUDA 12.6 backend image for older driver stacks | `cuda126` |
|
||||
|
||||
All backend variants are published for `linux/amd64` and `linux/arm64`. CUDA on `linux/arm64` is best-effort.
|
||||
|
||||
<Callout type="info">
|
||||
GPU acceleration needs two settings: `SURFSENSE_VARIANT` selects the CUDA image, and `COMPOSE_FILE` enables the GPU device overlay. The host must have the NVIDIA Container Toolkit installed.
|
||||
</Callout>
|
||||
|
||||
### NVIDIA GPU Acceleration
|
||||
|
||||
For most NVIDIA systems, add these values to `.env` to use the CUDA 12.8 image:
|
||||
|
||||
```dotenv
|
||||
SURFSENSE_VARIANT=cuda
|
||||
COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml
|
||||
SURFSENSE_GPU_COUNT=1
|
||||
```
|
||||
|
||||
Use `SURFSENSE_VARIANT=cuda126` for older NVIDIA driver stacks or older GPUs that need the CUDA 12.6 fallback image.
|
||||
|
||||
On Windows, use `;` instead of `:` in `COMPOSE_FILE` inside `.env`:
|
||||
|
||||
```dotenv
|
||||
COMPOSE_FILE=docker-compose.yml;docker-compose.gpu.yml
|
||||
```
|
||||
|
||||
To switch variants later, edit `SURFSENSE_VARIANT` and `COMPOSE_FILE` in `.env`, then run:
|
||||
|
||||
```bash
|
||||
docker compose pull
|
||||
docker compose up -d --wait
|
||||
```
|
||||
|
||||
### Automatic Updates
|
||||
|
||||
Manual Docker Compose installs do not start Watchtower automatically. To enable external automatic updates, run Watchtower separately:
|
||||
|
||||
```bash
|
||||
docker run -d --name watchtower \
|
||||
--restart unless-stopped \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
nickfedor/watchtower \
|
||||
--label-enable \
|
||||
--interval 86400
|
||||
```
|
||||
|
||||
SurfSense containers are labeled for Watchtower, so `--label-enable` limits updates to the SurfSense services.
|
||||
|
||||
### Public URL and Ports
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `SURFSENSE_PUBLIC_URL` | Public origin used by the frontend, backend OAuth callbacks, and Zero browser URL | `http://localhost:3929` |
|
||||
| `SURFSENSE_SITE_ADDRESS` | Caddy site address. `:80` means local plain HTTP; a hostname enables automatic HTTPS | `:80` |
|
||||
| `LISTEN_HTTP_PORT` | Host port mapped to Caddy's HTTP listener | `3929` |
|
||||
| `LISTEN_HTTPS_PORT` | Host port mapped to Caddy's HTTPS listener for domain mode | `443` |
|
||||
|
||||
SurfSense includes Caddy by default. The `frontend`, `backend`, and
|
||||
`zero-cache` containers are internal-only in the production compose file; the
|
||||
browser reaches them through Caddy path routing.
|
||||
|
||||
### Custom Domain / Automatic HTTPS
|
||||
|
||||
For a real domain, point DNS at the Docker host and set:
|
||||
|
||||
```dotenv
|
||||
SURFSENSE_SITE_ADDRESS=surf.example.com
|
||||
LISTEN_HTTP_PORT=80
|
||||
LISTEN_HTTPS_PORT=443
|
||||
CERT_EMAIL=you@example.com
|
||||
SURFSENSE_PUBLIC_URL=https://surf.example.com
|
||||
```
|
||||
|
||||
Caddy will issue and renew Let's Encrypt certificates automatically. Ports 80
|
||||
and 443 must be reachable from the internet for the default HTTP-01 challenge.
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `CERT_EMAIL` | Optional ACME contact email |
|
||||
| `CERT_ACME_CA` | ACME directory URL; use Let's Encrypt staging when testing cert issuance |
|
||||
| `CERT_ACME_DNS` | DNS-01 challenge config; requires the custom Caddy build |
|
||||
| `TRUSTED_PROXIES` | CIDR ranges trusted for forwarded client IP headers |
|
||||
| `SURFSENSE_MAX_BODY_SIZE` | Upload limit enforced at the proxy |
|
||||
|
||||
### Bring Your Own Proxy
|
||||
|
||||
If you already run nginx, Traefik, Cloudflare Tunnel, or another ingress, you
|
||||
can comment out the `proxy` service and route traffic to the internal services
|
||||
with the same path contract:
|
||||
|
||||
| Public path | Upstream |
|
||||
|-------------|----------|
|
||||
| `/auth/*` | `backend:8000` |
|
||||
| `/api/v1/*` | `backend:8000` |
|
||||
| `/zero/*` | `zero-cache:4848` |
|
||||
| `/*` | `frontend:3000` |
|
||||
|
||||
Alternative proxies must preserve WebSocket upgrades for `/zero`, avoid
|
||||
buffering streaming responses, allow long-running requests, and support large
|
||||
uploads. For DNS-01 or wildcard certificates with Caddy, build
|
||||
`docker/proxy/Dockerfile` and set `CERT_ACME_DNS` for your DNS provider.
|
||||
|
||||
### Zero-cache (Real-Time Sync)
|
||||
|
||||
Defaults work out of the box. Change `ZERO_ADMIN_PASSWORD` for security in production.
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `ZERO_ADMIN_PASSWORD` | Password for the zero-cache admin UI and `/statz` endpoint | `surfsense-zero-admin` |
|
||||
| `ZERO_UPSTREAM_DB` | PostgreSQL connection URL for replication (must be a direct connection, not via pgbouncer) | *(built from DB_* vars)* |
|
||||
| `ZERO_CVR_DB` | PostgreSQL connection URL for client view records | *(built from DB_* vars)* |
|
||||
| `ZERO_CHANGE_DB` | PostgreSQL connection URL for replication log entries | *(built from DB_* vars)* |
|
||||
| `ZERO_APP_PUBLICATIONS` | PostgreSQL publication restricting which tables are replicated (created by migration 116, verified by the `migrations` service before `zero-cache` starts) | `zero_publication` |
|
||||
| `ZERO_NUM_SYNC_WORKERS` | Number of view-sync worker processes. Must be ≤ connection pool sizes | `4` |
|
||||
| `ZERO_UPSTREAM_MAX_CONNS` | Max connections to upstream PostgreSQL for mutations | `20` |
|
||||
| `ZERO_CVR_MAX_CONNS` | Max connections to the CVR database | `30` |
|
||||
|
||||
### Database
|
||||
|
||||
Defaults work out of the box. Change for security in production.
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `DB_USER` | PostgreSQL username | `surfsense` |
|
||||
| `DB_PASSWORD` | PostgreSQL password | `surfsense` |
|
||||
| `DB_NAME` | PostgreSQL database name | `surfsense` |
|
||||
| `DB_HOST` | PostgreSQL host | `db` |
|
||||
| `DB_PORT` | PostgreSQL port | `5432` |
|
||||
| `DB_SSLMODE` | SSL mode: `disable`, `require`, `verify-ca`, `verify-full` | `disable` |
|
||||
| `DATABASE_URL` | Full connection URL override. Use for managed databases (RDS, Supabase, etc.) | *(built from above)* |
|
||||
|
||||
### Authentication
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `GOOGLE_OAUTH_CLIENT_ID` | Google OAuth client ID (required if `AUTH_TYPE=GOOGLE`) |
|
||||
| `GOOGLE_OAUTH_CLIENT_SECRET` | Google OAuth client secret (required if `AUTH_TYPE=GOOGLE`) |
|
||||
|
||||
Create credentials at the [Google Cloud Console](https://console.cloud.google.com/apis/credentials).
|
||||
|
||||
### External API Keys
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `UNSTRUCTURED_API_KEY` | [Unstructured.io](https://unstructured.io/) API key (required if `ETL_SERVICE=UNSTRUCTURED`) |
|
||||
| `LLAMA_CLOUD_API_KEY` | [LlamaCloud](https://cloud.llamaindex.ai/) API key (required if `ETL_SERVICE=LLAMACLOUD`) |
|
||||
|
||||
### Connector OAuth Keys
|
||||
|
||||
Uncomment the connectors you want to use. Redirect URIs follow the single-origin
|
||||
pattern `${SURFSENSE_PUBLIC_URL}/api/v1/auth/<connector>/connector/callback`.
|
||||
For local Docker defaults, that means
|
||||
`http://localhost:3929/api/v1/auth/<connector>/connector/callback`.
|
||||
|
||||
| Connector | Variables |
|
||||
|-----------|-----------|
|
||||
| Google Drive / Gmail / Calendar | `GOOGLE_DRIVE_REDIRECT_URI`, `GOOGLE_GMAIL_REDIRECT_URI`, `GOOGLE_CALENDAR_REDIRECT_URI` |
|
||||
| Notion | `NOTION_CLIENT_ID`, `NOTION_CLIENT_SECRET`, `NOTION_REDIRECT_URI` |
|
||||
| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `SLACK_REDIRECT_URI` |
|
||||
| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `DISCORD_REDIRECT_URI` |
|
||||
| Atlassian (Jira & Confluence) | `ATLASSIAN_CLIENT_ID`, `ATLASSIAN_CLIENT_SECRET`, `JIRA_REDIRECT_URI`, `CONFLUENCE_REDIRECT_URI` |
|
||||
| Linear | `LINEAR_CLIENT_ID`, `LINEAR_CLIENT_SECRET`, `LINEAR_REDIRECT_URI` |
|
||||
| ClickUp | `CLICKUP_CLIENT_ID`, `CLICKUP_CLIENT_SECRET`, `CLICKUP_REDIRECT_URI` |
|
||||
| Airtable | `AIRTABLE_CLIENT_ID`, `AIRTABLE_CLIENT_SECRET`, `AIRTABLE_REDIRECT_URI` |
|
||||
| Microsoft (Teams & OneDrive) | `MICROSOFT_CLIENT_ID`, `MICROSOFT_CLIENT_SECRET`, `TEAMS_REDIRECT_URI`, `ONEDRIVE_REDIRECT_URI` |
|
||||
| Dropbox | `DROPBOX_APP_KEY`, `DROPBOX_APP_SECRET`, `DROPBOX_REDIRECT_URI` |
|
||||
|
||||
### Messaging Channels
|
||||
|
||||
Configure these in the same `docker/.env` file when you want users to chat with
|
||||
SurfSense from external apps. See [Messaging Channels](/docs/messaging-channels)
|
||||
for full setup.
|
||||
|
||||
| Channel | Variables |
|
||||
|---------|-----------|
|
||||
| Telegram | `TELEGRAM_SHARED_BOT_TOKEN`, `TELEGRAM_SHARED_BOT_USERNAME`, `TELEGRAM_WEBHOOK_SECRET`, `GATEWAY_BASE_URL`, `GATEWAY_TELEGRAM_INTAKE_MODE` |
|
||||
| WhatsApp | `GATEWAY_WHATSAPP_INTAKE_MODE`, `WHATSAPP_SHARED_BUSINESS_TOKEN`, `WHATSAPP_SHARED_PHONE_NUMBER_ID`, `WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER`, `WHATSAPP_SHARED_WABA_ID`, `WHATSAPP_WEBHOOK_VERIFY_TOKEN`, `WHATSAPP_WEBHOOK_APP_SECRET` |
|
||||
| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `GATEWAY_SLACK_ENABLED`, `GATEWAY_SLACK_SIGNING_SECRET`, `GATEWAY_SLACK_REDIRECT_URI` |
|
||||
| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `GATEWAY_DISCORD_ENABLED`, `GATEWAY_DISCORD_REDIRECT_URI` |
|
||||
|
||||
### Observability (optional)
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `LANGSMITH_TRACING` | Enable LangSmith tracing (`true` / `false`) |
|
||||
| `LANGSMITH_ENDPOINT` | LangSmith API endpoint |
|
||||
| `LANGSMITH_API_KEY` | LangSmith API key |
|
||||
| `LANGSMITH_PROJECT` | LangSmith project name |
|
||||
|
||||
### Advanced (optional)
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `SCHEDULE_CHECKER_INTERVAL` | How often to check for scheduled connector tasks (e.g. `5m`, `1h`) | `5m` |
|
||||
| `RERANKERS_ENABLED` | Enable document reranking for improved search | `FALSE` |
|
||||
| `RERANKERS_MODEL_NAME` | Reranker model name (e.g. `ms-marco-MiniLM-L-12-v2`) | |
|
||||
| `RERANKERS_MODEL_TYPE` | Reranker model type (e.g. `flashrank`) | |
|
||||
| `PAGES_LIMIT` | Max pages per user for ETL services | unlimited |
|
||||
|
||||
---
|
||||
|
||||
## Docker Services
|
||||
|
||||
| Service | Description |
|
||||
|---------|-------------|
|
||||
| `proxy` | Caddy reverse proxy; the only public ingress in production Docker |
|
||||
| `db` | PostgreSQL with pgvector extension |
|
||||
| `migrations` | Short-lived: runs `alembic upgrade head` and verifies `zero_publication`, then exits |
|
||||
| `redis` | Message broker for Celery |
|
||||
| `searxng` | Local privacy-respecting search backend |
|
||||
| `backend` | FastAPI application server |
|
||||
| `celery_worker` | Background task processing (document indexing, etc.) |
|
||||
| `celery_beat` | Periodic task scheduler (connector sync) |
|
||||
| `zero-cache` | Rocicorp Zero real-time sync (replicates Postgres to clients) |
|
||||
| `frontend` | Next.js web application, internal behind Caddy |
|
||||
|
||||
All services start automatically with `docker compose up -d`.
|
||||
|
||||
### How startup ordering works
|
||||
|
||||
Schema migrations run as a dedicated `migrations` service that exits 0 on
|
||||
success and non-zero on failure. Every other backend-image service gates on
|
||||
it via `condition: service_completed_successfully`:
|
||||
|
||||
```text
|
||||
db (healthy) ──▶ migrations (alembic upgrade head + verify zero_publication)
|
||||
│
|
||||
├── exit 0 ─▶ backend ──▶ frontend
|
||||
│ celery_worker
|
||||
│ celery_beat
|
||||
│ zero-cache ──▶ frontend
|
||||
│
|
||||
└── exit ≠ 0 ─▶ compose halts the rest of the stack
|
||||
```
|
||||
|
||||
This guarantees `zero-cache` only starts after `zero_publication` exists in
|
||||
Postgres. Before this design, a silent migration failure would leave
|
||||
`zero-cache` crash-looping with `Unknown or invalid publications. Specified:
|
||||
[zero_publication]. Found: []`.
|
||||
|
||||
### Readiness vs liveness
|
||||
|
||||
The backend exposes two endpoints:
|
||||
|
||||
- `GET /health`: lightweight liveness probe (always returns 200 if the
|
||||
process is up).
|
||||
- `GET /ready`: readiness probe that confirms `zero_publication` exists.
|
||||
Returns 503 if not. The compose `backend.healthcheck` uses `/ready` so the
|
||||
container only reports `healthy` once the schema is actually usable by
|
||||
zero-cache.
|
||||
|
||||
You can also monitor startup progress with `docker compose ps` (look for
|
||||
`(health: starting)` → `(healthy)`). The install script polls these states
|
||||
automatically and times out after 5 minutes if the stack does not converge.
|
||||
|
||||
---
|
||||
|
||||
## Useful Commands
|
||||
|
||||
```bash
|
||||
# View logs (all services)
|
||||
docker compose logs -f
|
||||
|
||||
# View logs for a specific service
|
||||
docker compose logs -f backend
|
||||
|
||||
# Stop all services
|
||||
docker compose down
|
||||
|
||||
# Restart a specific service
|
||||
docker compose restart backend
|
||||
|
||||
# Stop and remove all containers + volumes (destructive!)
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **Port already in use**: Change `LISTEN_HTTP_PORT` in `.env` and restart. In domain mode, use ports `80` and `443` so Caddy can complete certificate issuance.
|
||||
- **Permission errors on Linux**: You may need to prefix `docker` commands with `sudo`.
|
||||
- **Real-time updates not working**: Open DevTools → Console and check for WebSocket errors. In production Docker the expected URL is `${SURFSENSE_PUBLIC_URL}/zero`.
|
||||
- **Line ending issues on Windows**: Run `git config --global core.autocrlf true` before cloning.
|
||||
|
||||
### Migration service exited non-zero
|
||||
|
||||
The `migrations` service exits non-zero in two cases:
|
||||
|
||||
1. `alembic upgrade head` failed (timeout or SQL error).
|
||||
2. `alembic` succeeded but `zero_publication` is still missing from
|
||||
`pg_publication`.
|
||||
|
||||
Inspect the logs and the alembic state:
|
||||
|
||||
```bash
|
||||
docker compose logs migrations
|
||||
docker compose exec db psql -U surfsense -d surfsense \
|
||||
-c 'SELECT * FROM alembic_version;'
|
||||
docker compose exec db psql -U surfsense -d surfsense \
|
||||
-c 'SELECT pubname FROM pg_publication;'
|
||||
```
|
||||
|
||||
The default migration timeout is 900 seconds. Slow disks (Windows / WSL2)
|
||||
may need more. Set `MIGRATION_TIMEOUT` in `.env` to increase it.
|
||||
|
||||
### Zero-cache stuck on `Unknown or invalid publications`
|
||||
|
||||
Symptom (in `docker compose logs zero-cache`):
|
||||
|
||||
```text
|
||||
Error: Unknown or invalid publications. Specified: [zero_publication]. Found: []
|
||||
```
|
||||
|
||||
This means `zero-cache` started before `zero_publication` was created or the
|
||||
publication does not match SurfSense's canonical Zero shape. With the current
|
||||
compose files this should be impossible: the `migrations` service blocks
|
||||
`zero-cache` from starting and verifies the publication before exiting
|
||||
successfully. If you see it, your stack predates the fix or you brought up
|
||||
`zero-cache` manually with `docker compose up zero-cache` before the migrations
|
||||
service ran.
|
||||
|
||||
Recovery:
|
||||
|
||||
```bash
|
||||
docker compose down
|
||||
docker volume rm surfsense-zero-cache # wipe half-built SQLite replica
|
||||
docker compose up -d # migrations runs first, then zero-cache
|
||||
```
|
||||
|
||||
### Zero-cache crashes with `_zero.tableMetadata` errors
|
||||
|
||||
This indicates a half-initialized SQLite replica left behind by a previous
|
||||
crash. Zero's own event triggers and `ZERO_AUTO_RESET` handle schema and
|
||||
replication halts automatically. If the local SQLite replica is wedged, run the
|
||||
recovery one-liner above to wipe `surfsense-zero-cache`; zero-cache will
|
||||
re-sync from Postgres on the next start.
|
||||
|
||||
### Ensuring `wal_level = logical`
|
||||
|
||||
Logical replication is required by zero-cache. The bundled
|
||||
`docker/postgresql.conf` sets `wal_level = logical` automatically. If you
|
||||
swap in your own config or use a managed Postgres, confirm with:
|
||||
|
||||
```bash
|
||||
docker compose exec db psql -U surfsense -d surfsense \
|
||||
-c "SHOW wal_level;"
|
||||
```
|
||||
|
||||
### Using `docker-compose.deps-only.yml`
|
||||
|
||||
`docker-compose.deps-only.yml` runs only the dependencies (Postgres, Redis,
|
||||
SearXNG, zero-cache) on Docker while the backend and frontend run on the
|
||||
host. Because there is no backend container in this stack, there is no
|
||||
`migrations` service either, and you must run alembic on the host **before**
|
||||
bringing the stack up:
|
||||
|
||||
```bash
|
||||
cd surfsense_backend
|
||||
uv run alembic upgrade head
|
||||
cd ../docker
|
||||
docker compose -f docker-compose.deps-only.yml up -d
|
||||
```
|
||||
|
||||
If you skip the alembic step, `zero-cache` will crash-loop with `Unknown or
|
||||
invalid publications. Specified: [zero_publication]`.
|
||||
|
|
@ -1,36 +1,181 @@
|
|||
---
|
||||
title: Docker Installation
|
||||
description: Deploy SurfSense using Docker
|
||||
description: Run SurfSense with Docker in minutes
|
||||
---
|
||||
|
||||
import { Card, Cards } from 'fumadocs-ui/components/card';
|
||||
Docker is the recommended way to run SurfSense. Everything — database, backend, frontend, background workers, real-time sync, and a reverse proxy — comes pre-configured.
|
||||
|
||||
Choose your preferred Docker deployment method below.
|
||||
**Prerequisites:** [Docker Desktop](https://www.docker.com/products/docker-desktop/) (or Docker Engine with Compose) must be installed and running.
|
||||
|
||||
<Cards>
|
||||
<Card
|
||||
title="One-Line Install Script"
|
||||
description="One-command installation of SurfSense using Docker"
|
||||
href="/docs/docker-installation/install-script"
|
||||
/>
|
||||
<Card
|
||||
title="Docker Compose"
|
||||
description="Manual Docker Compose setup for SurfSense"
|
||||
href="/docs/docker-installation/docker-compose"
|
||||
/>
|
||||
<Card
|
||||
title="Updating"
|
||||
description="How to update your SurfSense Docker deployment"
|
||||
href="/docs/docker-installation/updating"
|
||||
/>
|
||||
<Card
|
||||
title="Docker Compose Development"
|
||||
description="Building SurfSense from source using docker-compose.dev.yml"
|
||||
href="/docs/docker-installation/dev-compose"
|
||||
/>
|
||||
<Card
|
||||
title="Migrate from the All-in-One Container"
|
||||
description="Migrate your data from the legacy all-in-one Docker image"
|
||||
href="/docs/docker-installation/migrate-from-allinone"
|
||||
/>
|
||||
</Cards>
|
||||
## Option 1: One-Line Install Script (Recommended)
|
||||
|
||||
**Linux/macOS:**
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
|
||||
```
|
||||
|
||||
**Windows (PowerShell):**
|
||||
|
||||
```powershell
|
||||
irm https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.ps1 | iex
|
||||
```
|
||||
|
||||
The script creates a `./surfsense/` directory with the compose files and a `.env`, generates a secret key, starts all services, and waits until they're healthy. It also:
|
||||
|
||||
- Detects NVIDIA GPUs and asks whether to use GPU acceleration, picking the compatible backend image automatically.
|
||||
- Asks whether to enable automatic daily updates via [Watchtower](https://github.com/nicholas-fedor/watchtower) (updates can download several GB in the background — pass `--no-watchtower` to skip).
|
||||
- Detects a legacy all-in-one installation and migrates its data automatically (see [Updating](/docs/docker-installation/updating#migrating-from-the-all-in-one-container)).
|
||||
|
||||
## Option 2: Manual Docker Compose
|
||||
|
||||
```bash
|
||||
git clone https://github.com/MODSetter/SurfSense.git
|
||||
cd SurfSense/docker
|
||||
cp .env.example .env
|
||||
# Edit .env — at minimum set SECRET_KEY (generate with: openssl rand -base64 32)
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## Access SurfSense
|
||||
|
||||
After starting, everything is served through the bundled Caddy reverse proxy on a single origin:
|
||||
|
||||
- **SurfSense**: [http://localhost:3929](http://localhost:3929)
|
||||
- **Backend API**: [http://localhost:3929/api/v1](http://localhost:3929/api/v1)
|
||||
- **Zero sync**: `ws://localhost:3929/zero`
|
||||
|
||||
Sign up with email/password (local auth is the default) and you're in.
|
||||
|
||||
## Configuration
|
||||
|
||||
All configuration lives in a single `.env` file — `surfsense/.env` if you used the install script, `docker/.env` if you cloned manually. The bundled `.env.example` documents every option inline with comments and working defaults; edit the values you need and restart:
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
The defaults give you local email/password auth, local document parsing with Docling (no API keys), and local TTS/STT. Common things you might change, all documented in the file:
|
||||
|
||||
- **Authentication** — switch to Google OAuth login.
|
||||
- **Document parsing** — switch to Unstructured or LlamaCloud (both need API keys).
|
||||
- **Connector credentials** — OAuth apps for [external connectors](/docs/connectors/external) that need them on self-hosted deployments.
|
||||
- **Messaging channels** — Telegram, WhatsApp, Slack, and Discord bots (see [Messaging Channels](/docs/messaging-channels)).
|
||||
|
||||
### NVIDIA GPU Acceleration
|
||||
|
||||
Add these values to `.env` to use the CUDA backend image (the host needs the NVIDIA Container Toolkit):
|
||||
|
||||
```dotenv
|
||||
SURFSENSE_VARIANT=cuda
|
||||
COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml
|
||||
SURFSENSE_GPU_COUNT=1
|
||||
```
|
||||
|
||||
Use `SURFSENSE_VARIANT=cuda126` for older NVIDIA driver stacks. On Windows, use `;` instead of `:` in `COMPOSE_FILE`. Then apply:
|
||||
|
||||
```bash
|
||||
docker compose pull && docker compose up -d --wait
|
||||
```
|
||||
|
||||
### Custom Domain / Automatic HTTPS
|
||||
|
||||
Point DNS at your Docker host and set in `.env`:
|
||||
|
||||
```dotenv
|
||||
SURFSENSE_SITE_ADDRESS=surf.example.com
|
||||
LISTEN_HTTP_PORT=80
|
||||
LISTEN_HTTPS_PORT=443
|
||||
CERT_EMAIL=you@example.com
|
||||
SURFSENSE_PUBLIC_URL=https://surf.example.com
|
||||
```
|
||||
|
||||
Then `docker compose up -d --wait`. Caddy issues and renews Let's Encrypt certificates automatically — ports 80 and 443 must be reachable from the internet.
|
||||
|
||||
### Bring Your Own Proxy
|
||||
|
||||
If you already run nginx, Traefik, Cloudflare Tunnel, or another ingress, comment out the `proxy` service and route traffic to the internal services with the same path contract:
|
||||
|
||||
| Public path | Upstream |
|
||||
|-------------|----------|
|
||||
| `/auth/*` | `backend:8000` |
|
||||
| `/api/v1/*` | `backend:8000` |
|
||||
| `/zero/*` | `zero-cache:4848` |
|
||||
| `/*` | `frontend:3000` |
|
||||
|
||||
Your proxy must preserve WebSocket upgrades for `/zero`, avoid buffering streaming responses, allow long-running requests, and support large uploads.
|
||||
|
||||
## What's Running
|
||||
|
||||
| Service | Description |
|
||||
|---------|-------------|
|
||||
| `proxy` | Caddy reverse proxy — the only public ingress |
|
||||
| `db` | PostgreSQL with pgvector |
|
||||
| `migrations` | Short-lived: runs database migrations, then exits |
|
||||
| `redis` | Message broker for background tasks |
|
||||
| `backend` | FastAPI application server |
|
||||
| `celery_worker` | Background task processing (document indexing, etc.) |
|
||||
| `celery_beat` | Periodic task scheduler |
|
||||
| `zero-cache` | Real-time sync (replicates Postgres to browsers) |
|
||||
| `frontend` | Next.js web application |
|
||||
|
||||
Migrations run first on every startup; everything else waits for them to succeed, so `docker compose up -d` after an update is always safe. Monitor startup with `docker compose ps` — services go from `(health: starting)` to `(healthy)`.
|
||||
|
||||
## Useful Commands
|
||||
|
||||
```bash
|
||||
# View logs (all services, or one)
|
||||
docker compose logs -f
|
||||
docker compose logs -f backend
|
||||
|
||||
# Restart a service
|
||||
docker compose restart backend
|
||||
|
||||
# Stop all services
|
||||
docker compose down
|
||||
|
||||
# Stop and remove containers + volumes (destroys your data!)
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
To update SurfSense, see [Updating](/docs/docker-installation/updating).
|
||||
|
||||
## Building from Source (Contributors)
|
||||
|
||||
If you're contributing to SurfSense and want to build the images from your local checkout, use the dev compose file:
|
||||
|
||||
```bash
|
||||
cd SurfSense/docker
|
||||
docker compose -f docker-compose.dev.yml up --build
|
||||
```
|
||||
|
||||
It builds the backend and frontend from source, publishes raw service ports for debugging, and includes pgAdmin at [http://localhost:5050](http://localhost:5050). There's also a `docker-compose.deps-only.yml` that runs just the dependencies (Postgres, Redis, zero-cache) in Docker while you run the backend and frontend natively — see [Manual Installation](/docs/manual-installation#alternative-let-docker-manage-all-dependencies).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **Port already in use**: Change `LISTEN_HTTP_PORT` in `.env` and restart. In domain mode, ports 80/443 are required for certificate issuance.
|
||||
- **Permission errors on Linux**: Prefix `docker` commands with `sudo`, or add your user to the `docker` group.
|
||||
- **Real-time updates not working**: Open DevTools → Console and check for WebSocket errors. The expected URL is `${SURFSENSE_PUBLIC_URL}/zero`.
|
||||
- **Line ending issues on Windows**: Run `git config --global core.autocrlf true` before cloning.
|
||||
|
||||
### Migration service exited non-zero
|
||||
|
||||
The stack halts if migrations fail. Inspect what happened:
|
||||
|
||||
```bash
|
||||
docker compose logs migrations
|
||||
```
|
||||
|
||||
Slow disks (Windows / WSL2) may hit the default 900-second migration timeout — set `MIGRATION_TIMEOUT` in `.env` to increase it.
|
||||
|
||||
### Zero-cache stuck on `Unknown or invalid publications`
|
||||
|
||||
This means zero-cache started before migrations created its publication — usually only possible if you started services individually. Recover with:
|
||||
|
||||
```bash
|
||||
docker compose down
|
||||
docker volume rm surfsense-zero-cache # wipe the half-built replica
|
||||
docker compose up -d # migrations run first, then zero-cache
|
||||
```
|
||||
|
||||
The same recovery fixes `_zero.tableMetadata` crashes (a half-initialized replica left behind by a previous crash).
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
---
|
||||
title: One-Line Install Script
|
||||
description: One-command installation of SurfSense using Docker
|
||||
---
|
||||
|
||||
Downloads the compose files, generates a `SECRET_KEY`, starts all services with `docker compose up -d --wait`, and starts [Watchtower](https://github.com/nicholas-fedor/watchtower) as an external updater for automatic daily updates.
|
||||
|
||||
**Prerequisites:** [Docker Desktop](https://www.docker.com/products/docker-desktop/) must be installed and running.
|
||||
|
||||
### For Linux/macOS users:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
|
||||
```
|
||||
|
||||
### For Windows users (PowerShell):
|
||||
|
||||
```powershell
|
||||
irm https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.ps1 | iex
|
||||
```
|
||||
|
||||
This creates a `./surfsense/` directory with `docker-compose.yml`, `docker-compose.gpu.yml`, and `.env`, then runs `docker compose up -d --wait`.
|
||||
|
||||
If an NVIDIA GPU and NVIDIA Container Toolkit are detected, the installer asks whether to use GPU acceleration and chooses the compatible backend image automatically. Non-interactive installs default to CPU unless you pass an explicit flag.
|
||||
|
||||
Interactive installs also ask whether to enable automatic daily updates with Watchtower, noting that updates may download several GB in the background.
|
||||
|
||||
### GPU options
|
||||
|
||||
Linux/macOS:
|
||||
|
||||
```bash
|
||||
# CUDA 12.8
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --variant=cuda
|
||||
|
||||
# CUDA 12.6 fallback for older driver stacks
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --variant=cuda126
|
||||
|
||||
# Reserve all available GPUs
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --gpu --gpu-count=all
|
||||
```
|
||||
|
||||
PowerShell:
|
||||
|
||||
```powershell
|
||||
# Save the script locally first when passing PowerShell parameters.
|
||||
.\install.ps1 -Variant cuda
|
||||
.\install.ps1 -Variant cuda126 -GpuCount all
|
||||
```
|
||||
|
||||
The installer writes the same `.env` settings you would configure manually: `SURFSENSE_VARIANT` selects the backend image and `COMPOSE_FILE` enables the GPU overlay.
|
||||
|
||||
To skip Watchtower (e.g. in production where you manage updates yourself, or to avoid large background image downloads):
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --no-watchtower
|
||||
```
|
||||
|
||||
To customise the check interval (default 24h), use `--watchtower-interval=SECONDS`.
|
||||
|
||||
Manual updates use the same compose state stored in `.env`, so GPU overlays and variants are preserved:
|
||||
|
||||
```bash
|
||||
cd surfsense
|
||||
docker compose pull
|
||||
docker compose up -d --wait
|
||||
```
|
||||
|
||||
If Watchtower is enabled, it preserves the running image variant tag automatically. Because SurfSense images are large, use `--no-watchtower` when you prefer to manage update timing yourself.
|
||||
|
||||
---
|
||||
|
||||
## Access SurfSense
|
||||
|
||||
After starting, access SurfSense at:
|
||||
|
||||
- **SurfSense**: [http://localhost:3929](http://localhost:3929)
|
||||
- **Backend API**: [http://localhost:3929/api/v1](http://localhost:3929/api/v1)
|
||||
- **Zero sync**: `ws://localhost:3929/zero`
|
||||
|
||||
The installer uses the bundled Caddy reverse proxy by default. The backend and
|
||||
zero-cache containers are not published on separate host ports in the production
|
||||
stack.
|
||||
|
||||
For a custom domain, edit `surfsense/.env` after installation:
|
||||
|
||||
```dotenv
|
||||
SURFSENSE_SITE_ADDRESS=surf.example.com
|
||||
LISTEN_HTTP_PORT=80
|
||||
LISTEN_HTTPS_PORT=443
|
||||
CERT_EMAIL=you@example.com
|
||||
SURFSENSE_PUBLIC_URL=https://surf.example.com
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```bash
|
||||
cd surfsense
|
||||
docker compose up -d --wait
|
||||
```
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"title": "Docker Installation",
|
||||
"pages": ["install-script", "docker-compose", "updating", "dev-compose", "migrate-from-allinone"],
|
||||
"pages": ["updating"],
|
||||
"icon": "Container",
|
||||
"defaultOpen": false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,114 +0,0 @@
|
|||
---
|
||||
title: Migrate from the All-in-One Container
|
||||
description: How to migrate your data from the legacy surfsense all-in-one Docker image to the current multi-container setup
|
||||
---
|
||||
|
||||
The original SurfSense all-in-one image (`ghcr.io/modsetter/surfsense:latest`, run via `docker-compose.quickstart.yml`) stored all data — PostgreSQL, Redis, and configuration — in a single Docker volume named `surfsense-data`. The current setup uses separate named volumes and has upgraded PostgreSQL from **version 14 to 17**.
|
||||
|
||||
Because PostgreSQL data files are not compatible between major versions, a **logical dump and restore** is required. This is a one-time migration.
|
||||
|
||||
<Callout type="warn">
|
||||
This guide only applies to users who ran the legacy `docker-compose.quickstart.yml` (the all-in-one `surfsense` container). If you were already using `docker/docker-compose.yml`, you do not need to migrate.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
## Option A — One command (recommended)
|
||||
|
||||
`install.sh` detects the legacy `surfsense-data` volume and handles the full migration automatically — no separate migration script needed. Just run the same install command you would use for a fresh install:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
|
||||
```
|
||||
|
||||
**What it does automatically:**
|
||||
|
||||
1. Downloads all SurfSense files (including `migrate-database.sh`) into `./surfsense/`
|
||||
2. Detects the `surfsense-data` volume and enters migration mode
|
||||
3. Stops the old all-in-one container if it is still running
|
||||
4. Starts a temporary PostgreSQL 14 container and dumps your database
|
||||
5. Recovers your `SECRET_KEY` from the old volume
|
||||
6. Starts PostgreSQL 17, restores the dump, runs a smoke test
|
||||
7. Starts all services
|
||||
|
||||
Your original `surfsense-data` volume is **never deleted** — you remove it manually after verifying.
|
||||
|
||||
### After it completes
|
||||
|
||||
1. Open [http://localhost:3000](http://localhost:3000) and confirm your data is intact.
|
||||
2. Once satisfied, remove the old volume (irreversible):
|
||||
```bash
|
||||
docker volume rm surfsense-data
|
||||
```
|
||||
3. Delete the dump file once you no longer need it as a backup:
|
||||
```bash
|
||||
rm ./surfsense_migration_backup.sql
|
||||
```
|
||||
|
||||
### If the migration fails mid-way
|
||||
|
||||
The dump file is saved to `./surfsense_migration_backup.sql` as a checkpoint. Simply re-run `install.sh` — it will detect the existing dump and skip straight to the restore step without re-extracting.
|
||||
|
||||
---
|
||||
|
||||
## Option B — Manual migration script (custom credentials)
|
||||
|
||||
If you launched the old all-in-one container with custom database credentials (`POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` environment variables), the automatic path will use wrong credentials. Run `migrate-database.sh` manually first:
|
||||
|
||||
```bash
|
||||
# 1. Extract data with your custom credentials
|
||||
bash ./surfsense/scripts/migrate-database.sh --db-user myuser --db-password mypass --db-name mydb
|
||||
|
||||
# 2. Install and restore (detects the dump automatically)
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
|
||||
```
|
||||
|
||||
Or download and run if you haven't run `install.sh` yet:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/migrate-database.sh -o migrate-database.sh
|
||||
bash migrate-database.sh --db-user myuser --db-password mypass --db-name mydb
|
||||
```
|
||||
|
||||
### Migration script options
|
||||
|
||||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--db-user USER` | Old PostgreSQL username | `surfsense` |
|
||||
| `--db-password PASS` | Old PostgreSQL password | `surfsense` |
|
||||
| `--db-name NAME` | Old PostgreSQL database | `surfsense` |
|
||||
| `--yes` / `-y` | Skip confirmation prompts (used automatically by `install.sh`) | — |
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### `install.sh` runs normally with a blank database (no migration happened)
|
||||
|
||||
The legacy volume was not detected. Confirm it exists:
|
||||
|
||||
```bash
|
||||
docker volume ls | grep surfsense-data
|
||||
```
|
||||
|
||||
If it doesn't appear, the old container may have used a different volume name. Check with:
|
||||
|
||||
```bash
|
||||
docker volume ls | grep -i surfsense
|
||||
```
|
||||
|
||||
### Extraction fails with permission errors
|
||||
|
||||
The script detects the UID of the data files and runs the temporary PG14 container as that user. If you see permission errors in `./surfsense-migration.log`, run `migrate-database.sh` manually and check the log for details.
|
||||
|
||||
### Cannot find `/data/.secret_key`
|
||||
|
||||
The all-in-one entrypoint always writes the key to `/data/.secret_key` unless you explicitly set `SECRET_KEY=` as an environment variable. If the key is missing, the migration script auto-generates a new one (with a warning). You can update it manually in `./surfsense/.env` afterwards. Note that a new key invalidates all existing browser sessions — users will need to log in again.
|
||||
|
||||
### Restore errors after re-running `install.sh`
|
||||
|
||||
If `surfsense-postgres` volume already exists from a previous partial run, remove it before retrying:
|
||||
|
||||
```bash
|
||||
docker volume rm surfsense-postgres
|
||||
```
|
||||
|
|
@ -3,11 +3,20 @@ title: Updating
|
|||
description: How to update your SurfSense Docker deployment
|
||||
---
|
||||
|
||||
## Watchtower Daemon (recommended)
|
||||
## Manual Update
|
||||
|
||||
Auto-updates every 24 hours. If you used the [install script](/docs/docker-installation/install-script), Watchtower is already running. No extra setup needed.
|
||||
```bash
|
||||
cd surfsense # or SurfSense/docker if you cloned manually
|
||||
docker compose pull && docker compose up -d
|
||||
```
|
||||
|
||||
For [manual Docker Compose](/docs/docker-installation/docker-compose) installs, start Watchtower separately:
|
||||
Database migrations are applied automatically on every startup. GPU overlays and image variants set in `.env` are preserved.
|
||||
|
||||
## Automatic Updates with Watchtower
|
||||
|
||||
Auto-updates every 24 hours. If you used the [install script](/docs/docker-installation) and enabled updates, Watchtower is already running — no extra setup needed.
|
||||
|
||||
For manual Docker Compose installs, start Watchtower separately:
|
||||
|
||||
```bash
|
||||
docker run -d --name watchtower \
|
||||
|
|
@ -18,7 +27,9 @@ docker run -d --name watchtower \
|
|||
--interval 86400
|
||||
```
|
||||
|
||||
## Watchtower One-Time Update
|
||||
SurfSense containers are labeled for Watchtower, so `--label-enable` limits updates to the SurfSense services.
|
||||
|
||||
For a one-time update instead of a daemon:
|
||||
|
||||
```bash
|
||||
docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
|
||||
|
|
@ -30,21 +41,32 @@ docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
|
|||
Use `nickfedor/watchtower`. The original `containrrr/watchtower` is no longer maintained and may fail with newer Docker versions.
|
||||
</Callout>
|
||||
|
||||
## Manual Update
|
||||
|
||||
```bash
|
||||
cd surfsense # or SurfSense/docker if you cloned manually
|
||||
docker compose pull && docker compose up -d
|
||||
```
|
||||
|
||||
Database migrations are applied automatically on every startup.
|
||||
|
||||
---
|
||||
|
||||
## Migrating from the All-in-One Container
|
||||
|
||||
<Callout type="warn">
|
||||
If you were previously using `docker-compose.quickstart.yml` (the legacy all-in-one `surfsense` container), your data lives in a `surfsense-data` volume and requires a **one-time migration** before switching to the current setup. PostgreSQL has been upgraded from version 14 to 17, so a simple volume swap will not work.
|
||||
If you previously ran the legacy all-in-one image (`ghcr.io/modsetter/surfsense:latest` via `docker-compose.quickstart.yml`), your data lives in a single `surfsense-data` volume and PostgreSQL has since been upgraded from version 14 to 17 — so a simple volume swap won't work. A one-time dump and restore is required, and the install script does it for you.
|
||||
|
||||
See the full step-by-step guide: [Migrate from the All-in-One Container](/docs/docker-installation/migrate-from-allinone).
|
||||
Just run the normal install command:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
|
||||
```
|
||||
|
||||
It detects the legacy `surfsense-data` volume, stops the old container, dumps your database with a temporary PostgreSQL 14 container, recovers your `SECRET_KEY`, restores everything into PostgreSQL 17, and starts the new stack. Your original volume is **never deleted** — the dump is also saved to `./surfsense_migration_backup.sql` as a checkpoint, so if anything fails mid-way you can simply re-run the script.
|
||||
|
||||
After it completes:
|
||||
|
||||
1. Open SurfSense and confirm your data is intact.
|
||||
2. Remove the old volume (irreversible): `docker volume rm surfsense-data`
|
||||
3. Delete the dump file once you no longer need it: `rm ./surfsense_migration_backup.sql`
|
||||
|
||||
<Callout type="info">
|
||||
If your old container used custom database credentials (`POSTGRES_USER` / `POSTGRES_PASSWORD` / `POSTGRES_DB`), run the migration script manually first with your credentials, then run the installer:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/migrate-database.sh -o migrate-database.sh
|
||||
bash migrate-database.sh --db-user myuser --db-password mypass --db-name mydb
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash
|
||||
```
|
||||
</Callout>
|
||||
|
|
|
|||
|
|
@ -57,10 +57,14 @@ If running the frontend outside Docker (e.g. `pnpm dev`), you need:
|
|||
-e ZERO_REPLICA_FILE=/data/zero.db \
|
||||
-e ZERO_ADMIN_PASSWORD=surfsense-zero-admin \
|
||||
-e ZERO_APP_PUBLICATIONS=zero_publication \
|
||||
-e ZERO_NUM_SYNC_WORKERS=4 \
|
||||
-e ZERO_UPSTREAM_MAX_CONNS=20 \
|
||||
-e ZERO_CVR_MAX_CONNS=30 \
|
||||
-e ZERO_QUERY_URL="http://host.docker.internal:3000/api/zero/query" \
|
||||
-e ZERO_MUTATE_URL="http://host.docker.internal:3000/api/zero/mutate" \
|
||||
-e ZERO_QUERY_FORWARD_COOKIES=true \
|
||||
-v surfsense-zero-cache:/data \
|
||||
rocicorp/zero:1.4.0
|
||||
rocicorp/zero:1.6.0
|
||||
```
|
||||
|
||||
Run `uv run alembic upgrade head` from `surfsense_backend/` **before** starting this container so the `zero_publication` exists.
|
||||
|
|
|
|||
|
|
@ -5,47 +5,39 @@ icon: BookOpen
|
|||
---
|
||||
|
||||
import { Card, Cards } from 'fumadocs-ui/components/card';
|
||||
import { ClipboardCheck, Download, Container, Wrench, Cable, BookOpen, FlaskConical, Heart, MessageCircle, Cpu } from 'lucide-react';
|
||||
import { Download, Container, Wrench, Cable, BookOpen, FlaskConical, Heart, MessageCircle, Cpu } from 'lucide-react';
|
||||
|
||||
Welcome to **SurfSense's Documentation!** Here, you'll find everything you need to get the most out of SurfSense. Dive in to explore how SurfSense can be your AI-powered research companion.
|
||||
Welcome to **SurfSense's Documentation!** Here, you'll find everything you need to get the most out of SurfSense.
|
||||
|
||||
## Getting Started
|
||||
|
||||
The fastest way to run SurfSense is the [Docker installation](/docs/docker-installation), one command and you're up. If you want full control over each component (or want to contribute), use the [manual installation](/docs/manual-installation).
|
||||
|
||||
<Cards>
|
||||
<Card
|
||||
icon={<ClipboardCheck />}
|
||||
title="Prerequisites"
|
||||
description="Required setup before installing SurfSense"
|
||||
href="/docs/prerequisites"
|
||||
/>
|
||||
<Card
|
||||
icon={<Download />}
|
||||
title="Installation"
|
||||
description="Choose your installation method"
|
||||
href="/docs/installation"
|
||||
/>
|
||||
<Card
|
||||
icon={<Container />}
|
||||
title="Docker Installation"
|
||||
description="Deploy SurfSense with Docker Compose"
|
||||
description="One command, all dependencies pre-configured (recommended)"
|
||||
href="/docs/docker-installation"
|
||||
/>
|
||||
<Card
|
||||
icon={<Wrench />}
|
||||
title="Manual Installation"
|
||||
description="Set up SurfSense manually from source"
|
||||
description="Set up SurfSense from source, component by component"
|
||||
href="/docs/manual-installation"
|
||||
/>
|
||||
<Card
|
||||
icon={<Cpu />}
|
||||
title="Local Models"
|
||||
description="Connect local model servers"
|
||||
href="/docs/local-models"
|
||||
/>
|
||||
<Card
|
||||
icon={<Cable />}
|
||||
title="Connectors"
|
||||
description="Integrate with third-party services"
|
||||
description="Built-in scraper APIs plus Notion, Slack, Google, Jira, and more"
|
||||
href="/docs/connectors"
|
||||
/>
|
||||
<Card
|
||||
icon={<Cpu />}
|
||||
title="Local Models"
|
||||
description="Connect local model servers like Ollama and LM Studio"
|
||||
href="/docs/local-models"
|
||||
/>
|
||||
<Card
|
||||
icon={<MessageCircle />}
|
||||
title="Messaging Channels"
|
||||
|
|
|
|||
|
|
@ -1,56 +1,45 @@
|
|||
---
|
||||
title: Manual Installation
|
||||
description: Setting up SurfSense manually for customized deployments
|
||||
description: Set up SurfSense from source, component by component
|
||||
icon: Wrench
|
||||
---
|
||||
|
||||
# Manual Installation (Preferred)
|
||||
This guide sets up SurfSense without Docker (except for zero-cache, which is simplest to run as a container). Choose this path if you want to contribute to SurfSense or need full control over each component. If you just want to run SurfSense, the [Docker installation](/docs/docker-installation) is much faster.
|
||||
|
||||
This guide provides step-by-step instructions for setting up SurfSense without Docker. This approach gives you more control over the installation process and allows for customization of the environment.
|
||||
## What You'll Need
|
||||
|
||||
## Prerequisites
|
||||
- **Python 3.12+** — backend runtime
|
||||
- **Node.js 20+** and **pnpm** — frontend runtime
|
||||
- **PostgreSQL 14+** with the **pgvector** extension — database
|
||||
- **Redis** — message broker for background tasks
|
||||
- **Docker** — to run zero-cache (the real-time sync server)
|
||||
- **Git** — to clone the repository
|
||||
- **uv** — Python package manager ([install instructions](https://docs.astral.sh/uv/getting-started/installation/))
|
||||
|
||||
Before beginning the manual installation, ensure you have the following installed and configured:
|
||||
Clone the repository first:
|
||||
|
||||
### Required Software
|
||||
- **Python 3.12+** - Backend runtime environment
|
||||
- **Node.js 20+** - Frontend runtime environment
|
||||
- **PostgreSQL 14+** - Database server (must be configured with `wal_level = logical` for [Zero real-time sync](/docs/how-to/zero-sync))
|
||||
- **PGVector** - PostgreSQL extension for vector similarity search
|
||||
- **Redis** - Message broker for Celery task queue
|
||||
- **Zero-cache** - Rocicorp Zero real-time sync server (run via Docker; see [Zero-Cache Setup](#zero-cache-setup) below)
|
||||
- **Docker** - Required to run zero-cache (the simplest way; the Postgres + Redis can be installed natively)
|
||||
- **Git** - Version control (to clone the repository)
|
||||
```bash
|
||||
git clone https://github.com/MODSetter/SurfSense.git
|
||||
cd SurfSense
|
||||
```
|
||||
|
||||
### Required Services & API Keys
|
||||
## Decisions to Make Up Front
|
||||
|
||||
Complete all the [setup steps](/docs), including:
|
||||
**Authentication.** SurfSense supports local email/password auth (the default, no setup needed) or Google OAuth login. For Google login you need an OAuth client from the [Google Cloud Console](https://console.cloud.google.com/apis/credentials) — the [Google connectors guide](/docs/connectors/external/google) walks through creating one.
|
||||
|
||||
- **Authentication Setup** (choose one):
|
||||
- Google OAuth credentials (for `AUTH_TYPE=GOOGLE`)
|
||||
- Local authentication setup (for `AUTH_TYPE=LOCAL`)
|
||||
- **File Processing ETL Service** (choose one):
|
||||
- Unstructured.io API key (Supports 34+ formats)
|
||||
- LlamaCloud API key (enhanced parsing, supports 50+ formats)
|
||||
- Docling (local processing, no API key required, supports PDF, Office docs, images, HTML, CSV)
|
||||
- **Other API keys** as needed for your use case
|
||||
**Document parsing (ETL).** SurfSense converts uploaded files with one of three services:
|
||||
|
||||
- **Docling** (default) — runs locally, no API key, privacy-friendly. Supports PDF, Office docs, images, HTML, CSV.
|
||||
- **Unstructured** — needs an API key from [Unstructured Platform](https://platform.unstructured.io/). Supports 34+ formats.
|
||||
- **LlamaCloud** — needs an API key from [LlamaCloud](https://cloud.llamaindex.ai/). Supports 50+ formats.
|
||||
|
||||
You only need one. Docling is the easiest way to start.
|
||||
|
||||
## Backend Setup
|
||||
|
||||
The backend is the core of SurfSense. Follow these steps to set it up:
|
||||
### 1. Configure the Environment
|
||||
|
||||
### Optional: Messaging Channels
|
||||
|
||||
SurfSense can expose the same backend agent through Telegram, WhatsApp, Slack,
|
||||
and Discord. For manual installs, configure the relevant channel variables in
|
||||
`surfsense_backend/.env`.
|
||||
|
||||
See [Messaging Channels](/docs/messaging-channels) for the channel-specific
|
||||
setup guides.
|
||||
|
||||
### 1. Environment Configuration
|
||||
|
||||
First, create and configure your environment variables by copying the example file:
|
||||
Copy the example environment file:
|
||||
|
||||
**Linux/macOS:**
|
||||
|
||||
|
|
@ -59,13 +48,6 @@ cd surfsense_backend
|
|||
cp .env.example .env
|
||||
```
|
||||
|
||||
**Windows (Command Prompt):**
|
||||
|
||||
```cmd
|
||||
cd surfsense_backend
|
||||
copy .env.example .env
|
||||
```
|
||||
|
||||
**Windows (PowerShell):**
|
||||
|
||||
```powershell
|
||||
|
|
@ -73,157 +55,18 @@ cd surfsense_backend
|
|||
Copy-Item -Path .env.example -Destination .env
|
||||
```
|
||||
|
||||
Edit the `.env` file and set the following variables:
|
||||
|
||||
| ENV VARIABLE | DESCRIPTION |
|
||||
| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| DATABASE_URL | PostgreSQL connection string (e.g., `postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense`) |
|
||||
| SECRET_KEY | JWT Secret key for authentication (should be a secure random string) |
|
||||
| NEXT_FRONTEND_URL | URL where your frontend application is hosted (e.g., `http://localhost:3000`) |
|
||||
| BACKEND_URL | (Optional) Public URL of the backend for OAuth callbacks (e.g., `https://api.yourdomain.com`). Required when running behind a reverse proxy with HTTPS. Used to set correct OAuth redirect URLs and secure cookies. |
|
||||
| AUTH_TYPE | Authentication method: `GOOGLE` for OAuth with Google, `LOCAL` for email/password authentication |
|
||||
| GOOGLE_OAUTH_CLIENT_ID | (Optional) Client ID from Google Cloud Console (required if AUTH_TYPE=GOOGLE) |
|
||||
| GOOGLE_OAUTH_CLIENT_SECRET | (Optional) Client secret from Google Cloud Console (required if AUTH_TYPE=GOOGLE) |
|
||||
| EMBEDDING_MODEL | Name of the embedding model (e.g., `sentence-transformers/all-MiniLM-L6-v2`, `openai://text-embedding-ada-002`) |
|
||||
| RERANKERS_ENABLED | (Optional) Enable or disable document reranking for improved search results (e.g., `TRUE` or `FALSE`, default: `FALSE`) |
|
||||
| RERANKERS_MODEL_NAME | Name of the reranker model (e.g., `ms-marco-MiniLM-L-12-v2`) (required if RERANKERS_ENABLED=TRUE) |
|
||||
| RERANKERS_MODEL_TYPE | Type of reranker model (e.g., `flashrank`) (required if RERANKERS_ENABLED=TRUE) |
|
||||
| TTS_SERVICE | Text-to-Speech API provider for Podcasts (e.g., `local/kokoro`, `openai/tts-1`). See [supported providers](https://docs.litellm.ai/docs/text_to_speech#supported-providers) |
|
||||
| TTS_SERVICE_API_KEY | (Optional if local) API key for the Text-to-Speech service |
|
||||
| TTS_SERVICE_API_BASE | (Optional) Custom API base URL for the Text-to-Speech service |
|
||||
| STT_SERVICE | Speech-to-Text API provider for Audio Files (e.g., `local/base`, `openai/whisper-1`). See [supported providers](https://docs.litellm.ai/docs/audio_transcription#supported-providers) |
|
||||
| STT_SERVICE_API_KEY | (Optional if local) API key for the Speech-to-Text service |
|
||||
| STT_SERVICE_API_BASE | (Optional) Custom API base URL for the Speech-to-Text service |
|
||||
| ETL_SERVICE | Document parsing service: `UNSTRUCTURED` (supports 34+ formats), `LLAMACLOUD` (supports 50+ formats including legacy document types), or `DOCLING` (local processing, supports PDF, Office docs, images, HTML, CSV) |
|
||||
| UNSTRUCTURED_API_KEY | API key for Unstructured.io service for document parsing (required if ETL_SERVICE=UNSTRUCTURED) |
|
||||
| LLAMA_CLOUD_API_KEY | API key for LlamaCloud service for document parsing (required if ETL_SERVICE=LLAMACLOUD) |
|
||||
| REDIS_URL | Single Redis connection URL for the Celery broker, result backend, and app cache (e.g., `redis://localhost:6379/0`). Optionally override per use with `CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND`, or `REDIS_APP_URL`. |
|
||||
| SCHEDULE_CHECKER_INTERVAL | (Optional) How often to check for scheduled connector tasks. Format: `<number><unit>` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`) |
|
||||
| REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) |
|
||||
| PAGES_LIMIT | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) |
|
||||
|
||||
**Google Connector OAuth Configuration:**
|
||||
| ENV VARIABLE | DESCRIPTION |
|
||||
| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| GOOGLE_CALENDAR_REDIRECT_URI | (Optional) Redirect URI for Google Calendar connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/calendar/connector/callback`) |
|
||||
| GOOGLE_GMAIL_REDIRECT_URI | (Optional) Redirect URI for Gmail connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/gmail/connector/callback`) |
|
||||
| GOOGLE_DRIVE_REDIRECT_URI | (Optional) Redirect URI for Google Drive connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/drive/connector/callback`) |
|
||||
|
||||
**Connector OAuth Configurations (Optional):**
|
||||
|
||||
| ENV VARIABLE | DESCRIPTION |
|
||||
| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| AIRTABLE_CLIENT_ID | (Optional) Airtable OAuth client ID from [Airtable Developer Hub](https://airtable.com/create/oauth) |
|
||||
| AIRTABLE_CLIENT_SECRET | (Optional) Airtable OAuth client secret |
|
||||
| AIRTABLE_REDIRECT_URI | (Optional) Redirect URI for Airtable connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/airtable/connector/callback`) |
|
||||
| CLICKUP_CLIENT_ID | (Optional) ClickUp OAuth client ID |
|
||||
| CLICKUP_CLIENT_SECRET | (Optional) ClickUp OAuth client secret |
|
||||
| CLICKUP_REDIRECT_URI | (Optional) Redirect URI for ClickUp connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/clickup/connector/callback`) |
|
||||
| DISCORD_CLIENT_ID | (Optional) Discord OAuth client ID |
|
||||
| DISCORD_CLIENT_SECRET | (Optional) Discord OAuth client secret |
|
||||
| DISCORD_REDIRECT_URI | (Optional) Redirect URI for Discord connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/discord/connector/callback`) |
|
||||
| DISCORD_BOT_TOKEN | (Optional) Discord bot token from Developer Portal |
|
||||
| ATLASSIAN_CLIENT_ID | (Optional) Atlassian OAuth client ID (for Jira and Confluence) |
|
||||
| ATLASSIAN_CLIENT_SECRET | (Optional) Atlassian OAuth client secret |
|
||||
| JIRA_REDIRECT_URI | (Optional) Redirect URI for Jira connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/jira/connector/callback`) |
|
||||
| CONFLUENCE_REDIRECT_URI | (Optional) Redirect URI for Confluence connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/confluence/connector/callback`) |
|
||||
| LINEAR_CLIENT_ID | (Optional) Linear OAuth client ID |
|
||||
| LINEAR_CLIENT_SECRET | (Optional) Linear OAuth client secret |
|
||||
| LINEAR_REDIRECT_URI | (Optional) Redirect URI for Linear connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/linear/connector/callback`) |
|
||||
| NOTION_CLIENT_ID | (Optional) Notion OAuth client ID |
|
||||
| NOTION_CLIENT_SECRET | (Optional) Notion OAuth client secret |
|
||||
| NOTION_REDIRECT_URI | (Optional) Redirect URI for Notion connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/notion/connector/callback`) |
|
||||
| SLACK_CLIENT_ID | (Optional) Slack OAuth client ID |
|
||||
| SLACK_CLIENT_SECRET | (Optional) Slack OAuth client secret |
|
||||
| SLACK_REDIRECT_URI | (Optional) Redirect URI for Slack connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/slack/connector/callback`) |
|
||||
| MICROSOFT_CLIENT_ID | (Optional) Microsoft OAuth client ID (shared for Teams and OneDrive) |
|
||||
| MICROSOFT_CLIENT_SECRET | (Optional) Microsoft OAuth client secret (shared for Teams and OneDrive) |
|
||||
| TEAMS_REDIRECT_URI | (Optional) Redirect URI for Teams connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/teams/connector/callback`) |
|
||||
| ONEDRIVE_REDIRECT_URI | (Optional) Redirect URI for OneDrive connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/onedrive/connector/callback`) |
|
||||
| DROPBOX_APP_KEY | (Optional) Dropbox OAuth app key |
|
||||
| DROPBOX_APP_SECRET | (Optional) Dropbox OAuth app secret |
|
||||
| DROPBOX_REDIRECT_URI | (Optional) Redirect URI for Dropbox connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/dropbox/connector/callback`) |
|
||||
|
||||
**(Optional) Backend LangSmith Observability:**
|
||||
| ENV VARIABLE | DESCRIPTION |
|
||||
|--------------|-------------|
|
||||
| LANGSMITH_TRACING | Enable LangSmith tracing (e.g., `true`) |
|
||||
| LANGSMITH_ENDPOINT | LangSmith API endpoint (e.g., `https://api.smith.langchain.com`) |
|
||||
| LANGSMITH_API_KEY | Your LangSmith API key |
|
||||
| LANGSMITH_PROJECT | LangSmith project name (e.g., `surfsense`) |
|
||||
|
||||
**(Optional) Uvicorn Server Configuration**
|
||||
| ENV VARIABLE | DESCRIPTION | DEFAULT VALUE |
|
||||
|------------------------------|---------------------------------------------|---------------|
|
||||
| UVICORN_HOST | Host address to bind the server | 0.0.0.0 |
|
||||
| UVICORN_PORT | Port to run the backend API | 8000 |
|
||||
| UVICORN_LOG_LEVEL | Logging level (e.g., info, debug, warning) | info |
|
||||
| UVICORN_PROXY_HEADERS | Enable/disable proxy headers | false |
|
||||
| UVICORN_FORWARDED_ALLOW_IPS | Comma-separated list of allowed IPs | 127.0.0.1 |
|
||||
| UVICORN_WORKERS | Number of worker processes | 1 |
|
||||
| UVICORN_ACCESS_LOG | Enable/disable access log (true/false) | true |
|
||||
| UVICORN_LOOP | Event loop implementation | auto |
|
||||
| UVICORN_HTTP | HTTP protocol implementation | auto |
|
||||
| UVICORN_WS | WebSocket protocol implementation | auto |
|
||||
| UVICORN_LIFESPAN | Lifespan implementation | auto |
|
||||
| UVICORN_LOG_CONFIG | Path to logging config file or empty string | |
|
||||
| UVICORN_SERVER_HEADER | Enable/disable Server header | true |
|
||||
| UVICORN_DATE_HEADER | Enable/disable Date header | true |
|
||||
| UVICORN_LIMIT_CONCURRENCY | Max concurrent connections | |
|
||||
| UVICORN_LIMIT_MAX_REQUESTS | Max requests before worker restart | |
|
||||
| UVICORN_TIMEOUT_KEEP_ALIVE | Keep-alive timeout (seconds) | 5 |
|
||||
| UVICORN_TIMEOUT_NOTIFY | Worker shutdown notification timeout (sec) | 30 |
|
||||
| UVICORN_SSL_KEYFILE | Path to SSL key file | |
|
||||
| UVICORN_SSL_CERTFILE | Path to SSL certificate file | |
|
||||
| UVICORN_SSL_KEYFILE_PASSWORD | Password for SSL key file | |
|
||||
| UVICORN_SSL_VERSION | SSL version | |
|
||||
| UVICORN_SSL_CERT_REQS | SSL certificate requirements | |
|
||||
| UVICORN_SSL_CA_CERTS | Path to CA certificates file | |
|
||||
| UVICORN_SSL_CIPHERS | SSL ciphers | |
|
||||
| UVICORN_HEADERS | Comma-separated list of headers | |
|
||||
| UVICORN_USE_COLORS | Enable/disable colored logs | true |
|
||||
| UVICORN_UDS | Unix domain socket path | |
|
||||
| UVICORN_FD | File descriptor to bind to | |
|
||||
| UVICORN_ROOT_PATH | Root path for the application | |
|
||||
|
||||
Refer to the `.env.example` file for all available Uvicorn options and their usage. Uncomment and set in your `.env` file as needed.
|
||||
|
||||
For more details, see the [Uvicorn documentation](https://www.uvicorn.org/#command-line-options).
|
||||
`.env.example` is the source of truth for configuration — every variable is documented inline with comments and sensible defaults. At minimum, set your PostgreSQL connection string and a JWT secret key (generate one with `openssl rand -base64 32`). Everything else — auth type, ETL service, embeddings, TTS/STT, connector credentials — is optional and explained in the file itself.
|
||||
|
||||
### 2. Install Dependencies
|
||||
|
||||
Install the backend dependencies using `uv`:
|
||||
|
||||
**Linux/macOS:**
|
||||
|
||||
```bash
|
||||
# Install uv if you don't have it
|
||||
curl -fsSL https://astral.sh/uv/install.sh | bash
|
||||
|
||||
# Install dependencies
|
||||
uv sync
|
||||
```
|
||||
|
||||
**Windows (PowerShell):**
|
||||
|
||||
```powershell
|
||||
# Install uv if you don't have it
|
||||
iwr -useb https://astral.sh/uv/install.ps1 | iex
|
||||
|
||||
# Install dependencies
|
||||
uv sync
|
||||
```
|
||||
|
||||
**Windows (Command Prompt):**
|
||||
|
||||
```cmd
|
||||
# Install dependencies with uv (after installing uv)
|
||||
# From surfsense_backend/
|
||||
uv sync
|
||||
```
|
||||
|
||||
### 3. Configure PostgreSQL for Zero Sync
|
||||
|
||||
SurfSense uses [Rocicorp Zero](https://zero.rocicorp.dev/) for real-time data synchronization (notifications, document status, chat comments, indexing progress). Zero replicates data from PostgreSQL via **logical replication**, which requires a one-time PostgreSQL configuration change.
|
||||
SurfSense uses [Rocicorp Zero](https://zero.rocicorp.dev/) for real-time updates (notifications, document status, chat comments, indexing progress). Zero replicates data from PostgreSQL via **logical replication**, which requires a one-time PostgreSQL configuration change.
|
||||
|
||||
Edit your `postgresql.conf` (typical locations: `/etc/postgresql/<version>/main/postgresql.conf` on Linux, `/usr/local/var/postgres/postgresql.conf` on macOS via Homebrew, `C:\Program Files\PostgreSQL\<version>\data\postgresql.conf` on Windows) and set:
|
||||
|
||||
|
|
@ -260,7 +103,7 @@ psql -U postgres -d surfsense -c "SHOW wal_level;"
|
|||
# Should return: logical
|
||||
```
|
||||
|
||||
**Managed databases (RDS, Supabase, Cloud SQL, etc.):** Enable logical replication via your provider's parameter group (e.g. `rds.logical_replication=1` on RDS) and grant your database user the `REPLICATION` privilege:
|
||||
**Managed databases (RDS, Supabase, Cloud SQL, etc.):** enable logical replication via your provider's parameter group (e.g. `rds.logical_replication=1` on RDS) and grant your database user the `REPLICATION` privilege:
|
||||
|
||||
```sql
|
||||
ALTER USER surfsense WITH REPLICATION;
|
||||
|
|
@ -269,26 +112,13 @@ GRANT CREATE ON DATABASE surfsense TO surfsense;
|
|||
|
||||
### 4. Run Database Migrations
|
||||
|
||||
Before starting the backend, run Alembic migrations. This creates the schema **and** the `zero_publication` that zero-cache needs to start. Skipping this step will cause zero-cache to crash-loop with `Unknown or invalid publications. Specified: [zero_publication]`.
|
||||
|
||||
**If using uv:**
|
||||
This creates the schema **and** the `zero_publication` that zero-cache needs to start. Skipping this step will cause zero-cache to crash-loop with `Unknown or invalid publications. Specified: [zero_publication]`.
|
||||
|
||||
```bash
|
||||
# From surfsense_backend/
|
||||
uv run alembic upgrade head
|
||||
```
|
||||
|
||||
**If using pip/venv:**
|
||||
|
||||
```bash
|
||||
# Activate virtual environment first
|
||||
source .venv/bin/activate # Linux/macOS
|
||||
# OR
|
||||
.venv\Scripts\activate # Windows
|
||||
|
||||
alembic upgrade head
|
||||
```
|
||||
|
||||
Verify the publication was created:
|
||||
|
||||
```bash
|
||||
|
|
@ -296,182 +126,87 @@ psql -U postgres -d surfsense -c "SELECT pubname FROM pg_publication;"
|
|||
# Should include: zero_publication
|
||||
```
|
||||
|
||||
### 5. Start Redis Server
|
||||
|
||||
Redis is required for Celery task queue. Start the Redis server:
|
||||
### 5. Start Redis
|
||||
|
||||
**Linux:**
|
||||
|
||||
```bash
|
||||
# Start Redis server
|
||||
sudo systemctl start redis
|
||||
|
||||
# Or if using Redis installed via package manager
|
||||
# or run directly
|
||||
redis-server
|
||||
```
|
||||
|
||||
**macOS:**
|
||||
**macOS (Homebrew):**
|
||||
|
||||
```bash
|
||||
# If installed via Homebrew
|
||||
brew services start redis
|
||||
|
||||
# Or run directly
|
||||
redis-server
|
||||
```
|
||||
|
||||
**Windows:**
|
||||
**Windows — run Redis in Docker (easiest):**
|
||||
|
||||
```powershell
|
||||
# Option 1: If using Redis on Windows (via WSL or Windows port)
|
||||
redis-server
|
||||
|
||||
# Option 2: If installed as a Windows service
|
||||
net start Redis
|
||||
```
|
||||
|
||||
**Alternative for Windows - Run Redis in Docker:**
|
||||
|
||||
If you have Docker Desktop installed, you can run Redis in a container:
|
||||
|
||||
```powershell
|
||||
# Pull and run Redis container
|
||||
docker run -d --name redis -p 6379:6379 redis:latest
|
||||
|
||||
# To stop Redis
|
||||
docker stop redis
|
||||
|
||||
# To start Redis again
|
||||
docker start redis
|
||||
|
||||
# To remove Redis container
|
||||
docker rm -f redis
|
||||
```
|
||||
|
||||
Verify Redis is running by connecting to it:
|
||||
Verify it's running:
|
||||
|
||||
```bash
|
||||
redis-cli ping
|
||||
# Should return: PONG
|
||||
```
|
||||
|
||||
### 6. Start Celery Worker
|
||||
### 6. Start the Celery Worker
|
||||
|
||||
In a new terminal window, start the Celery worker to handle background tasks. For external chat surfaces, Celery only runs maintenance tasks; agent turns run inside the FastAPI process.
|
||||
In a new terminal, start the worker that handles background tasks (document indexing, connector syncs):
|
||||
|
||||
**If using uv:**
|
||||
**Linux/macOS:**
|
||||
|
||||
```bash
|
||||
# Make sure you're in the surfsense_backend directory
|
||||
cd surfsense_backend
|
||||
|
||||
# Start Celery worker (consume default, connectors, and external chat maintenance queues)
|
||||
DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}"
|
||||
uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway"
|
||||
```
|
||||
|
||||
**If using pip/venv:**
|
||||
**Windows (PowerShell):**
|
||||
|
||||
```bash
|
||||
# Make sure you're in the surfsense_backend directory
|
||||
```powershell
|
||||
cd surfsense_backend
|
||||
|
||||
# Activate virtual environment
|
||||
source .venv/bin/activate # Linux/macOS
|
||||
# OR
|
||||
.venv\Scripts\activate # Windows
|
||||
|
||||
# Start Celery worker (consume default, connectors, and external chat maintenance queues)
|
||||
DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}"
|
||||
celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway"
|
||||
uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="surfsense,surfsense.connectors,surfsense.gateway"
|
||||
```
|
||||
|
||||
**Optional: Start Flower for monitoring Celery tasks:**
|
||||
|
||||
In another terminal window:
|
||||
Optionally, run [Flower](https://flower.readthedocs.io/) in another terminal to monitor tasks at [http://localhost:5555](http://localhost:5555):
|
||||
|
||||
```bash
|
||||
# If using uv
|
||||
uv run celery -A celery_worker.celery_app flower --port=5555
|
||||
|
||||
# If using pip/venv (activate venv first)
|
||||
celery -A celery_worker.celery_app flower --port=5555
|
||||
```
|
||||
|
||||
Access Flower at [http://localhost:5555](http://localhost:5555) to monitor your Celery tasks.
|
||||
|
||||
### 7. Start Celery Beat (Scheduler)
|
||||
|
||||
In another new terminal window, start Celery Beat to enable periodic tasks (like scheduled connector indexing):
|
||||
|
||||
**If using uv:**
|
||||
In another terminal, start the scheduler that triggers periodic tasks (like scheduled connector syncs). Without it, scheduled tasks won't run.
|
||||
|
||||
```bash
|
||||
# Make sure you're in the surfsense_backend directory
|
||||
cd surfsense_backend
|
||||
|
||||
# Start Celery Beat
|
||||
uv run celery -A celery_worker.celery_app beat --loglevel=info
|
||||
```
|
||||
|
||||
**If using pip/venv:**
|
||||
|
||||
```bash
|
||||
# Make sure you're in the surfsense_backend directory
|
||||
cd surfsense_backend
|
||||
|
||||
# Activate virtual environment
|
||||
source .venv/bin/activate # Linux/macOS
|
||||
# OR
|
||||
.venv\Scripts\activate # Windows
|
||||
|
||||
# Start Celery Beat
|
||||
celery -A celery_worker.celery_app beat --loglevel=info
|
||||
```
|
||||
|
||||
**Important**: Celery Beat is required for the periodic indexing functionality to work. Without it, scheduled connector tasks won't run automatically. The schedule interval can be configured using the `SCHEDULE_CHECKER_INTERVAL` environment variable.
|
||||
|
||||
### 8. Run the Backend
|
||||
|
||||
Start the backend server:
|
||||
|
||||
**If using uv:**
|
||||
|
||||
```bash
|
||||
# Run without hot reloading
|
||||
# From surfsense_backend/
|
||||
uv run main.py
|
||||
|
||||
# Or with hot reloading for development
|
||||
uv run main.py --reload
|
||||
```
|
||||
|
||||
**If using pip/venv:**
|
||||
|
||||
```bash
|
||||
# Activate virtual environment if not already activated
|
||||
source .venv/bin/activate # Linux/macOS
|
||||
# OR
|
||||
.venv\Scripts\activate # Windows
|
||||
|
||||
# Run without hot reloading
|
||||
python main.py
|
||||
|
||||
# Or with hot reloading for development
|
||||
python main.py --reload
|
||||
```
|
||||
|
||||
If everything is set up correctly, you should see output indicating the server is running on `http://localhost:8000`.
|
||||
You should see the server running on `http://localhost:8000`.
|
||||
|
||||
## Zero-Cache Setup
|
||||
|
||||
**zero-cache** is the Rocicorp Zero server that sits between PostgreSQL and the browser. It streams real-time updates (notifications, document indexing status, chat comments, collaboration indicators) to all connected clients via WebSocket. The frontend connects to it on startup. Without zero-cache running, you will not see live updates and many parts of the UI will sit on stale data.
|
||||
|
||||
For an overview of how Zero works and the list of synced tables, see the [Real-Time Sync with Zero](/docs/how-to/zero-sync) guide.
|
||||
**zero-cache** is the Rocicorp Zero server that sits between PostgreSQL and the browser. It streams real-time updates to all connected clients via WebSocket. Without it, the UI sits on stale data. For an overview of how Zero works, see the [Real-Time Sync with Zero](/docs/how-to/zero-sync) guide.
|
||||
|
||||
### 1. Run Zero-Cache via Docker
|
||||
|
||||
The simplest way to run zero-cache is the official Docker image. Open a new terminal:
|
||||
|
||||
**Linux/macOS:**
|
||||
|
||||
```bash
|
||||
|
|
@ -489,8 +224,9 @@ docker run -d --name surfsense-zero-cache \
|
|||
-e ZERO_CVR_MAX_CONNS=30 \
|
||||
-e ZERO_QUERY_URL="http://host.docker.internal:3000/api/zero/query" \
|
||||
-e ZERO_MUTATE_URL="http://host.docker.internal:3000/api/zero/mutate" \
|
||||
-e ZERO_QUERY_FORWARD_COOKIES=true \
|
||||
-v surfsense-zero-cache:/data \
|
||||
rocicorp/zero:1.4.0
|
||||
rocicorp/zero:1.6.0
|
||||
```
|
||||
|
||||
**Windows (PowerShell):**
|
||||
|
|
@ -510,34 +246,30 @@ docker run -d --name surfsense-zero-cache `
|
|||
-e ZERO_CVR_MAX_CONNS=30 `
|
||||
-e ZERO_QUERY_URL="http://host.docker.internal:3000/api/zero/query" `
|
||||
-e ZERO_MUTATE_URL="http://host.docker.internal:3000/api/zero/mutate" `
|
||||
-e ZERO_QUERY_FORWARD_COOKIES=true `
|
||||
-v surfsense-zero-cache:/data `
|
||||
rocicorp/zero:1.4.0
|
||||
rocicorp/zero:1.6.0
|
||||
```
|
||||
|
||||
**Adjustments to make for your setup:**
|
||||
**Adjustments for your setup:**
|
||||
|
||||
- Replace `postgres:postgres` in the connection URLs with your actual `DB_USER:DB_PASSWORD`.
|
||||
- Replace `postgres:postgres` in the connection URLs with your actual database user and password.
|
||||
- On Linux without Docker Desktop, `host.docker.internal` may not resolve. Either keep the `--add-host=host.docker.internal:host-gateway` flag (Docker 20.10+) or replace `host.docker.internal` with your host's IP / `--network=host` + `localhost`.
|
||||
- For production / custom domains, set `ZERO_QUERY_URL` and `ZERO_MUTATE_URL` to your public frontend URL (e.g. `https://app.yourdomain.com/api/zero/query`).
|
||||
|
||||
### 2. Verify Zero-Cache
|
||||
|
||||
Confirm zero-cache is healthy:
|
||||
|
||||
```bash
|
||||
curl http://localhost:4848/keepalive
|
||||
# Should return HTTP 200
|
||||
```
|
||||
|
||||
Tail the logs to confirm initial replication completed without errors:
|
||||
|
||||
```bash
|
||||
# Tail logs to confirm initial replication completed without errors
|
||||
docker logs -f surfsense-zero-cache
|
||||
```
|
||||
|
||||
### Alternative: Use `docker-compose.deps-only.yml`
|
||||
### Alternative: Let Docker Manage All Dependencies
|
||||
|
||||
If you would rather have Docker manage Postgres, Redis, SearXNG, and zero-cache together (while still running the backend and frontend natively), the repository ships a deps-only compose file. **Run alembic migrations on the host first** so `zero_publication` exists before zero-cache starts:
|
||||
If you'd rather have Docker manage Postgres, Redis, and zero-cache together (while still running the backend and frontend natively), the repository ships a deps-only compose file. **Run alembic migrations on the host first** so `zero_publication` exists before zero-cache starts:
|
||||
|
||||
```bash
|
||||
cd surfsense_backend
|
||||
|
|
@ -546,16 +278,11 @@ cd ../docker
|
|||
docker compose -f docker-compose.deps-only.yml up -d
|
||||
```
|
||||
|
||||
The deps-only stack exposes zero-cache on port `4848` by default. If your
|
||||
frontend is not behind a reverse proxy that serves `/zero`, set
|
||||
`NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848` before building/running the
|
||||
frontend.
|
||||
The deps-only stack exposes zero-cache on port `4848`. Point the frontend at it by setting the zero-cache URL in `surfsense_web/.env` (see the comments in `surfsense_web/.env.example`).
|
||||
|
||||
## Frontend Setup
|
||||
|
||||
### 1. Environment Configuration
|
||||
|
||||
Set up the frontend environment:
|
||||
### 1. Configure the Environment
|
||||
|
||||
**Linux/macOS:**
|
||||
|
||||
|
|
@ -564,13 +291,6 @@ cd surfsense_web
|
|||
cp .env.example .env
|
||||
```
|
||||
|
||||
**Windows (Command Prompt):**
|
||||
|
||||
```cmd
|
||||
cd surfsense_web
|
||||
copy .env.example .env
|
||||
```
|
||||
|
||||
**Windows (PowerShell):**
|
||||
|
||||
```powershell
|
||||
|
|
@ -578,145 +298,57 @@ cd surfsense_web
|
|||
Copy-Item -Path .env.example -Destination .env
|
||||
```
|
||||
|
||||
Edit the `.env` file and set:
|
||||
As with the backend, `.env.example` documents every option inline. Make sure the auth type and ETL service match what you configured for the backend, and that the backend URL points at `http://localhost:8000`.
|
||||
|
||||
| ENV VARIABLE | DESCRIPTION |
|
||||
| --- | --- |
|
||||
| `SURFSENSE_BACKEND_INTERNAL_URL` | Backend URL used by Next.js server routes, e.g. `http://localhost:8000` or `http://backend:8000` in Docker |
|
||||
| `AUTH_TYPE` | Same value as backend auth type: `GOOGLE` for OAuth with Google, `LOCAL` for email/password authentication |
|
||||
| `ETL_SERVICE` | Document parsing service (should match backend ETL_SERVICE): `UNSTRUCTURED`, `LLAMACLOUD`, or `DOCLING`; affects supported file formats in the upload interface |
|
||||
| `DEPLOYMENT_MODE` | `self-hosted` or `cloud`; controls self-hosted-only connector visibility |
|
||||
| `NEXT_PUBLIC_ZERO_CACHE_URL` | Only needed when the browser cannot reach Zero through same-origin `/zero`, e.g. manual local dev at `http://localhost:4848` |
|
||||
|
||||
### 2. Install Dependencies
|
||||
|
||||
Install the frontend dependencies:
|
||||
|
||||
**Linux/macOS:**
|
||||
### 2. Install Dependencies and Run
|
||||
|
||||
```bash
|
||||
# Install pnpm if you don't have it
|
||||
npm install -g pnpm
|
||||
|
||||
# Install dependencies
|
||||
pnpm install
|
||||
```
|
||||
|
||||
**Windows:**
|
||||
|
||||
```powershell
|
||||
# Install pnpm if you don't have it
|
||||
npm install -g pnpm
|
||||
|
||||
# Install dependencies
|
||||
pnpm install
|
||||
```
|
||||
|
||||
### 3. Run the Frontend
|
||||
|
||||
Start the Next.js development server:
|
||||
|
||||
**Linux/macOS/Windows:**
|
||||
|
||||
```bash
|
||||
pnpm run dev
|
||||
```
|
||||
|
||||
The frontend should now be running at `http://localhost:3000`.
|
||||
The frontend should now be running at [http://localhost:3000](http://localhost:3000).
|
||||
|
||||
## Browser Extension Setup (Optional)
|
||||
## Browser Extension (Optional)
|
||||
|
||||
The SurfSense browser extension allows you to save any webpage, including those protected behind authentication.
|
||||
|
||||
### 1. Environment Configuration
|
||||
|
||||
**Linux/macOS:**
|
||||
The SurfSense browser extension saves any webpage — including those behind authentication — straight into your knowledge base.
|
||||
|
||||
```bash
|
||||
cd surfsense_browser_extension
|
||||
cp .env.example .env
|
||||
```
|
||||
cp .env.example .env # set the backend URL, documented inline
|
||||
|
||||
**Windows (Command Prompt):**
|
||||
|
||||
```cmd
|
||||
cd surfsense_browser_extension
|
||||
copy .env.example .env
|
||||
```
|
||||
|
||||
**Windows (PowerShell):**
|
||||
|
||||
```powershell
|
||||
cd surfsense_browser_extension
|
||||
Copy-Item -Path .env.example -Destination .env
|
||||
```
|
||||
|
||||
Edit the `.env` file:
|
||||
|
||||
| ENV VARIABLE | DESCRIPTION |
|
||||
| ------------------------- | ----------------------------------------------------- |
|
||||
| PLASMO_PUBLIC_BACKEND_URL | SurfSense Backend URL (e.g., `http://127.0.0.1:8000`) |
|
||||
|
||||
### 2. Build the Extension
|
||||
|
||||
Build the extension for your browser using the [Plasmo framework](https://docs.plasmo.com/framework/workflows/build#with-a-specific-target).
|
||||
|
||||
**Linux/macOS/Windows:**
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
pnpm install
|
||||
|
||||
# Build for Chrome (default)
|
||||
pnpm build
|
||||
|
||||
# Or for other browsers
|
||||
pnpm build --target=firefox
|
||||
pnpm build --target=edge
|
||||
pnpm build # Chrome (default)
|
||||
pnpm build --target=firefox # or Firefox
|
||||
pnpm build --target=edge # or Edge
|
||||
```
|
||||
|
||||
### 3. Load the Extension
|
||||
Load the built extension in your browser's developer mode and configure it with your SurfSense API key. See the [Plasmo build docs](https://docs.plasmo.com/framework/workflows/build#with-a-specific-target) for details.
|
||||
|
||||
Load the extension in your browser's developer mode and configure it with your SurfSense API key.
|
||||
## Verify Your Installation
|
||||
|
||||
## Verification
|
||||
|
||||
To verify your installation:
|
||||
|
||||
1. Open your browser and navigate to `http://localhost:3000`
|
||||
2. Sign in with your Google account (or local credentials if `AUTH_TYPE=LOCAL`)
|
||||
3. Create a workspace and try uploading a document
|
||||
4. Watch the upload status update live without refreshing. This confirms zero-cache is wired up correctly
|
||||
5. Test the chat functionality with your uploaded content
|
||||
1. Open [http://localhost:3000](http://localhost:3000) and sign in.
|
||||
2. Create a workspace and upload a document.
|
||||
3. Watch the upload status update live without refreshing — this confirms zero-cache is wired up correctly.
|
||||
4. Chat with your uploaded content.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **Database Connection Issues**: Verify your PostgreSQL server is running and pgvector is properly installed
|
||||
- **Redis Connection Issues**: Ensure Redis server is running (`redis-cli ping` should return `PONG`). Check that `REDIS_URL` is correctly set in your `.env` file
|
||||
- **Celery Worker Issues**: Make sure the Celery worker is running in a separate terminal. Check worker logs for any errors
|
||||
- **Authentication Problems**: Check your Google OAuth configuration and ensure redirect URIs are set correctly
|
||||
- **LLM Errors**: Confirm your LLM API keys are valid and the selected models are accessible
|
||||
- **File Upload Failures**: Validate your ETL service API key (Unstructured.io or LlamaCloud) or ensure Docling is properly configured
|
||||
- **Real-time updates not working / stale UI**: Verify zero-cache is running (`curl http://localhost:4848/keepalive` returns 200). Open browser DevTools → Console and look for WebSocket errors. In default Docker, confirm `/zero` is routed by Caddy. In manual local development, confirm `NEXT_PUBLIC_ZERO_CACHE_URL` in `surfsense_web/.env` matches the running zero-cache address.
|
||||
- **Zero-cache stuck on `Unknown or invalid publications. Specified: [zero_publication]`**: You skipped (or never ran) `uv run alembic upgrade head` from `surfsense_backend/`. Run it, then restart the zero-cache container with `docker restart surfsense-zero-cache`.
|
||||
- **Zero-cache crashes with `_zero.tableMetadata` errors**: A previous run left a half-built SQLite replica behind. Stop the container, remove the volume, and start fresh: `docker rm -f surfsense-zero-cache && docker volume rm surfsense-zero-cache && docker run -d ...` (re-run the command from [Zero-Cache Setup](#zero-cache-setup)).
|
||||
- **`wal_level` is not set to `logical`**: zero-cache requires logical replication. Set `wal_level = logical` in `postgresql.conf`, restart PostgreSQL, and verify with `SHOW wal_level;` in psql.
|
||||
- **Database connection issues**: Verify PostgreSQL is running and pgvector is installed.
|
||||
- **Redis connection issues**: `redis-cli ping` should return `PONG`. Check the Redis URL in your backend `.env`.
|
||||
- **Celery worker issues**: Make sure the worker is running in a separate terminal and check its logs.
|
||||
- **File upload failures**: Validate your ETL service API key, or use Docling which needs none.
|
||||
- **Real-time updates not working / stale UI**: Verify zero-cache is running (`curl http://localhost:4848/keepalive` returns 200). Open browser DevTools → Console and look for WebSocket errors. Confirm the zero-cache URL in `surfsense_web/.env` matches the running zero-cache address.
|
||||
- **Zero-cache stuck on `Unknown or invalid publications. Specified: [zero_publication]`**: You skipped `uv run alembic upgrade head`. Run it from `surfsense_backend/`, then `docker restart surfsense-zero-cache`.
|
||||
- **Zero-cache crashes with `_zero.tableMetadata` errors**: A previous run left a half-built SQLite replica behind. Start fresh: `docker rm -f surfsense-zero-cache && docker volume rm surfsense-zero-cache`, then re-run the command from [Zero-Cache Setup](#zero-cache-setup).
|
||||
- **`wal_level` is not `logical`**: zero-cache requires logical replication. Set it in `postgresql.conf`, restart PostgreSQL, and verify with `SHOW wal_level;`.
|
||||
- **Backend `/ready` returns 503**: The readiness probe verifies `zero_publication` exists. Run `uv run alembic upgrade head` to create it.
|
||||
- **Windows-specific**: If you encounter path issues, ensure you're using the correct path separator (`\` instead of `/`)
|
||||
- **macOS-specific**: If you encounter permission issues, you may need to use `sudo` for some installation commands
|
||||
|
||||
## Next Steps
|
||||
|
||||
Now that you have SurfSense running locally, you can explore its features:
|
||||
|
||||
- Create workspaces for organizing your content
|
||||
- Upload documents or use the browser extension to save webpages
|
||||
- Ask questions about your saved content
|
||||
- Explore the advanced RAG capabilities
|
||||
|
||||
For production deployments, consider setting up:
|
||||
|
||||
- A reverse proxy like Nginx
|
||||
- SSL certificates for secure connections
|
||||
- Proper database backups
|
||||
- User access controls
|
||||
- Set up [connectors](/docs/connectors) to bring in your tools and services.
|
||||
- Connect [local models](/docs/local-models) like Ollama or LM Studio.
|
||||
- For production, put a reverse proxy in front, add SSL, and set up database backups.
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@
|
|||
"pages": [
|
||||
"---Guides---",
|
||||
"index",
|
||||
"prerequisites",
|
||||
"installation",
|
||||
"manual-installation",
|
||||
"docker-installation",
|
||||
|
|
|
|||
|
|
@ -1,86 +0,0 @@
|
|||
---
|
||||
title: Prerequisites
|
||||
description: Required setup's before setting up SurfSense
|
||||
icon: ClipboardCheck
|
||||
---
|
||||
|
||||
|
||||
## Auth Setup
|
||||
|
||||
SurfSense supports both Google OAuth and local email/password authentication. Google OAuth is optional - if you prefer local authentication, you can skip this section.
|
||||
|
||||
**Note**: Google OAuth setup is **required** in your `.env` files if you want to use the Gmail and Google Calendar connectors in SurfSense.
|
||||
|
||||
To set up Google OAuth:
|
||||
|
||||
1. Login to your [Google Developer Console](https://console.cloud.google.com/)
|
||||
2. Enable the required APIs:
|
||||
- **People API** (required for basic Google OAuth)
|
||||
|
||||
3. Set up OAuth consent screen.
|
||||
|
||||
4. Create OAuth client ID and secret.
|
||||
|
||||
5. It should look like this.
|
||||
|
||||
|
||||
---
|
||||
|
||||
## File Upload's
|
||||
|
||||
SurfSense supports three ETL (Extract, Transform, Load) services for converting files to LLM-friendly formats:
|
||||
|
||||
### Option 1: Unstructured
|
||||
|
||||
Files are converted using [Unstructured](https://github.com/Unstructured-IO/unstructured)
|
||||
|
||||
1. Get an Unstructured.io API key from [Unstructured Platform](https://platform.unstructured.io/)
|
||||
2. You should be able to generate API keys once registered
|
||||
|
||||
|
||||
### Option 2: LlamaIndex (LlamaCloud)
|
||||
|
||||
Files are converted using [LlamaIndex](https://www.llamaindex.ai/) which offers 50+ file format support.
|
||||
|
||||
1. Get a LlamaIndex API key from [LlamaCloud](https://cloud.llamaindex.ai/)
|
||||
2. Sign up for a LlamaCloud account to access their parsing services
|
||||
3. LlamaCloud provides enhanced parsing capabilities for complex documents
|
||||
|
||||
### Option 3: Docling (Recommended for Privacy)
|
||||
|
||||
Files are processed locally using [Docling](https://github.com/DS4SD/docling) - IBM's open-source document parsing library.
|
||||
|
||||
1. **No API key required** - all processing happens locally
|
||||
2. **Privacy-focused** - documents never leave your system
|
||||
3. **Supported formats**: PDF, Office documents (Word, Excel, PowerPoint), images (PNG, JPEG, TIFF, BMP, WebP), HTML, CSV, AsciiDoc
|
||||
4. **Enhanced features**: Advanced table detection, image extraction, and structured document parsing
|
||||
5. **GPU acceleration** support for faster processing (when available)
|
||||
|
||||
**Note**: You only need to set up one of these services.
|
||||
|
||||
---
|
||||
|
||||
## LLM Observability (Optional)
|
||||
|
||||
This is not required for SurfSense to work. But it is always a good idea to monitor LLM interactions. So we do not have those WTH moments.
|
||||
|
||||
1. Get a LangSmith API key from [smith.langchain.com](https://smith.langchain.com/)
|
||||
2. This helps in observing SurfSense Researcher Agent.
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Crawler
|
||||
|
||||
SurfSense have 2 options for saving webpages:
|
||||
- [SurfSense Extension](https://github.com/MODSetter/SurfSense/tree/main/surfsense_browser_extension) (Overall better experience & ability to save private webpages, recommended)
|
||||
- Crawler (If you want to save public webpages)
|
||||
|
||||
**NOTE:** SurfSense currently uses [Firecrawl.py](https://www.firecrawl.dev/) for web crawling. If you plan on using the crawler, you will need to create a Firecrawl account and get an API key.
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
Once you have all prerequisites in place, proceed to the [installation guide](/docs/installation) to set up SurfSense.
|
||||
|
|
@ -6,6 +6,12 @@ import { z } from "zod";
|
|||
* consumed by the playground's generic form renderer, the output schema by the
|
||||
* API reference docs — so they are intentionally untyped here.
|
||||
*/
|
||||
/** One live per-item rate a verb charges on, e.g. 3500 micro-USD per place. */
|
||||
export const scraperPricingMeter = z.object({
|
||||
unit: z.string(),
|
||||
micros_per_unit: z.number(),
|
||||
});
|
||||
|
||||
export const scraperCapability = z.object({
|
||||
name: z.string(),
|
||||
description: z.string(),
|
||||
|
|
@ -13,6 +19,8 @@ export const scraperCapability = z.object({
|
|||
// Optional so a backend that predates output schemas degrades to just not
|
||||
// showing the output-schema block instead of failing the whole fetch.
|
||||
output_schema: z.record(z.string(), z.unknown()).optional(),
|
||||
// Optional for the same backward-compat reason; empty array = free.
|
||||
pricing: z.array(scraperPricingMeter).optional(),
|
||||
});
|
||||
|
||||
export const listCapabilitiesResponse = z.array(scraperCapability);
|
||||
|
|
@ -50,6 +58,7 @@ export const startAsyncRunResponse = z.object({
|
|||
status: z.string(),
|
||||
});
|
||||
|
||||
export type ScraperPricingMeter = z.infer<typeof scraperPricingMeter>;
|
||||
export type ScraperCapability = z.infer<typeof scraperCapability>;
|
||||
export type ScraperRunSummary = z.infer<typeof scraperRunSummary>;
|
||||
export type ScraperRunDetail = z.infer<typeof scraperRunDetail>;
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import {
|
||||
IconBrandGoogle,
|
||||
IconBrandGoogleMaps,
|
||||
IconBrandReddit,
|
||||
IconBrandYoutube,
|
||||
IconWorldWww,
|
||||
} from "@tabler/icons-react";
|
||||
import type { ComponentType } from "react";
|
||||
import {
|
||||
GoogleMapsIcon,
|
||||
GoogleSearchIcon,
|
||||
RedditIcon,
|
||||
WebIcon,
|
||||
YouTubeIcon,
|
||||
} from "./platform-icons";
|
||||
|
||||
/** Icon component that accepts a ``className`` (Tabler + Lucide both satisfy this). */
|
||||
export type PlatformIcon = ComponentType<{ className?: string }>;
|
||||
|
|
@ -36,13 +36,13 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [
|
|||
{
|
||||
id: "reddit",
|
||||
label: "Reddit",
|
||||
icon: IconBrandReddit,
|
||||
icon: RedditIcon,
|
||||
verbs: [{ name: "reddit.scrape", verb: "scrape", label: "Scrape" }],
|
||||
},
|
||||
{
|
||||
id: "youtube",
|
||||
label: "YouTube",
|
||||
icon: IconBrandYoutube,
|
||||
icon: YouTubeIcon,
|
||||
verbs: [
|
||||
{ name: "youtube.scrape", verb: "scrape", label: "Scrape" },
|
||||
{ name: "youtube.comments", verb: "comments", label: "Comments" },
|
||||
|
|
@ -51,7 +51,7 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [
|
|||
{
|
||||
id: "google_maps",
|
||||
label: "Google Maps",
|
||||
icon: IconBrandGoogleMaps,
|
||||
icon: GoogleMapsIcon,
|
||||
verbs: [
|
||||
{ name: "google_maps.scrape", verb: "scrape", label: "Scrape" },
|
||||
{ name: "google_maps.reviews", verb: "reviews", label: "Reviews" },
|
||||
|
|
@ -60,13 +60,13 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [
|
|||
{
|
||||
id: "google_search",
|
||||
label: "Google Search",
|
||||
icon: IconBrandGoogle,
|
||||
icon: GoogleSearchIcon,
|
||||
verbs: [{ name: "google_search.scrape", verb: "scrape", label: "Scrape" }],
|
||||
},
|
||||
{
|
||||
id: "web",
|
||||
label: "Web",
|
||||
icon: IconWorldWww,
|
||||
icon: WebIcon,
|
||||
verbs: [{ name: "web.crawl", verb: "crawl", label: "Crawl" }],
|
||||
},
|
||||
];
|
||||
|
|
|
|||
|
|
@ -1,11 +1,28 @@
|
|||
/** Formatting helpers shared by the playground runs table and runner output. */
|
||||
|
||||
import type { ScraperPricingMeter } from "@/contracts/types/scraper.types";
|
||||
|
||||
export function formatCost(costMicros: number | null | undefined): string {
|
||||
if (costMicros == null) return "—";
|
||||
if (costMicros === 0) return "Free";
|
||||
return `$${(costMicros / 1_000_000).toFixed(4)}`;
|
||||
}
|
||||
|
||||
/** One meter as a per-1k rate, e.g. 3500 micros/place -> "$3.50 / 1k places". */
|
||||
export function formatRate(meter: ScraperPricingMeter): string {
|
||||
const perThousand = (meter.micros_per_unit * 1000) / 1_000_000;
|
||||
const dollars = Number.isInteger(perThousand)
|
||||
? perThousand.toString()
|
||||
: perThousand.toFixed(2);
|
||||
return `$${dollars} / 1k ${meter.unit}s`;
|
||||
}
|
||||
|
||||
/** A capability's full price line: "Free", one rate, or "rate + rate". */
|
||||
export function formatPricing(pricing: ScraperPricingMeter[] | undefined): string {
|
||||
if (!pricing || pricing.length === 0) return "Free";
|
||||
return pricing.map(formatRate).join(" + ");
|
||||
}
|
||||
|
||||
export function formatDuration(ms: number | null | undefined): string {
|
||||
if (ms == null) return "—";
|
||||
if (ms < 1000) return `${ms}ms`;
|
||||
|
|
|
|||
29
surfsense_web/lib/playground/platform-icons.tsx
Normal file
29
surfsense_web/lib/playground/platform-icons.tsx
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import Image from "next/image";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
/**
|
||||
* Full-color brand marks for the platform-native scraper verbs, served from
|
||||
* `/public/connectors/*.svg` (same asset library the connector UI uses). Each
|
||||
* is a `ComponentType<{ className?: string }>` so it drops into the playground
|
||||
* catalog and the composer badge exactly like a Lucide/Tabler icon.
|
||||
*/
|
||||
function brandIcon(src: string, alt: string) {
|
||||
return function BrandIcon({ className }: { className?: string }) {
|
||||
return (
|
||||
<Image
|
||||
src={src}
|
||||
alt={alt}
|
||||
width={20}
|
||||
height={20}
|
||||
className={cn("select-none object-contain pointer-events-none", className)}
|
||||
draggable={false}
|
||||
/>
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit");
|
||||
export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube");
|
||||
export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps");
|
||||
export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search");
|
||||
export const WebIcon = brandIcon("/connectors/web.svg", "Web");
|
||||
|
|
@ -205,7 +205,7 @@
|
|||
"manage_documents": "Manage Documents",
|
||||
"connectors": "Connectors",
|
||||
"add_connector": "Add Connector",
|
||||
"manage_connectors": "Manage Connectors",
|
||||
"manage_connectors": "Manage External MCP Connectors",
|
||||
"logs": "Logs",
|
||||
"all_workspaces": "All Workspaces",
|
||||
"team": "Team"
|
||||
|
|
|
|||
1
surfsense_web/public/connectors/google-maps.svg
Normal file
1
surfsense_web/public/connectors/google-maps.svg
Normal file
|
|
@ -0,0 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 92.3 132.3"><path fill="#1a73e8" d="M60.2 2.2C55.8.8 51 0 46.1 0 32 0 19.3 6.4 10.8 16.5l21.8 18.3L60.2 2.2z"/><path fill="#ea4335" d="M10.8 16.5C4.1 24.5 0 34.9 0 46.1c0 8.7 1.7 15.7 4.6 22l28-33.3-21.8-18.3z"/><path fill="#4285f4" d="M46.2 28.5c9.8 0 17.7 7.9 17.7 17.7 0 4.3-1.6 8.3-4.2 11.4 0 0 13.9-16.6 27.5-32.7-5.6-10.8-15.3-19-27-22.7L32.6 34.8c3.3-3.8 8.1-6.3 13.6-6.3"/><path fill="#fbbc04" d="M46.2 63.8c-9.8 0-17.7-7.9-17.7-17.7 0-4.3 1.5-8.3 4.1-11.3l-28 33.3c4.8 10.6 12.8 19.2 21 29.9l34.1-40.5c-3.3 3.9-8.1 6.3-13.5 6.3"/><path fill="#34a853" d="M59.1 109.2c15.4-24.1 33.3-35 33.3-63 0-7.7-1.9-14.9-5.2-21.3L25.6 98c2.6 3.4 5.3 7.3 7.9 11.3 9.4 14.5 6.8 23.1 12.8 23.1s3.4-8.7 12.8-23.2"/></svg>
|
||||
|
After Width: | Height: | Size: 765 B |
1
surfsense_web/public/connectors/google-search.svg
Normal file
1
surfsense_web/public/connectors/google-search.svg
Normal file
|
|
@ -0,0 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" height="24" viewBox="0 0 24 24" width="24"><path d="M22.56 12.25c0-.78-.07-1.53-.2-2.25H12v4.26h5.92c-.26 1.37-1.04 2.53-2.21 3.31v2.77h3.57c2.08-1.92 3.28-4.74 3.28-8.09z" fill="#4285F4"/><path d="M12 23c2.97 0 5.46-.98 7.28-2.66l-3.57-2.77c-.98.66-2.23 1.06-3.71 1.06-2.86 0-5.29-1.93-6.16-4.53H2.18v2.84C3.99 20.53 7.7 23 12 23z" fill="#34A853"/><path d="M5.84 14.09c-.22-.66-.35-1.36-.35-2.09s.13-1.43.35-2.09V7.07H2.18C1.43 8.55 1 10.22 1 12s.43 3.45 1.18 4.93l2.85-2.22.81-.62z" fill="#FBBC05"/><path d="M12 5.38c1.62 0 3.06.56 4.21 1.64l3.15-3.15C17.45 2.09 14.97 1 12 1 7.7 1 3.99 3.47 2.18 7.07l3.66 2.84c.87-2.6 3.3-4.53 6.16-4.53z" fill="#EA4335"/></svg>
|
||||
|
After Width: | Height: | Size: 705 B |
1
surfsense_web/public/connectors/reddit.svg
Normal file
1
surfsense_web/public/connectors/reddit.svg
Normal file
|
|
@ -0,0 +1 @@
|
|||
<svg role="img" viewBox="0 0 24 24" fill="#FF4500" xmlns="http://www.w3.org/2000/svg"><title>Reddit</title><path d="M12 0C5.373 0 0 5.373 0 12c0 3.314 1.343 6.314 3.515 8.485l-2.286 2.286C.775 23.225 1.097 24 1.738 24H12c6.627 0 12-5.373 12-12S18.627 0 12 0Zm4.388 3.199c1.104 0 1.999.895 1.999 1.999 0 1.105-.895 2-1.999 2-.946 0-1.739-.657-1.947-1.539v.002c-1.147.162-2.032 1.15-2.032 2.341v.007c1.776.067 3.4.567 4.686 1.363.473-.363 1.064-.58 1.707-.58 1.547 0 2.802 1.254 2.802 2.802 0 1.117-.655 2.081-1.601 2.531-.088 3.256-3.637 5.876-7.997 5.876-4.361 0-7.905-2.617-7.998-5.87-.954-.447-1.614-1.415-1.614-2.538 0-1.548 1.255-2.802 2.803-2.802.645 0 1.239.218 1.712.585 1.275-.79 2.881-1.291 4.64-1.365v-.01c0-1.663 1.263-3.034 2.88-3.207.188-.911.993-1.595 1.959-1.595Zm-8.085 8.376c-.784 0-1.459.78-1.506 1.797-.047 1.016.64 1.429 1.426 1.429.786 0 1.371-.369 1.418-1.385.047-1.017-.553-1.841-1.338-1.841Zm7.406 0c-.786 0-1.385.824-1.338 1.841.047 1.017.634 1.385 1.418 1.385.785 0 1.473-.413 1.426-1.429-.046-1.017-.721-1.797-1.506-1.797Zm-3.703 4.013c-.974 0-1.907.048-2.77.135-.147.015-.241.168-.183.305.483 1.154 1.622 1.964 2.953 1.964 1.33 0 2.47-.81 2.953-1.964.057-.137-.037-.29-.184-.305-.863-.087-1.795-.135-2.769-.135Z"/></svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
1
surfsense_web/public/connectors/web.svg
Normal file
1
surfsense_web/public/connectors/web.svg
Normal file
|
|
@ -0,0 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="#1a73e8" stroke-width="1.6" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="9.25"/><path d="M2.75 12h18.5"/><ellipse cx="12" cy="12" rx="4.1" ry="9.25"/><path d="M5 6.4h14M5 17.6h14"/></svg>
|
||||
|
After Width: | Height: | Size: 296 B |
|
|
@ -44,7 +44,7 @@ test.describe("ClickUp connector journey", () => {
|
|||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
await expect(connectorDialog.getByText("ClickUp")).toBeVisible();
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ test.describe("Composio Calendar journey", () => {
|
|||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
|
||||
const beforeChatDocs = await listDocuments(request, apiToken, workspace.id);
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import {
|
|||
} from "../../../helpers/api/connectors";
|
||||
import { getEditorContent, listDocuments } from "../../../helpers/api/documents";
|
||||
import { CANARY_TOKENS, FAKE_DRIVE_FILES } from "../../../helpers/canary";
|
||||
import { openConnectorPopup } from "../../../helpers/ui/connector-popup";
|
||||
import { expectImportConnectorAvailable } from "../../../helpers/ui/connector-popup";
|
||||
import { waitForDocumentByTitle, waitForIndexingComplete } from "../../../helpers/waits/indexing";
|
||||
|
||||
/**
|
||||
|
|
@ -31,9 +31,7 @@ test.describe("Composio Drive journey", () => {
|
|||
await page.goto(`/dashboard/${workspace.id}/new-chat`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
await expectImportConnectorAvailable(page, "Google Drive");
|
||||
|
||||
const selectedFiles = [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ test.describe("Composio Gmail journey", () => {
|
|||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
|
||||
const beforeChatDocs = await listDocuments(request, apiToken, workspace.id);
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ test.describe("Confluence connector journey", () => {
|
|||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
await connectorDialog.getByPlaceholder("Search").fill("Confluence");
|
||||
await expect(connectorDialog.getByText("Confluence", { exact: true })).toBeVisible();
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { streamChatToCompletion } from "../../helpers/api/chat";
|
|||
import { listConnectors, triggerIndex, updateConnectorConfig } from "../../helpers/api/connectors";
|
||||
import { getEditorContent, listDocuments } from "../../helpers/api/documents";
|
||||
import { CANARY_TOKENS, FAKE_DROPBOX_FILES } from "../../helpers/canary";
|
||||
import { openConnectorPopup } from "../../helpers/ui/connector-popup";
|
||||
import { expectImportConnectorAvailable } from "../../helpers/ui/connector-popup";
|
||||
import { waitForDocumentByTitle, waitForIndexingComplete } from "../../helpers/waits/indexing";
|
||||
|
||||
/**
|
||||
|
|
@ -32,9 +32,7 @@ test.describe("Native Dropbox journey", () => {
|
|||
await page.goto(`/dashboard/${workspace.id}/new-chat`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
await expectImportConnectorAvailable(page, "Dropbox");
|
||||
|
||||
const selectedFiles = [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ test.describe("Native Google Calendar journey", () => {
|
|||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
|
||||
const beforeDocs = await listDocuments(request, apiToken, workspace.id);
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import {
|
|||
} from "../../../helpers/api/connectors";
|
||||
import { getEditorContent, listDocuments } from "../../../helpers/api/documents";
|
||||
import { CANARY_TOKENS, FAKE_DRIVE_FILES } from "../../../helpers/canary";
|
||||
import { openConnectorPopup } from "../../../helpers/ui/connector-popup";
|
||||
import { expectImportConnectorAvailable } from "../../../helpers/ui/connector-popup";
|
||||
import { waitForDocumentByTitle, waitForIndexingComplete } from "../../../helpers/waits/indexing";
|
||||
|
||||
/**
|
||||
|
|
@ -36,9 +36,7 @@ test.describe("Native Google Drive journey", () => {
|
|||
await page.goto(`/dashboard/${workspace.id}/new-chat`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
await expectImportConnectorAvailable(page, "Google Drive");
|
||||
|
||||
const selectedFiles = [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ test.describe("Native Google Gmail journey", () => {
|
|||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
|
||||
const beforeDocs = await listDocuments(request, apiToken, workspace.id);
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ test.describe("Jira connector journey", () => {
|
|||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
await connectorDialog.getByPlaceholder("Search").fill("Jira");
|
||||
await expect(connectorDialog.getByText("Jira", { exact: true })).toBeVisible();
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ test.describe("Linear connector journey", () => {
|
|||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
|
||||
const beforeDocs = await listDocuments(request, apiToken, workspace.id);
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ test.describe("Notion connector journey", () => {
|
|||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
await connectorDialog.getByPlaceholder("Search").fill("Notion");
|
||||
await expect(connectorDialog.getByText("Notion", { exact: true })).toBeVisible();
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { streamChatToCompletion } from "../../helpers/api/chat";
|
|||
import { listConnectors, triggerIndex, updateConnectorConfig } from "../../helpers/api/connectors";
|
||||
import { getEditorContent, listDocuments } from "../../helpers/api/documents";
|
||||
import { CANARY_TOKENS, FAKE_ONEDRIVE_FILES } from "../../helpers/canary";
|
||||
import { openConnectorPopup } from "../../helpers/ui/connector-popup";
|
||||
import { expectImportConnectorAvailable } from "../../helpers/ui/connector-popup";
|
||||
import { waitForDocumentByTitle, waitForIndexingComplete } from "../../helpers/waits/indexing";
|
||||
|
||||
/**
|
||||
|
|
@ -32,9 +32,7 @@ test.describe("Native OneDrive journey", () => {
|
|||
await page.goto(`/dashboard/${workspace.id}/new-chat`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
await expectImportConnectorAvailable(page, "OneDrive");
|
||||
|
||||
const selectedFiles = [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ test.describe("Slack connector journey", () => {
|
|||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await openConnectorPopup(page);
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" });
|
||||
const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" });
|
||||
await expect(connectorDialog).toBeVisible();
|
||||
await connectorDialog.getByPlaceholder("Search").fill("Slack");
|
||||
await expect(connectorDialog.getByText("Slack", { exact: true })).toBeVisible();
|
||||
|
|
|
|||
|
|
@ -21,14 +21,27 @@ export async function openConnectorPopup(page: Page): Promise<void> {
|
|||
await expect(trigger).toBeVisible({ timeout: 60_000 });
|
||||
await trigger.click();
|
||||
|
||||
await expect(page.getByRole("dialog", { name: "Manage Connectors" })).toBeVisible();
|
||||
await expect(page.getByRole("dialog", { name: "Manage External MCP Connectors" })).toBeVisible();
|
||||
}
|
||||
|
||||
export async function clickComposioDriveCard(page: Page): Promise<void> {
|
||||
const composioDriveCard = page.getByText("Search your Drive files via Composio");
|
||||
await composioDriveCard.scrollIntoViewIfNeeded();
|
||||
const card = composioDriveCard
|
||||
.locator("xpath=ancestor::*[self::article or self::div][1]")
|
||||
.first();
|
||||
await card.getByRole("button", { name: "Connect" }).click();
|
||||
/**
|
||||
* Opens the Documents sidebar "Import" menu. Import connectors
|
||||
* (Google Drive / OneDrive / Dropbox) are surfaced here instead of in the
|
||||
* external-MCP connector catalog.
|
||||
*/
|
||||
export async function openDocumentsImportMenu(page: Page): Promise<void> {
|
||||
const trigger = page.getByRole("button", { name: "Import documents" }).first();
|
||||
// Long timeout absorbs Next.js dev cold-compile of the dashboard route.
|
||||
await expect(trigger).toBeVisible({ timeout: 60_000 });
|
||||
await trigger.click();
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the Import menu and asserts a cloud-drive import connector is offered.
|
||||
* `label` is the visible menu label ("Google Drive", "OneDrive", "Dropbox").
|
||||
*/
|
||||
export async function expectImportConnectorAvailable(page: Page, label: string): Promise<void> {
|
||||
await openDocumentsImportMenu(page);
|
||||
await expect(page.getByRole("menuitem", { name: label })).toBeVisible();
|
||||
await page.keyboard.press("Escape");
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue