apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-06-06 20:15:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(UI): reorganized connectors

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2025-11-26 13:44:38 -08:00
parent 8f30cfd69a
commit ab6ea7e0ab
11 changed files with 133 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

1
surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py
View file

@ -187,6 +187,7 @@ async def index_crawled_urls(
)
# Generate content hash
# TODO: To fix this by not including dynamic content like date, time, etc.
content_hash = generate_content_hash(
structured_document, search_space_id
)

4
surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx
View file

@ -316,7 +316,8 @@ export default function EditConnectorPage() {
/>
</FormControl>
<FormDescription>
Enter URLs to crawl (one per line). These URLs will be indexed when you trigger indexing.
Enter URLs to crawl (one per line). These URLs will be indexed when you
trigger indexing.
</FormDescription>
<FormMessage />
</FormItem>
@ -324,7 +325,6 @@ export default function EditConnectorPage() {
/>
</div>
)}
</CardContent>
<CardFooter className="border-t pt-6">
<Button type="submit" disabled={isSaving} className="w-full sm:w-auto">

27
surfsense_web/app/dashboard/[search_space_id]/connectors/add/webcrawler-connector/page.tsx
View file

@ -75,7 +75,8 @@ export default function WebcrawlerConnectorPage() {
.then((data) => {
if (data && Array.isArray(data)) {
const connector = data.find(
(c: SearchSourceConnector) => c.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR
(c: SearchSourceConnector) =>
c.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR
);
if (connector) {
setDoesConnectorExist(true);
@ -92,7 +93,7 @@ export default function WebcrawlerConnectorPage() {
setIsSubmitting(true);
try {
const config: Record<string, string> = {};
// Only add API key to config if provided
if (values.api_key && values.api_key.trim()) {
config.FIRECRAWL_API_KEY = values.api_key;
@ -162,8 +163,8 @@ export default function WebcrawlerConnectorPage() {
<CardHeader>
<CardTitle>Set Up Web Page crawler</CardTitle>
<CardDescription>
Configure your web page crawler to index web pages. Optionally add a Firecrawl API key
for enhanced crawling capabilities.
Configure your web page crawler to index web pages. Optionally add a Firecrawl API
key for enhanced crawling capabilities.
</CardDescription>
</CardHeader>
<Form {...form}>
@ -193,11 +194,7 @@ export default function WebcrawlerConnectorPage() {
<FormItem>
<FormLabel>Firecrawl API Key (Optional)</FormLabel>
<FormControl>
<Input
type="password"
placeholder="fc-xxxxxxxxxxxxx"
{...field}
/>
<Input type="password" placeholder="fc-xxxxxxxxxxxxx" {...field} />
</FormControl>
<FormDescription>
Add a Firecrawl API key for enhanced crawling. If not provided, will use
@ -215,10 +212,10 @@ export default function WebcrawlerConnectorPage() {
<FormItem>
<FormLabel>Initial URLs (Optional)</FormLabel>
<FormControl>
<Textarea
<Textarea
placeholder="https://example.com&#10;https://docs.example.com&#10;https://blog.example.com"
className="min-h-[100px] font-mono text-sm"
{...field}
{...field}
/>
</FormControl>
<FormDescription>
@ -296,9 +293,9 @@ export default function WebcrawlerConnectorPage() {
<h4 className="font-medium mb-2">1. Choose Your Crawler Method</h4>
<p className="text-sm text-muted-foreground">
<strong>With Firecrawl (Recommended):</strong> Get your API key from{" "}
<a
href="https://firecrawl.dev"
target="_blank"
<a
href="https://firecrawl.dev"
target="_blank"
rel="noopener noreferrer"
className="text-primary hover:underline"
>
@ -331,4 +328,4 @@ export default function WebcrawlerConnectorPage() {
</motion.div>
</div>
);
}
}

28
surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx
View file

@ -1,9 +1,9 @@
"use client";
import { IconBrandYoutube } from "@tabler/icons-react";
import { Cable, Database, Upload } from "lucide-react";
import { Cable, Database, Globe, Upload } from "lucide-react";
import { motion } from "motion/react";
import { useParams, useSearchParams } from "next/navigation";
import { useParams, useRouter, useSearchParams } from "next/navigation";
import { useEffect, useState } from "react";
import { ConnectorsTab } from "@/components/sources/ConnectorsTab";
import { DocumentUploadTab } from "@/components/sources/DocumentUploadTab";
@ -12,6 +12,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
export default function AddSourcesPage() {
const params = useParams();
const router = useRouter();
const searchParams = useSearchParams();
const search_space_id = params.search_space_id as string;
const [activeTab, setActiveTab] = useState("documents");
@ -24,6 +25,14 @@ export default function AddSourcesPage() {
}
}, [searchParams]);
const handleTabChange = (value: string) => {
if (value === "webpages") {
router.push(`/dashboard/${search_space_id}/connectors/add/webcrawler-connector`);
} else {
setActiveTab(value);
}
};
return (
<div className="container mx-auto py-8 px-4">
<motion.div
@ -42,19 +51,26 @@ export default function AddSourcesPage() {
</div>
{/* Tabs */}
<Tabs value={activeTab} onValueChange={setActiveTab} className="w-full">
<TabsList className="grid w-full max-w-2xl mx-auto grid-cols-3 h-12">
<Tabs value={activeTab} onValueChange={handleTabChange} className="w-full">
<TabsList className="grid w-full max-w-3xl mx-auto grid-cols-4 h-12">
<TabsTrigger value="documents" className="flex items-center gap-2">
<Upload className="h-4 w-4" />
Documents
<span className="hidden sm:inline">Documents</span>
<span className="sm:hidden">Docs</span>
</TabsTrigger>
<TabsTrigger value="youtube" className="flex items-center gap-2">
<IconBrandYoutube className="h-4 w-4" />
YouTube
</TabsTrigger>
<TabsTrigger value="webpages" className="flex items-center gap-2">
<Globe className="h-4 w-4" />
<span className="hidden sm:inline">Web Pages</span>
<span className="sm:hidden">Web</span>
</TabsTrigger>
<TabsTrigger value="connectors" className="flex items-center gap-2">
<Cable className="h-4 w-4" />
Connectors
<span className="hidden sm:inline">Connectors</span>
<span className="sm:hidden">More</span>
</TabsTrigger>
</TabsList>

2
surfsense_web/components/editConnector/types.ts
View file

@ -53,6 +53,6 @@ export const editConnectorSchema = z.object({
LUMA_API_KEY: z.string().optional(),
ELASTICSEARCH_API_KEY: z.string().optional(),
FIRECRAWL_API_KEY: z.string().optional(),
INITIAL_URLS: z.string().optional()
INITIAL_URLS: z.string().optional(),
});
export type EditConnectorFormValues = z.infer<typeof editConnectorSchema>;

2
surfsense_web/components/homepage/integrations.tsx
View file

@ -29,7 +29,7 @@ const INTEGRATIONS: Integration[] = [
// Documentation & Knowledge
{ name: "Confluence", icon: "https://cdn.simpleicons.org/confluence/172B4D" },
{ name: "Notion", icon: "https://cdn.simpleicons.org/notion/000000/ffffff" },
{ name: "Web Pages", icon: "https://cdn.jsdelivr.net/npm/lucide-static@0.294.0/icons/globe.svg"},
{ name: "Web Pages", icon: "https://cdn.jsdelivr.net/npm/lucide-static@0.294.0/icons/globe.svg" },
// Cloud Storage
{ name: "Google Drive", icon: "https://cdn.simpleicons.org/googledrive/4285F4" },

11
surfsense_web/components/sources/ConnectorsTab.tsx
View file

@ -19,11 +19,14 @@ interface ConnectorsTabProps {
export function ConnectorsTab({ searchSpaceId }: ConnectorsTabProps) {
const t = useTranslations("add_connector");
const [expandedCategories, setExpandedCategories] = useState<string[]>([
"search-engines",
"knowledge-bases",
"web-search",
"messaging",
"project-management",
"team-chats",
"communication",
"documentation",
"development",
"databases",
"productivity",
"web-crawling",
]);
const toggleCategory = (categoryId: string) => {

98
surfsense_web/components/sources/connector-data.tsx
View file

@ -5,8 +5,21 @@ import type { ConnectorCategory } from "./types";
export const connectorCategories: ConnectorCategory[] = [
{
id: "search-engines",
title: "search_engines",
id: "web-crawling",
title: "web_crawling",
connectors: [
{
id: "webcrawler-connector",
title: "Web Pages",
description: "webcrawler_desc",
icon: getConnectorIcon(EnumConnectorName.WEBCRAWLER_CONNECTOR, "h-6 w-6"),
status: "available",
},
],
},
{
id: "web-search",
title: "web_search",
connectors: [
{
id: "tavily-api",
@ -29,13 +42,6 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.LINKUP_API, "h-6 w-6"),
status: "available",
},
{
id: "elasticsearch-connector",
title: "Elasticsearch",
description: "elasticsearch_desc",
icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"),
status: "available",
},
{
id: "baidu-search-api",
title: "Baidu Search",
@ -46,8 +52,8 @@ export const connectorCategories: ConnectorCategory[] = [
],
},
{
id: "team-chats",
title: "team_chats",
id: "messaging",
title: "messaging",
connectors: [
{
id: "slack-connector",
@ -56,13 +62,6 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.SLACK_CONNECTOR, "h-6 w-6"),
status: "available",
},
{
id: "ms-teams",
title: "Microsoft Teams",
description: "teams_desc",
icon: <IconBrandWindows className="h-6 w-6" />,
status: "coming-soon",
},
{
id: "discord-connector",
title: "Discord",
@ -70,6 +69,13 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.DISCORD_CONNECTOR, "h-6 w-6"),
status: "available",
},
{
id: "ms-teams",
title: "Microsoft Teams",
description: "teams_desc",
icon: <IconBrandWindows className="h-6 w-6" />,
status: "coming-soon",
},
],
},
{
@ -100,8 +106,8 @@ export const connectorCategories: ConnectorCategory[] = [
],
},
{
id: "knowledge-bases",
title: "knowledge_bases",
id: "documentation",
title: "documentation",
connectors: [
{
id: "notion-connector",
@ -110,6 +116,19 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.NOTION_CONNECTOR, "h-6 w-6"),
status: "available",
},
{
id: "confluence-connector",
title: "Confluence",
description: "confluence_desc",
icon: getConnectorIcon(EnumConnectorName.CONFLUENCE_CONNECTOR, "h-6 w-6"),
status: "available",
},
],
},
{
id: "development",
title: "development",
connectors: [
{
id: "github-connector",
title: "GitHub",
@ -117,11 +136,17 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.GITHUB_CONNECTOR, "h-6 w-6"),
status: "available",
},
],
},
{
id: "databases",
title: "databases",
connectors: [
{
id: "confluence-connector",
title: "Confluence",
description: "confluence_desc",
icon: getConnectorIcon(EnumConnectorName.CONFLUENCE_CONNECTOR, "h-6 w-6"),
id: "elasticsearch-connector",
title: "Elasticsearch",
description: "elasticsearch_desc",
icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"),
status: "available",
},
{
@ -131,25 +156,11 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.AIRTABLE_CONNECTOR, "h-6 w-6"),
status: "available",
},
{
id: "luma-connector",
title: "Luma",
description: "luma_desc",
icon: getConnectorIcon(EnumConnectorName.LUMA_CONNECTOR, "h-6 w-6"),
status: "available",
},
{
id: "webcrawler-connector",
title: "Web Pages",
description: "webcrawler_desc",
icon: getConnectorIcon(EnumConnectorName.WEBCRAWLER_CONNECTOR, "h-6 w-6"),
status: "available",
},
],
},
{
id: "communication",
title: "communication",
id: "productivity",
title: "productivity",
connectors: [
{
id: "google-calendar-connector",
@ -165,6 +176,13 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.GOOGLE_GMAIL_CONNECTOR, "h-6 w-6"),
status: "available",
},
{
id: "luma-connector",
title: "Luma",
description: "luma_desc",
icon: getConnectorIcon(EnumConnectorName.LUMA_CONNECTOR, "h-6 w-6"),
status: "available",
},
{
id: "zoom",
title: "Zoom",

16
surfsense_web/hooks/use-connector-edit-page.ts
View file

@ -98,7 +98,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
LUMA_API_KEY: "",
ELASTICSEARCH_API_KEY: "",
FIRECRAWL_API_KEY: "",
INITIAL_URLS: ""
INITIAL_URLS: "",
},
});
@ -145,7 +145,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
LUMA_API_KEY: config.LUMA_API_KEY || "",
ELASTICSEARCH_API_KEY: config.ELASTICSEARCH_API_KEY || "",
FIRECRAWL_API_KEY: config.FIRECRAWL_API_KEY || "",
INITIAL_URLS: config.INITIAL_URLS || ""
INITIAL_URLS: config.INITIAL_URLS || "",
});
if (currentConnector.connector_type === "GITHUB_CONNECTOR") {
const savedRepos = config.repo_full_names || [];
@ -479,14 +479,18 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
formData.INITIAL_URLS !== originalConfig.INITIAL_URLS
) {
newConfig = {};
if (formData.FIRECRAWL_API_KEY && formData.FIRECRAWL_API_KEY.trim()) {
if (!formData.FIRECRAWL_API_KEY.startsWith("fc-")) {
toast.warning("Firecrawl API keys typically start with 'fc-'. Please verify your key.");
toast.warning(
"Firecrawl API keys typically start with 'fc-'. Please verify your key."
);
}
newConfig.FIRECRAWL_API_KEY = formData.FIRECRAWL_API_KEY.trim();
} else if (originalConfig.FIRECRAWL_API_KEY) {
toast.info("Firecrawl API key removed. Web crawler will use AsyncChromiumLoader as fallback.");
toast.info(
"Firecrawl API key removed. Web crawler will use AsyncChromiumLoader as fallback."
);
}
if (formData.INITIAL_URLS !== undefined) {
@ -592,7 +596,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
newlySavedConfig.ELASTICSEARCH_API_KEY || ""
);
} else if (connector.connector_type == "WEBCRAWLER_CONNECTOR") {
editForm.setValue("FIRECRAWL_API_KEY",newlySavedConfig.FIRECRAWL_API_KEY || "");
editForm.setValue("FIRECRAWL_API_KEY", newlySavedConfig.FIRECRAWL_API_KEY || "");
editForm.setValue("INITIAL_URLS", newlySavedConfig.INITIAL_URLS || "");
}
}

15
surfsense_web/messages/en.json
View file

@ -304,11 +304,14 @@
"add_connector": {
"title": "Connect Your Tools",
"subtitle": "Integrate with your favorite services to enhance your research capabilities.",
"search_engines": "Search Engines",
"team_chats": "Team Chats",
"web_search": "Web Search",
"messaging": "Messaging",
"project_management": "Project Management",
"knowledge_bases": "Knowledge Bases",
"communication": "Communication",
"documentation": "Documentation",
"development": "Development",
"databases": "Databases",
"productivity": "Productivity",
"web_crawling": "Web Crawling",
"connect": "Connect",
"coming_soon": "Coming Soon",
"connected": "Connected",
@ -328,11 +331,11 @@
"github_desc": "Connect a GitHub PAT to index code and docs from accessible repositories.",
"confluence_desc": "Connect to Confluence to search pages, comments and documentation.",
"airtable_desc": "Connect to Airtable to search records, tables and database content.",
"luma_desc": "Connect to Luma to search events",
"luma_desc": "Connect to Luma to search events, meetups and gatherings.",
"calendar_desc": "Connect to Google Calendar to search events, meetings and schedules.",
"gmail_desc": "Connect to your Gmail account to search through your emails.",
"zoom_desc": "Connect to Zoom to access meeting recordings and transcripts.",
"webcrawler_desc": "Crawl web pages"
"webcrawler_desc": "Crawl and index content from any public web pages."
},
"upload_documents": {
"title": "Upload Documents",

16
surfsense_web/messages/zh.json
View file

@ -304,11 +304,14 @@
"add_connector": {
"title": "连接您的工具",
"subtitle": "集成您喜欢的服务以增强研究能力。",
"search_engines": "搜索引擎",
"team_chats": "团队聊天",
"web_search": "网络搜索",
"messaging": "即时通讯",
"project_management": "项目管理",
"knowledge_bases": "知识库",
"communication": "通讯",
"documentation": "文档协作",
"development": "开发工具",
"databases": "数据库",
"productivity": "效率工具",
"web_crawling": "网页爬取",
"connect": "连接",
"coming_soon": "即将推出",
"connected": "已连接",
@ -328,10 +331,11 @@
"github_desc": "连接 GitHub PAT 以索引可访问存储库的代码和文档。",
"confluence_desc": "连接到 Confluence 以搜索页面、评论和文档。",
"airtable_desc": "连接到 Airtable 以搜索记录、表格和数据库内容。",
"luma_desc": "连接到 Luma 以搜索活动",
"luma_desc": "连接到 Luma 以搜索活动、聚会和集会。",
"calendar_desc": "连接到 Google 日历以搜索活动、会议和日程。",
"gmail_desc": "连接到您的 Gmail 账户以搜索您的电子邮件。",
"zoom_desc": "连接到 Zoom 以访问会议录制和转录。"
"zoom_desc": "连接到 Zoom 以访问会议录制和转录。",
"webcrawler_desc": "爬取和索引任何公开网页的内容。"
},
"upload_documents": {
"title": "上传文档",