feat(UI): reorganized connectors

surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py

@@ -187,6 +187,7 @@ async def index_crawled_urls(
                 )
 
                 # Generate content hash
+                # TODO: To fix this by not including dynamic content like date, time, etc.
                 content_hash = generate_content_hash(
                     structured_document, search_space_id
                 )

surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx

@@ -316,7 +316,8 @@ export default function EditConnectorPage() {
 														/>
 													</FormControl>
 													<FormDescription>
-														Enter URLs to crawl (one per line). These URLs will be indexed when you trigger indexing.
+														Enter URLs to crawl (one per line). These URLs will be indexed when you
+														trigger indexing.
 													</FormDescription>
 													<FormMessage />
 												</FormItem>
@@ -324,7 +325,6 @@ export default function EditConnectorPage() {
 										/>
 									</div>
 								)}
-								
 							</CardContent>
 							<CardFooter className="border-t pt-6">
 								<Button type="submit" disabled={isSaving} className="w-full sm:w-auto">

surfsense_web/app/dashboard/[search_space_id]/connectors/add/webcrawler-connector/page.tsx

@@ -75,7 +75,8 @@ export default function WebcrawlerConnectorPage() {
 			.then((data) => {
 				if (data && Array.isArray(data)) {
 					const connector = data.find(
-						(c: SearchSourceConnector) => c.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR
+						(c: SearchSourceConnector) =>
+							c.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR
 					);
 					if (connector) {
 						setDoesConnectorExist(true);
@@ -92,7 +93,7 @@ export default function WebcrawlerConnectorPage() {
 		setIsSubmitting(true);
 		try {
 			const config: Record<string, string> = {};
-			
+
 			// Only add API key to config if provided
 			if (values.api_key && values.api_key.trim()) {
 				config.FIRECRAWL_API_KEY = values.api_key;
@@ -162,8 +163,8 @@ export default function WebcrawlerConnectorPage() {
 						<CardHeader>
 							<CardTitle>Set Up Web Page crawler</CardTitle>
 							<CardDescription>
-								Configure your web page crawler to index web pages. Optionally add a Firecrawl API key
+								Configure your web page crawler to index web pages. Optionally add a Firecrawl API
-								for enhanced crawling capabilities.
+								key for enhanced crawling capabilities.
 							</CardDescription>
 						</CardHeader>
 						<Form {...form}>
@@ -193,11 +194,7 @@ export default function WebcrawlerConnectorPage() {
 											<FormItem>
 												<FormLabel>Firecrawl API Key (Optional)</FormLabel>
 												<FormControl>
-													<Input 
+													<Input type="password" placeholder="fc-xxxxxxxxxxxxx" {...field} />
-														type="password" 
-														placeholder="fc-xxxxxxxxxxxxx" 
-														{...field} 
-													/>
 												</FormControl>
 												<FormDescription>
 													Add a Firecrawl API key for enhanced crawling. If not provided, will use
@@ -215,10 +212,10 @@ export default function WebcrawlerConnectorPage() {
 											<FormItem>
 												<FormLabel>Initial URLs (Optional)</FormLabel>
 												<FormControl>
-													<Textarea 
+													<Textarea
 														placeholder="https://example.com&#10;https://docs.example.com&#10;https://blog.example.com"
 														className="min-h-[100px] font-mono text-sm"
-														{...field} 
+														{...field}
 													/>
 												</FormControl>
 												<FormDescription>
@@ -296,9 +293,9 @@ export default function WebcrawlerConnectorPage() {
 								<h4 className="font-medium mb-2">1. Choose Your Crawler Method</h4>
 								<p className="text-sm text-muted-foreground">
 									<strong>With Firecrawl (Recommended):</strong> Get your API key from{" "}
-									<a 
+									<a
-										href="https://firecrawl.dev" 
+										href="https://firecrawl.dev"
-										target="_blank" 
+										target="_blank"
 										rel="noopener noreferrer"
 										className="text-primary hover:underline"
 									>
@@ -331,4 +328,4 @@ export default function WebcrawlerConnectorPage() {
 			</motion.div>
 		</div>
 	);
-}
+}

surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx

@@ -1,9 +1,9 @@
 "use client";
 
 import { IconBrandYoutube } from "@tabler/icons-react";
-import { Cable, Database, Upload } from "lucide-react";
+import { Cable, Database, Globe, Upload } from "lucide-react";
 import { motion } from "motion/react";
-import { useParams, useSearchParams } from "next/navigation";
+import { useParams, useRouter, useSearchParams } from "next/navigation";
 import { useEffect, useState } from "react";
 import { ConnectorsTab } from "@/components/sources/ConnectorsTab";
 import { DocumentUploadTab } from "@/components/sources/DocumentUploadTab";
@@ -12,6 +12,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 
 export default function AddSourcesPage() {
 	const params = useParams();
+	const router = useRouter();
 	const searchParams = useSearchParams();
 	const search_space_id = params.search_space_id as string;
 	const [activeTab, setActiveTab] = useState("documents");
@@ -24,6 +25,14 @@ export default function AddSourcesPage() {
 		}
 	}, [searchParams]);
 
+	const handleTabChange = (value: string) => {
+		if (value === "webpages") {
+			router.push(`/dashboard/${search_space_id}/connectors/add/webcrawler-connector`);
+		} else {
+			setActiveTab(value);
+		}
+	};
+
 	return (
 		<div className="container mx-auto py-8 px-4">
 			<motion.div
@@ -42,19 +51,26 @@ export default function AddSourcesPage() {
 				</div>
 
 				{/* Tabs */}
-				<Tabs value={activeTab} onValueChange={setActiveTab} className="w-full">
+				<Tabs value={activeTab} onValueChange={handleTabChange} className="w-full">
-					<TabsList className="grid w-full max-w-2xl mx-auto grid-cols-3 h-12">
+					<TabsList className="grid w-full max-w-3xl mx-auto grid-cols-4 h-12">
 						<TabsTrigger value="documents" className="flex items-center gap-2">
 							<Upload className="h-4 w-4" />
-							Documents
+							<span className="hidden sm:inline">Documents</span>
+							<span className="sm:hidden">Docs</span>
 						</TabsTrigger>
 						<TabsTrigger value="youtube" className="flex items-center gap-2">
 							<IconBrandYoutube className="h-4 w-4" />
 							YouTube
 						</TabsTrigger>
+						<TabsTrigger value="webpages" className="flex items-center gap-2">
+							<Globe className="h-4 w-4" />
+							<span className="hidden sm:inline">Web Pages</span>
+							<span className="sm:hidden">Web</span>
+						</TabsTrigger>
 						<TabsTrigger value="connectors" className="flex items-center gap-2">
 							<Cable className="h-4 w-4" />
-							Connectors
+							<span className="hidden sm:inline">Connectors</span>
+							<span className="sm:hidden">More</span>
 						</TabsTrigger>
 					</TabsList>
 

surfsense_web/components/editConnector/types.ts

@@ -53,6 +53,6 @@ export const editConnectorSchema = z.object({
 	LUMA_API_KEY: z.string().optional(),
 	ELASTICSEARCH_API_KEY: z.string().optional(),
 	FIRECRAWL_API_KEY: z.string().optional(),
-	INITIAL_URLS: z.string().optional()
+	INITIAL_URLS: z.string().optional(),
 });
 export type EditConnectorFormValues = z.infer<typeof editConnectorSchema>;

surfsense_web/components/homepage/integrations.tsx

@@ -29,7 +29,7 @@ const INTEGRATIONS: Integration[] = [
 	// Documentation & Knowledge
 	{ name: "Confluence", icon: "https://cdn.simpleicons.org/confluence/172B4D" },
 	{ name: "Notion", icon: "https://cdn.simpleicons.org/notion/000000/ffffff" },
-	{ name: "Web Pages", icon: "https://cdn.jsdelivr.net/npm/lucide-static@0.294.0/icons/globe.svg"},
+	{ name: "Web Pages", icon: "https://cdn.jsdelivr.net/npm/lucide-static@0.294.0/icons/globe.svg" },
 
 	// Cloud Storage
 	{ name: "Google Drive", icon: "https://cdn.simpleicons.org/googledrive/4285F4" },

surfsense_web/components/sources/ConnectorsTab.tsx

@@ -19,11 +19,14 @@ interface ConnectorsTabProps {
 export function ConnectorsTab({ searchSpaceId }: ConnectorsTabProps) {
 	const t = useTranslations("add_connector");
 	const [expandedCategories, setExpandedCategories] = useState<string[]>([
-		"search-engines",
+		"web-search",
-		"knowledge-bases",
+		"messaging",
 		"project-management",
-		"team-chats",
+		"documentation",
-		"communication",
+		"development",
+		"databases",
+		"productivity",
+		"web-crawling",
 	]);
 
 	const toggleCategory = (categoryId: string) => {

surfsense_web/components/sources/connector-data.tsx

@@ -5,8 +5,21 @@ import type { ConnectorCategory } from "./types";
 
 export const connectorCategories: ConnectorCategory[] = [
 	{
-		id: "search-engines",
+		id: "web-crawling",
-		title: "search_engines",
+		title: "web_crawling",
+		connectors: [
+			{
+				id: "webcrawler-connector",
+				title: "Web Pages",
+				description: "webcrawler_desc",
+				icon: getConnectorIcon(EnumConnectorName.WEBCRAWLER_CONNECTOR, "h-6 w-6"),
+				status: "available",
+			},
+		],
+	},
+	{
+		id: "web-search",
+		title: "web_search",
 		connectors: [
 			{
 				id: "tavily-api",
@@ -29,13 +42,6 @@ export const connectorCategories: ConnectorCategory[] = [
 				icon: getConnectorIcon(EnumConnectorName.LINKUP_API, "h-6 w-6"),
 				status: "available",
 			},
-			{
-				id: "elasticsearch-connector",
-				title: "Elasticsearch",
-				description: "elasticsearch_desc",
-				icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"),
-				status: "available",
-			},
 			{
 				id: "baidu-search-api",
 				title: "Baidu Search",
@@ -46,8 +52,8 @@ export const connectorCategories: ConnectorCategory[] = [
 		],
 	},
 	{
-		id: "team-chats",
+		id: "messaging",
-		title: "team_chats",
+		title: "messaging",
 		connectors: [
 			{
 				id: "slack-connector",
@@ -56,13 +62,6 @@ export const connectorCategories: ConnectorCategory[] = [
 				icon: getConnectorIcon(EnumConnectorName.SLACK_CONNECTOR, "h-6 w-6"),
 				status: "available",
 			},
-			{
-				id: "ms-teams",
-				title: "Microsoft Teams",
-				description: "teams_desc",
-				icon: <IconBrandWindows className="h-6 w-6" />,
-				status: "coming-soon",
-			},
 			{
 				id: "discord-connector",
 				title: "Discord",
@@ -70,6 +69,13 @@ export const connectorCategories: ConnectorCategory[] = [
 				icon: getConnectorIcon(EnumConnectorName.DISCORD_CONNECTOR, "h-6 w-6"),
 				status: "available",
 			},
+			{
+				id: "ms-teams",
+				title: "Microsoft Teams",
+				description: "teams_desc",
+				icon: <IconBrandWindows className="h-6 w-6" />,
+				status: "coming-soon",
+			},
 		],
 	},
 	{
@@ -100,8 +106,8 @@ export const connectorCategories: ConnectorCategory[] = [
 		],
 	},
 	{
-		id: "knowledge-bases",
+		id: "documentation",
-		title: "knowledge_bases",
+		title: "documentation",
 		connectors: [
 			{
 				id: "notion-connector",
@@ -110,6 +116,19 @@ export const connectorCategories: ConnectorCategory[] = [
 				icon: getConnectorIcon(EnumConnectorName.NOTION_CONNECTOR, "h-6 w-6"),
 				status: "available",
 			},
+			{
+				id: "confluence-connector",
+				title: "Confluence",
+				description: "confluence_desc",
+				icon: getConnectorIcon(EnumConnectorName.CONFLUENCE_CONNECTOR, "h-6 w-6"),
+				status: "available",
+			},
+		],
+	},
+	{
+		id: "development",
+		title: "development",
+		connectors: [
 			{
 				id: "github-connector",
 				title: "GitHub",
@@ -117,11 +136,17 @@ export const connectorCategories: ConnectorCategory[] = [
 				icon: getConnectorIcon(EnumConnectorName.GITHUB_CONNECTOR, "h-6 w-6"),
 				status: "available",
 			},
+		],
+	},
+	{
+		id: "databases",
+		title: "databases",
+		connectors: [
 			{
-				id: "confluence-connector",
+				id: "elasticsearch-connector",
-				title: "Confluence",
+				title: "Elasticsearch",
-				description: "confluence_desc",
+				description: "elasticsearch_desc",
-				icon: getConnectorIcon(EnumConnectorName.CONFLUENCE_CONNECTOR, "h-6 w-6"),
+				icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"),
 				status: "available",
 			},
 			{
@@ -131,25 +156,11 @@ export const connectorCategories: ConnectorCategory[] = [
 				icon: getConnectorIcon(EnumConnectorName.AIRTABLE_CONNECTOR, "h-6 w-6"),
 				status: "available",
 			},
-			{
-				id: "luma-connector",
-				title: "Luma",
-				description: "luma_desc",
-				icon: getConnectorIcon(EnumConnectorName.LUMA_CONNECTOR, "h-6 w-6"),
-				status: "available",
-			},
-			{
-				id: "webcrawler-connector",
-				title: "Web Pages",
-				description: "webcrawler_desc",
-				icon: getConnectorIcon(EnumConnectorName.WEBCRAWLER_CONNECTOR, "h-6 w-6"),
-				status: "available",
-			},
 		],
 	},
 	{
-		id: "communication",
+		id: "productivity",
-		title: "communication",
+		title: "productivity",
 		connectors: [
 			{
 				id: "google-calendar-connector",
@@ -165,6 +176,13 @@ export const connectorCategories: ConnectorCategory[] = [
 				icon: getConnectorIcon(EnumConnectorName.GOOGLE_GMAIL_CONNECTOR, "h-6 w-6"),
 				status: "available",
 			},
+			{
+				id: "luma-connector",
+				title: "Luma",
+				description: "luma_desc",
+				icon: getConnectorIcon(EnumConnectorName.LUMA_CONNECTOR, "h-6 w-6"),
+				status: "available",
+			},
 			{
 				id: "zoom",
 				title: "Zoom",

surfsense_web/hooks/use-connector-edit-page.ts

@@ -98,7 +98,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
 			LUMA_API_KEY: "",
 			ELASTICSEARCH_API_KEY: "",
 			FIRECRAWL_API_KEY: "",
-			INITIAL_URLS: ""
+			INITIAL_URLS: "",
 		},
 	});
 
@@ -145,7 +145,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
 					LUMA_API_KEY: config.LUMA_API_KEY || "",
 					ELASTICSEARCH_API_KEY: config.ELASTICSEARCH_API_KEY || "",
 					FIRECRAWL_API_KEY: config.FIRECRAWL_API_KEY || "",
-					INITIAL_URLS: config.INITIAL_URLS || ""
+					INITIAL_URLS: config.INITIAL_URLS || "",
 				});
 				if (currentConnector.connector_type === "GITHUB_CONNECTOR") {
 					const savedRepos = config.repo_full_names || [];
@@ -479,14 +479,18 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
 						formData.INITIAL_URLS !== originalConfig.INITIAL_URLS
 					) {
 						newConfig = {};
-						
+
 						if (formData.FIRECRAWL_API_KEY && formData.FIRECRAWL_API_KEY.trim()) {
 							if (!formData.FIRECRAWL_API_KEY.startsWith("fc-")) {
-								toast.warning("Firecrawl API keys typically start with 'fc-'. Please verify your key.");
+								toast.warning(
+									"Firecrawl API keys typically start with 'fc-'. Please verify your key."
+								);
 							}
 							newConfig.FIRECRAWL_API_KEY = formData.FIRECRAWL_API_KEY.trim();
 						} else if (originalConfig.FIRECRAWL_API_KEY) {
-							toast.info("Firecrawl API key removed. Web crawler will use AsyncChromiumLoader as fallback.");
+							toast.info(
+								"Firecrawl API key removed. Web crawler will use AsyncChromiumLoader as fallback."
+							);
 						}
 
 						if (formData.INITIAL_URLS !== undefined) {
@@ -592,7 +596,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
 							newlySavedConfig.ELASTICSEARCH_API_KEY || ""
 						);
 					} else if (connector.connector_type == "WEBCRAWLER_CONNECTOR") {
-						editForm.setValue("FIRECRAWL_API_KEY",newlySavedConfig.FIRECRAWL_API_KEY || "");
+						editForm.setValue("FIRECRAWL_API_KEY", newlySavedConfig.FIRECRAWL_API_KEY || "");
 						editForm.setValue("INITIAL_URLS", newlySavedConfig.INITIAL_URLS || "");
 					}
 				}

surfsense_web/messages/en.json

@@ -304,11 +304,14 @@
 	"add_connector": {
 		"title": "Connect Your Tools",
 		"subtitle": "Integrate with your favorite services to enhance your research capabilities.",
-		"search_engines": "Search Engines",
+		"web_search": "Web Search",
-		"team_chats": "Team Chats",
+		"messaging": "Messaging",
 		"project_management": "Project Management",
-		"knowledge_bases": "Knowledge Bases",
+		"documentation": "Documentation",
-		"communication": "Communication",
+		"development": "Development",
+		"databases": "Databases",
+		"productivity": "Productivity",
+		"web_crawling": "Web Crawling",
 		"connect": "Connect",
 		"coming_soon": "Coming Soon",
 		"connected": "Connected",
@@ -328,11 +331,11 @@
 		"github_desc": "Connect a GitHub PAT to index code and docs from accessible repositories.",
 		"confluence_desc": "Connect to Confluence to search pages, comments and documentation.",
 		"airtable_desc": "Connect to Airtable to search records, tables and database content.",
-		"luma_desc": "Connect to Luma to search events",
+		"luma_desc": "Connect to Luma to search events, meetups and gatherings.",
 		"calendar_desc": "Connect to Google Calendar to search events, meetings and schedules.",
 		"gmail_desc": "Connect to your Gmail account to search through your emails.",
 		"zoom_desc": "Connect to Zoom to access meeting recordings and transcripts.",
-		"webcrawler_desc": "Crawl web pages"
+		"webcrawler_desc": "Crawl and index content from any public web pages."
 	},
 	"upload_documents": {
 		"title": "Upload Documents",

surfsense_web/messages/zh.json

@@ -304,11 +304,14 @@
 	"add_connector": {
 		"title": "连接您的工具",
 		"subtitle": "集成您喜欢的服务以增强研究能力。",
-		"search_engines": "搜索引擎",
+		"web_search": "网络搜索",
-		"team_chats": "团队聊天",
+		"messaging": "即时通讯",
 		"project_management": "项目管理",
-		"knowledge_bases": "知识库",
+		"documentation": "文档协作",
-		"communication": "通讯",
+		"development": "开发工具",
+		"databases": "数据库",
+		"productivity": "效率工具",
+		"web_crawling": "网页爬取",
 		"connect": "连接",
 		"coming_soon": "即将推出",
 		"connected": "已连接",
@@ -328,10 +331,11 @@
 		"github_desc": "连接 GitHub PAT 以索引可访问存储库的代码和文档。",
 		"confluence_desc": "连接到 Confluence 以搜索页面、评论和文档。",
 		"airtable_desc": "连接到 Airtable 以搜索记录、表格和数据库内容。",
-		"luma_desc": "连接到 Luma 以搜索活动",
+		"luma_desc": "连接到 Luma 以搜索活动、聚会和集会。",
 		"calendar_desc": "连接到 Google 日历以搜索活动、会议和日程。",
 		"gmail_desc": "连接到您的 Gmail 账户以搜索您的电子邮件。",
-		"zoom_desc": "连接到 Zoom 以访问会议录制和转录。"
+		"zoom_desc": "连接到 Zoom 以访问会议录制和转录。",
+		"webcrawler_desc": "爬取和索引任何公开网页的内容。"
 	},
 	"upload_documents": {
 		"title": "上传文档",