@@ -331,4 +328,4 @@ export default function WebcrawlerConnectorPage() {
);
-}
\ No newline at end of file
+}
diff --git a/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx
index 5172a0fa8..335d6d235 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/sources/add/page.tsx
@@ -1,9 +1,9 @@
"use client";
import { IconBrandYoutube } from "@tabler/icons-react";
-import { Cable, Database, Upload } from "lucide-react";
+import { Cable, Database, Globe, Upload } from "lucide-react";
import { motion } from "motion/react";
-import { useParams, useSearchParams } from "next/navigation";
+import { useParams, useRouter, useSearchParams } from "next/navigation";
import { useEffect, useState } from "react";
import { ConnectorsTab } from "@/components/sources/ConnectorsTab";
import { DocumentUploadTab } from "@/components/sources/DocumentUploadTab";
@@ -12,6 +12,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
export default function AddSourcesPage() {
const params = useParams();
+ const router = useRouter();
const searchParams = useSearchParams();
const search_space_id = params.search_space_id as string;
const [activeTab, setActiveTab] = useState("documents");
@@ -24,6 +25,14 @@ export default function AddSourcesPage() {
}
}, [searchParams]);
+ const handleTabChange = (value: string) => {
+ if (value === "webpages") {
+ router.push(`/dashboard/${search_space_id}/connectors/add/webcrawler-connector`);
+ } else {
+ setActiveTab(value);
+ }
+ };
+
return (
{/* Tabs */}
-
-
+
+
- Documents
+ Documents
+ Docs
YouTube
+
+
+ Web Pages
+ Web
+
- Connectors
+ Connectors
+ More
diff --git a/surfsense_web/components/editConnector/types.ts b/surfsense_web/components/editConnector/types.ts
index f61eaaf54..490474222 100644
--- a/surfsense_web/components/editConnector/types.ts
+++ b/surfsense_web/components/editConnector/types.ts
@@ -53,6 +53,6 @@ export const editConnectorSchema = z.object({
LUMA_API_KEY: z.string().optional(),
ELASTICSEARCH_API_KEY: z.string().optional(),
FIRECRAWL_API_KEY: z.string().optional(),
- INITIAL_URLS: z.string().optional()
+ INITIAL_URLS: z.string().optional(),
});
export type EditConnectorFormValues = z.infer;
diff --git a/surfsense_web/components/homepage/integrations.tsx b/surfsense_web/components/homepage/integrations.tsx
index afa09db60..0a5d93a98 100644
--- a/surfsense_web/components/homepage/integrations.tsx
+++ b/surfsense_web/components/homepage/integrations.tsx
@@ -29,7 +29,7 @@ const INTEGRATIONS: Integration[] = [
// Documentation & Knowledge
{ name: "Confluence", icon: "https://cdn.simpleicons.org/confluence/172B4D" },
{ name: "Notion", icon: "https://cdn.simpleicons.org/notion/000000/ffffff" },
- { name: "Web Pages", icon: "https://cdn.jsdelivr.net/npm/lucide-static@0.294.0/icons/globe.svg"},
+ { name: "Web Pages", icon: "https://cdn.jsdelivr.net/npm/lucide-static@0.294.0/icons/globe.svg" },
// Cloud Storage
{ name: "Google Drive", icon: "https://cdn.simpleicons.org/googledrive/4285F4" },
diff --git a/surfsense_web/components/sources/ConnectorsTab.tsx b/surfsense_web/components/sources/ConnectorsTab.tsx
index b8d3486f6..c9640d09b 100644
--- a/surfsense_web/components/sources/ConnectorsTab.tsx
+++ b/surfsense_web/components/sources/ConnectorsTab.tsx
@@ -19,11 +19,14 @@ interface ConnectorsTabProps {
export function ConnectorsTab({ searchSpaceId }: ConnectorsTabProps) {
const t = useTranslations("add_connector");
const [expandedCategories, setExpandedCategories] = useState([
- "search-engines",
- "knowledge-bases",
+ "web-search",
+ "messaging",
"project-management",
- "team-chats",
- "communication",
+ "documentation",
+ "development",
+ "databases",
+ "productivity",
+ "web-crawling",
]);
const toggleCategory = (categoryId: string) => {
diff --git a/surfsense_web/components/sources/connector-data.tsx b/surfsense_web/components/sources/connector-data.tsx
index e96db2a4e..631bb8606 100644
--- a/surfsense_web/components/sources/connector-data.tsx
+++ b/surfsense_web/components/sources/connector-data.tsx
@@ -5,8 +5,21 @@ import type { ConnectorCategory } from "./types";
export const connectorCategories: ConnectorCategory[] = [
{
- id: "search-engines",
- title: "search_engines",
+ id: "web-crawling",
+ title: "web_crawling",
+ connectors: [
+ {
+ id: "webcrawler-connector",
+ title: "Web Pages",
+ description: "webcrawler_desc",
+ icon: getConnectorIcon(EnumConnectorName.WEBCRAWLER_CONNECTOR, "h-6 w-6"),
+ status: "available",
+ },
+ ],
+ },
+ {
+ id: "web-search",
+ title: "web_search",
connectors: [
{
id: "tavily-api",
@@ -29,13 +42,6 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.LINKUP_API, "h-6 w-6"),
status: "available",
},
- {
- id: "elasticsearch-connector",
- title: "Elasticsearch",
- description: "elasticsearch_desc",
- icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"),
- status: "available",
- },
{
id: "baidu-search-api",
title: "Baidu Search",
@@ -46,8 +52,8 @@ export const connectorCategories: ConnectorCategory[] = [
],
},
{
- id: "team-chats",
- title: "team_chats",
+ id: "messaging",
+ title: "messaging",
connectors: [
{
id: "slack-connector",
@@ -56,13 +62,6 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.SLACK_CONNECTOR, "h-6 w-6"),
status: "available",
},
- {
- id: "ms-teams",
- title: "Microsoft Teams",
- description: "teams_desc",
- icon: ,
- status: "coming-soon",
- },
{
id: "discord-connector",
title: "Discord",
@@ -70,6 +69,13 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.DISCORD_CONNECTOR, "h-6 w-6"),
status: "available",
},
+ {
+ id: "ms-teams",
+ title: "Microsoft Teams",
+ description: "teams_desc",
+ icon: ,
+ status: "coming-soon",
+ },
],
},
{
@@ -100,8 +106,8 @@ export const connectorCategories: ConnectorCategory[] = [
],
},
{
- id: "knowledge-bases",
- title: "knowledge_bases",
+ id: "documentation",
+ title: "documentation",
connectors: [
{
id: "notion-connector",
@@ -110,6 +116,19 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.NOTION_CONNECTOR, "h-6 w-6"),
status: "available",
},
+ {
+ id: "confluence-connector",
+ title: "Confluence",
+ description: "confluence_desc",
+ icon: getConnectorIcon(EnumConnectorName.CONFLUENCE_CONNECTOR, "h-6 w-6"),
+ status: "available",
+ },
+ ],
+ },
+ {
+ id: "development",
+ title: "development",
+ connectors: [
{
id: "github-connector",
title: "GitHub",
@@ -117,11 +136,17 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.GITHUB_CONNECTOR, "h-6 w-6"),
status: "available",
},
+ ],
+ },
+ {
+ id: "databases",
+ title: "databases",
+ connectors: [
{
- id: "confluence-connector",
- title: "Confluence",
- description: "confluence_desc",
- icon: getConnectorIcon(EnumConnectorName.CONFLUENCE_CONNECTOR, "h-6 w-6"),
+ id: "elasticsearch-connector",
+ title: "Elasticsearch",
+ description: "elasticsearch_desc",
+ icon: getConnectorIcon(EnumConnectorName.ELASTICSEARCH_CONNECTOR, "h-6 w-6"),
status: "available",
},
{
@@ -131,25 +156,11 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.AIRTABLE_CONNECTOR, "h-6 w-6"),
status: "available",
},
- {
- id: "luma-connector",
- title: "Luma",
- description: "luma_desc",
- icon: getConnectorIcon(EnumConnectorName.LUMA_CONNECTOR, "h-6 w-6"),
- status: "available",
- },
- {
- id: "webcrawler-connector",
- title: "Web Pages",
- description: "webcrawler_desc",
- icon: getConnectorIcon(EnumConnectorName.WEBCRAWLER_CONNECTOR, "h-6 w-6"),
- status: "available",
- },
],
},
{
- id: "communication",
- title: "communication",
+ id: "productivity",
+ title: "productivity",
connectors: [
{
id: "google-calendar-connector",
@@ -165,6 +176,13 @@ export const connectorCategories: ConnectorCategory[] = [
icon: getConnectorIcon(EnumConnectorName.GOOGLE_GMAIL_CONNECTOR, "h-6 w-6"),
status: "available",
},
+ {
+ id: "luma-connector",
+ title: "Luma",
+ description: "luma_desc",
+ icon: getConnectorIcon(EnumConnectorName.LUMA_CONNECTOR, "h-6 w-6"),
+ status: "available",
+ },
{
id: "zoom",
title: "Zoom",
diff --git a/surfsense_web/hooks/use-connector-edit-page.ts b/surfsense_web/hooks/use-connector-edit-page.ts
index cd2ba805d..039819dcf 100644
--- a/surfsense_web/hooks/use-connector-edit-page.ts
+++ b/surfsense_web/hooks/use-connector-edit-page.ts
@@ -98,7 +98,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
LUMA_API_KEY: "",
ELASTICSEARCH_API_KEY: "",
FIRECRAWL_API_KEY: "",
- INITIAL_URLS: ""
+ INITIAL_URLS: "",
},
});
@@ -145,7 +145,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
LUMA_API_KEY: config.LUMA_API_KEY || "",
ELASTICSEARCH_API_KEY: config.ELASTICSEARCH_API_KEY || "",
FIRECRAWL_API_KEY: config.FIRECRAWL_API_KEY || "",
- INITIAL_URLS: config.INITIAL_URLS || ""
+ INITIAL_URLS: config.INITIAL_URLS || "",
});
if (currentConnector.connector_type === "GITHUB_CONNECTOR") {
const savedRepos = config.repo_full_names || [];
@@ -479,14 +479,18 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
formData.INITIAL_URLS !== originalConfig.INITIAL_URLS
) {
newConfig = {};
-
+
if (formData.FIRECRAWL_API_KEY && formData.FIRECRAWL_API_KEY.trim()) {
if (!formData.FIRECRAWL_API_KEY.startsWith("fc-")) {
- toast.warning("Firecrawl API keys typically start with 'fc-'. Please verify your key.");
+ toast.warning(
+ "Firecrawl API keys typically start with 'fc-'. Please verify your key."
+ );
}
newConfig.FIRECRAWL_API_KEY = formData.FIRECRAWL_API_KEY.trim();
} else if (originalConfig.FIRECRAWL_API_KEY) {
- toast.info("Firecrawl API key removed. Web crawler will use AsyncChromiumLoader as fallback.");
+ toast.info(
+ "Firecrawl API key removed. Web crawler will use AsyncChromiumLoader as fallback."
+ );
}
if (formData.INITIAL_URLS !== undefined) {
@@ -592,7 +596,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string)
newlySavedConfig.ELASTICSEARCH_API_KEY || ""
);
} else if (connector.connector_type == "WEBCRAWLER_CONNECTOR") {
- editForm.setValue("FIRECRAWL_API_KEY",newlySavedConfig.FIRECRAWL_API_KEY || "");
+ editForm.setValue("FIRECRAWL_API_KEY", newlySavedConfig.FIRECRAWL_API_KEY || "");
editForm.setValue("INITIAL_URLS", newlySavedConfig.INITIAL_URLS || "");
}
}
diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json
index 55e25bee1..758e558b3 100644
--- a/surfsense_web/messages/en.json
+++ b/surfsense_web/messages/en.json
@@ -304,11 +304,14 @@
"add_connector": {
"title": "Connect Your Tools",
"subtitle": "Integrate with your favorite services to enhance your research capabilities.",
- "search_engines": "Search Engines",
- "team_chats": "Team Chats",
+ "web_search": "Web Search",
+ "messaging": "Messaging",
"project_management": "Project Management",
- "knowledge_bases": "Knowledge Bases",
- "communication": "Communication",
+ "documentation": "Documentation",
+ "development": "Development",
+ "databases": "Databases",
+ "productivity": "Productivity",
+ "web_crawling": "Web Crawling",
"connect": "Connect",
"coming_soon": "Coming Soon",
"connected": "Connected",
@@ -328,11 +331,11 @@
"github_desc": "Connect a GitHub PAT to index code and docs from accessible repositories.",
"confluence_desc": "Connect to Confluence to search pages, comments and documentation.",
"airtable_desc": "Connect to Airtable to search records, tables and database content.",
- "luma_desc": "Connect to Luma to search events",
+ "luma_desc": "Connect to Luma to search events, meetups and gatherings.",
"calendar_desc": "Connect to Google Calendar to search events, meetings and schedules.",
"gmail_desc": "Connect to your Gmail account to search through your emails.",
"zoom_desc": "Connect to Zoom to access meeting recordings and transcripts.",
- "webcrawler_desc": "Crawl web pages"
+ "webcrawler_desc": "Crawl and index content from any public web pages."
},
"upload_documents": {
"title": "Upload Documents",
diff --git a/surfsense_web/messages/zh.json b/surfsense_web/messages/zh.json
index 11880a2e8..857649b80 100644
--- a/surfsense_web/messages/zh.json
+++ b/surfsense_web/messages/zh.json
@@ -304,11 +304,14 @@
"add_connector": {
"title": "连接您的工具",
"subtitle": "集成您喜欢的服务以增强研究能力。",
- "search_engines": "搜索引擎",
- "team_chats": "团队聊天",
+ "web_search": "网络搜索",
+ "messaging": "即时通讯",
"project_management": "项目管理",
- "knowledge_bases": "知识库",
- "communication": "通讯",
+ "documentation": "文档协作",
+ "development": "开发工具",
+ "databases": "数据库",
+ "productivity": "效率工具",
+ "web_crawling": "网页爬取",
"connect": "连接",
"coming_soon": "即将推出",
"connected": "已连接",
@@ -328,10 +331,11 @@
"github_desc": "连接 GitHub PAT 以索引可访问存储库的代码和文档。",
"confluence_desc": "连接到 Confluence 以搜索页面、评论和文档。",
"airtable_desc": "连接到 Airtable 以搜索记录、表格和数据库内容。",
- "luma_desc": "连接到 Luma 以搜索活动",
+ "luma_desc": "连接到 Luma 以搜索活动、聚会和集会。",
"calendar_desc": "连接到 Google 日历以搜索活动、会议和日程。",
"gmail_desc": "连接到您的 Gmail 账户以搜索您的电子邮件。",
- "zoom_desc": "连接到 Zoom 以访问会议录制和转录。"
+ "zoom_desc": "连接到 Zoom 以访问会议录制和转录。",
+ "webcrawler_desc": "爬取和索引任何公开网页的内容。"
},
"upload_documents": {
"title": "上传文档",