diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/webcrawler-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/webcrawler-connector/page.tsx deleted file mode 100644 index 0b7e811d0..000000000 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/webcrawler-connector/page.tsx +++ /dev/null @@ -1,325 +0,0 @@ -"use client"; - -import { zodResolver } from "@hookform/resolvers/zod"; -import { useAtomValue } from "jotai"; -import { ArrowLeft, Check, Globe, Loader2 } from "lucide-react"; -import { motion } from "motion/react"; -import Link from "next/link"; -import { useParams, useRouter } from "next/navigation"; -import { useEffect, useState } from "react"; -import { useForm } from "react-hook-form"; -import { toast } from "sonner"; -import * as z from "zod"; -import { createConnectorMutationAtom } from "@/atoms/connectors/connector-mutation.atoms"; -import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; -import { Button } from "@/components/ui/button"; -import { - Card, - CardContent, - CardDescription, - CardFooter, - CardHeader, - CardTitle, -} from "@/components/ui/card"; -import { - Form, - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage, -} from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; -import { Textarea } from "@/components/ui/textarea"; -import { EnumConnectorName } from "@/contracts/enums/connector"; -import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import type { SearchSourceConnector } from "@/contracts/types/connector.types"; - -// Define the form schema with Zod -const webcrawlerConnectorFormSchema = z.object({ - name: z.string().min(3, { - message: "Connector name must be at least 3 characters.", - }), - api_key: z.string().optional(), - initial_urls: z.string().optional(), -}); - -// Define the type for the form values -type WebcrawlerConnectorFormValues = z.infer; - -export default function WebcrawlerConnectorPage() { - const router = useRouter(); - const params = useParams(); - const searchSpaceId = params.search_space_id as string; - const [isSubmitting, setIsSubmitting] = useState(false); - const [doesConnectorExist, setDoesConnectorExist] = useState(false); - - const { refetch: fetchConnectors } = useAtomValue(connectorsAtom); - const { mutateAsync: createConnector } = useAtomValue(createConnectorMutationAtom); - - // Initialize the form - const form = useForm({ - resolver: zodResolver(webcrawlerConnectorFormSchema), - defaultValues: { - name: "Web Pages", - api_key: "", - initial_urls: "", - }, - }); - - useEffect(() => { - fetchConnectors().then((data) => { - const connectors = data.data || []; - const connector = connectors.find( - (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.WEBCRAWLER_CONNECTOR - ); - if (connector) { - setDoesConnectorExist(true); - } - }); - }, []); - - // Handle form submission - const onSubmit = async (values: WebcrawlerConnectorFormValues) => { - setIsSubmitting(true); - try { - const config: Record = {}; - - // Only add API key to config if provided - if (values.api_key && values.api_key.trim()) { - config.FIRECRAWL_API_KEY = values.api_key; - } - - // Parse initial URLs if provided - if (values.initial_urls && values.initial_urls.trim()) { - config.INITIAL_URLS = values.initial_urls; - } - - await createConnector({ - data: { - name: values.name, - connector_type: EnumConnectorName.WEBCRAWLER_CONNECTOR, - config: config, - is_indexable: true, - last_indexed_at: null, - periodic_indexing_enabled: false, - indexing_frequency_minutes: null, - next_scheduled_at: null, - }, - queryParams: { - search_space_id: searchSpaceId, - }, - }); - - toast.success("Webcrawler connector created successfully!"); - - // Navigate back to connectors page - router.push(`/dashboard/${searchSpaceId}/connectors`); - } catch (error) { - console.error("Error creating connector:", error); - toast.error(error instanceof Error ? error.message : "Failed to create connector"); - } finally { - setIsSubmitting(false); - } - }; - - return ( -
- - {/* Header */} -
- - - Back to connectors - -
-
- {getConnectorIcon(EnumConnectorName.WEBCRAWLER_CONNECTOR, "h-6 w-6")} -
-
-

Connect Web Pages

-

Crawl and index web pages for search.

-
-
-
- - {/* Connection Card */} - {!doesConnectorExist ? ( - - - Set Up Web Page crawler - - Configure your web page crawler to index web pages. Optionally add a Firecrawl API - key for enhanced crawling capabilities. - - -
- - - ( - - Connector Name - - - - - A friendly name to identify this connector. - - - - )} - /> - - ( - - Firecrawl API Key (Optional) - - - - - Add a Firecrawl API key for enhanced crawling. If not provided, will use - AsyncChromiumLoader as fallback. - - - - )} - /> - - ( - - Initial URLs (Optional) - -