mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-06-06 20:15:17 +02:00
Renaming resources
This commit is contained in:
samkul-swe
parent
6d19e0fad8
commit
121e2f0c0e
24 changed files with 117 additions and 273 deletions
|
|
@ -18,7 +18,16 @@ import {
|
|||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import { Form } from "@/components/ui/form";
|
||||
import {
|
||||
Form,
|
||||
FormControl,
|
||||
FormDescription,
|
||||
FormField,
|
||||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
} from "@/components/ui/form";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
|
||||
import { useConnectorEditPage } from "@/hooks/use-connector-edit-page";
|
||||
// Import Utils, Types, Hook, and Components
|
||||
|
|
@ -285,13 +294,35 @@ export default function EditConnectorPage() {
|
|||
|
||||
{/* == Webcrawler == */}
|
||||
{connector.connector_type === "WEBCRAWLER_CONNECTOR" && (
|
||||
<EditSimpleTokenForm
|
||||
control={editForm.control}
|
||||
fieldName="FIRECRAWL_API_KEY"
|
||||
fieldLabel="Firecrawl API Key (Optional)"
|
||||
fieldDescription="Add a Firecrawl API key for enhanced crawling capabilities. If not provided, will use AsyncChromiumLoader as fallback."
|
||||
placeholder="fc-xxxxxxxxxxxxx"
|
||||
/>
|
||||
<div className="space-y-4">
|
||||
<EditSimpleTokenForm
|
||||
control={editForm.control}
|
||||
fieldName="FIRECRAWL_API_KEY"
|
||||
fieldLabel="Firecrawl API Key (Optional)"
|
||||
fieldDescription="Add a Firecrawl API key for enhanced crawling capabilities. If not provided, will use AsyncChromiumLoader as fallback."
|
||||
placeholder="fc-xxxxxxxxxxxxx"
|
||||
/>
|
||||
<FormField
|
||||
control={editForm.control}
|
||||
name="INITIAL_URLS"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>URLs to Crawl</FormLabel>
|
||||
<FormControl>
|
||||
<Textarea
|
||||
placeholder="https://example.com https://docs.example.com https://blog.example.com"
|
||||
className="min-h-[150px] font-mono text-sm"
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
Enter URLs to crawl (one per line). These URLs will be indexed when you trigger indexing.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
</CardContent>
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ const getConnectorTypeDisplay = (type: string): string => {
|
|||
AIRTABLE_CONNECTOR: "Airtable Connector",
|
||||
LUMA_CONNECTOR: "Luma Connector",
|
||||
ELASTICSEARCH_CONNECTOR: "Elasticsearch Connector",
|
||||
WEBCRAWLER_CONNECTOR: "Web Crawler Connector",
|
||||
WEBCRAWLER_CONNECTOR: "Web Page Connector",
|
||||
// Add other connector types here as needed
|
||||
};
|
||||
return typeMap[type] || type;
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ export default function WebcrawlerConnectorPage() {
|
|||
const form = useForm<WebcrawlerConnectorFormValues>({
|
||||
resolver: zodResolver(webcrawlerConnectorFormSchema),
|
||||
defaultValues: {
|
||||
name: "Web Crawler",
|
||||
name: "Web Pages",
|
||||
api_key: "",
|
||||
initial_urls: "",
|
||||
},
|
||||
|
|
@ -150,7 +150,7 @@ export default function WebcrawlerConnectorPage() {
|
|||
{getConnectorIcon(EnumConnectorName.WEBCRAWLER_CONNECTOR, "h-6 w-6")}
|
||||
</div>
|
||||
<div>
|
||||
<h1 className="text-3xl font-bold tracking-tight">Connect Web Crawler</h1>
|
||||
<h1 className="text-3xl font-bold tracking-tight">Connect Web Pages</h1>
|
||||
<p className="text-muted-foreground">Crawl and index web pages for search.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -160,9 +160,9 @@ export default function WebcrawlerConnectorPage() {
|
|||
{!doesConnectorExist ? (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Set Up Web Crawler</CardTitle>
|
||||
<CardTitle>Set Up Web Page crawler</CardTitle>
|
||||
<CardDescription>
|
||||
Configure your web crawler to index web pages. Optionally add a Firecrawl API key
|
||||
Configure your web page crawler to index web pages. Optionally add a Firecrawl API key
|
||||
for enhanced crawling capabilities.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
|
|
@ -277,7 +277,7 @@ export default function WebcrawlerConnectorPage() {
|
|||
/* Success Card */
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>✅ Your web crawler is successfully set up!</CardTitle>
|
||||
<CardTitle>✅ Your web page crawler is successfully set up!</CardTitle>
|
||||
<CardDescription>
|
||||
You can now add URLs to crawl from the connector management page.
|
||||
</CardDescription>
|
||||
|
|
|
|||
|
|
@ -1,201 +0,0 @@
|
|||
"use client";
|
||||
|
||||
import { type Tag, TagInput } from "emblor";
|
||||
import { Globe, Loader2 } from "lucide-react";
|
||||
import { useParams, useRouter } from "next/navigation";
|
||||
import { useTranslations } from "next-intl";
|
||||
import { useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardFooter,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import { Label } from "@/components/ui/label";
|
||||
|
||||
// URL validation regex
|
||||
const urlRegex = /^(https?:\/\/)?([\da-z.-]+)\.([a-z.]{2,6})([/\w .-]*)*\/?$/;
|
||||
|
||||
export default function WebpageCrawler() {
|
||||
const t = useTranslations("add_webpage");
|
||||
const params = useParams();
|
||||
const router = useRouter();
|
||||
const search_space_id = params.search_space_id as string;
|
||||
|
||||
const [urlTags, setUrlTags] = useState<Tag[]>([]);
|
||||
const [activeTagIndex, setActiveTagIndex] = useState<number | null>(null);
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
// Function to validate a URL
|
||||
const isValidUrl = (url: string): boolean => {
|
||||
return urlRegex.test(url);
|
||||
};
|
||||
|
||||
// Function to handle URL submission
|
||||
const handleSubmit = async () => {
|
||||
// Validate that we have at least one URL
|
||||
if (urlTags.length === 0) {
|
||||
setError(t("error_no_url"));
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate all URLs
|
||||
const invalidUrls = urlTags.filter((tag) => !isValidUrl(tag.text));
|
||||
if (invalidUrls.length > 0) {
|
||||
setError(t("error_invalid_urls", { urls: invalidUrls.map((tag) => tag.text).join(", ") }));
|
||||
return;
|
||||
}
|
||||
|
||||
setError(null);
|
||||
setIsSubmitting(true);
|
||||
|
||||
try {
|
||||
toast(t("crawling_toast"), {
|
||||
description: t("crawling_toast_desc"),
|
||||
});
|
||||
|
||||
// Extract URLs from tags
|
||||
const urls = urlTags.map((tag) => tag.text);
|
||||
|
||||
// Make API call to backend
|
||||
const response = await fetch(
|
||||
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
document_type: "CRAWLED_URL",
|
||||
content: urls,
|
||||
search_space_id: parseInt(search_space_id),
|
||||
}),
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error("Failed to crawl URLs");
|
||||
}
|
||||
|
||||
await response.json();
|
||||
|
||||
toast(t("success_toast"), {
|
||||
description: t("success_toast_desc"),
|
||||
});
|
||||
|
||||
// Redirect to documents page
|
||||
router.push(`/dashboard/${search_space_id}/documents`);
|
||||
} catch (error: any) {
|
||||
setError(error.message || t("error_generic"));
|
||||
toast(t("error_toast"), {
|
||||
description: `${t("error_toast_desc")}: ${error.message}`,
|
||||
});
|
||||
} finally {
|
||||
setIsSubmitting(false);
|
||||
}
|
||||
};
|
||||
|
||||
// Function to add a new URL tag
|
||||
const handleAddTag = (text: string) => {
|
||||
// Basic URL validation
|
||||
if (!isValidUrl(text)) {
|
||||
toast(t("invalid_url_toast"), {
|
||||
description: t("invalid_url_toast_desc"),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Check for duplicates
|
||||
if (urlTags.some((tag) => tag.text === text)) {
|
||||
toast(t("duplicate_url_toast"), {
|
||||
description: t("duplicate_url_toast_desc"),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Add the new tag
|
||||
const newTag: Tag = {
|
||||
id: Date.now().toString(),
|
||||
text: text,
|
||||
};
|
||||
|
||||
setUrlTags([...urlTags, newTag]);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="container mx-auto py-8">
|
||||
<Card className="max-w-2xl mx-auto">
|
||||
<CardHeader>
|
||||
<CardTitle className="flex items-center gap-2">
|
||||
<Globe className="h-5 w-5" />
|
||||
{t("title")}
|
||||
</CardTitle>
|
||||
<CardDescription>{t("subtitle")}</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="space-y-4">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="url-input">{t("label")}</Label>
|
||||
<TagInput
|
||||
id="url-input"
|
||||
tags={urlTags}
|
||||
setTags={setUrlTags}
|
||||
placeholder={t("placeholder")}
|
||||
onAddTag={handleAddTag}
|
||||
styleClasses={{
|
||||
inlineTagsContainer:
|
||||
"border-input rounded-lg bg-background shadow-sm shadow-black/5 transition-shadow focus-within:border-ring focus-within:outline-none focus-within:ring-[3px] focus-within:ring-ring/20 p-1 gap-1",
|
||||
input: "w-full min-w-[80px] focus-visible:outline-none shadow-none px-2 h-7",
|
||||
tag: {
|
||||
body: "h-7 relative bg-background border border-input hover:bg-background rounded-md font-medium text-xs ps-2 pe-7 flex",
|
||||
closeButton:
|
||||
"absolute -inset-y-px -end-px p-0 rounded-e-lg flex size-7 transition-colors outline-0 focus-visible:outline focus-visible:outline-2 focus-visible:outline-ring/70 text-muted-foreground/80 hover:text-foreground",
|
||||
},
|
||||
}}
|
||||
activeTagIndex={activeTagIndex}
|
||||
setActiveTagIndex={setActiveTagIndex}
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground mt-1">{t("hint")}</p>
|
||||
</div>
|
||||
|
||||
{error && <div className="text-sm text-red-500 mt-2">{error}</div>}
|
||||
|
||||
<div className="bg-muted/50 rounded-lg p-4 text-sm">
|
||||
<h4 className="font-medium mb-2">{t("tips_title")}</h4>
|
||||
<ul className="list-disc pl-5 space-y-1 text-muted-foreground">
|
||||
<li>{t("tip_1")}</li>
|
||||
<li>{t("tip_2")}</li>
|
||||
<li>{t("tip_3")}</li>
|
||||
<li>{t("tip_4")}</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
<CardFooter className="flex justify-between">
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={() => router.push(`/dashboard/${search_space_id}/documents`)}
|
||||
>
|
||||
{t("cancel")}
|
||||
</Button>
|
||||
<Button onClick={handleSubmit} disabled={isSubmitting || urlTags.length === 0}>
|
||||
{isSubmitting ? (
|
||||
<>
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
{t("submitting")}
|
||||
</>
|
||||
) : (
|
||||
t("submit")
|
||||
)}
|
||||
</Button>
|
||||
</CardFooter>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue