add github connector, add alembic for db migrations, fix bug updating connectors

This commit is contained in:
Adamsmith6300 2025-04-13 13:56:22 -07:00
parent fa5dbb786f
commit bb198e38c0
18 changed files with 1232 additions and 184 deletions

View file

@ -44,6 +44,7 @@ const getConnectorTypeDisplay = (type: string): string => {
"TAVILY_API": "Tavily API",
"SLACK_CONNECTOR": "Slack",
"NOTION_CONNECTOR": "Notion",
"GITHUB_CONNECTOR": "GitHub",
// Add other connector types here as needed
};
return typeMap[type] || type;
@ -253,4 +254,4 @@ export default function ConnectorsPage() {
</Card>
</div>
);
}
}

View file

@ -51,6 +51,7 @@ const getConnectorTypeDisplay = (type: string): string => {
"TAVILY_API": "Tavily API",
"SLACK_CONNECTOR": "Slack Connector",
"NOTION_CONNECTOR": "Notion Connector",
"GITHUB_CONNECTOR": "GitHub Connector",
// Add other connector types here as needed
};
return typeMap[type] || type;
@ -85,7 +86,8 @@ export default function EditConnectorPage() {
"SERPER_API": "SERPER_API_KEY",
"TAVILY_API": "TAVILY_API_KEY",
"SLACK_CONNECTOR": "SLACK_BOT_TOKEN",
"NOTION_CONNECTOR": "NOTION_INTEGRATION_TOKEN"
"NOTION_CONNECTOR": "NOTION_INTEGRATION_TOKEN",
"GITHUB_CONNECTOR": "GITHUB_PAT"
};
return fieldMap[connectorType] || "";
};
@ -136,6 +138,8 @@ export default function EditConnectorPage() {
name: values.name,
connector_type: connector.connector_type,
config: updatedConfig,
is_indexable: connector.is_indexable,
last_indexed_at: connector.last_indexed_at,
});
toast.success("Connector updated successfully!");
@ -223,17 +227,21 @@ export default function EditConnectorPage() {
? "Slack Bot Token"
: connector?.connector_type === "NOTION_CONNECTOR"
? "Notion Integration Token"
: "API Key"}
: connector?.connector_type === "GITHUB_CONNECTOR"
? "GitHub Personal Access Token (PAT)"
: "API Key"}
</FormLabel>
<FormControl>
<Input
type="password"
placeholder={
connector?.connector_type === "SLACK_CONNECTOR"
? "Enter your Slack Bot Token"
? "Enter new Slack Bot Token (optional)"
: connector?.connector_type === "NOTION_CONNECTOR"
? "Enter your Notion Integration Token"
: "Enter your API key"
? "Enter new Notion Token (optional)"
: connector?.connector_type === "GITHUB_CONNECTOR"
? "Enter new GitHub PAT (optional)"
: "Enter new API key (optional)"
}
{...field}
/>
@ -243,7 +251,9 @@ export default function EditConnectorPage() {
? "Enter a new Slack Bot Token or leave blank to keep your existing token."
: connector?.connector_type === "NOTION_CONNECTOR"
? "Enter a new Notion Integration Token or leave blank to keep your existing token."
: "Enter a new API key or leave blank to keep your existing key."}
: connector?.connector_type === "GITHUB_CONNECTOR"
? "Enter a new GitHub PAT or leave blank to keep your existing token."
: "Enter a new API key or leave blank to keep your existing key."}
</FormDescription>
<FormMessage />
</FormItem>
@ -276,4 +286,4 @@ export default function EditConnectorPage() {
</motion.div>
</div>
);
}
}

View file

@ -0,0 +1,298 @@
"use client";
import { useState } from "react";
import { useRouter, useParams } from "next/navigation";
import { motion } from "framer-motion";
import { zodResolver } from "@hookform/resolvers/zod";
import { useForm } from "react-hook-form";
import * as z from "zod";
import { toast } from "sonner";
import { ArrowLeft, Check, Info, Loader2, Github } from "lucide-react";
// Assuming useSearchSourceConnectors hook exists and works similarly
import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors";
import {
Form,
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import { Button } from "@/components/ui/button";
import {
Card,
CardContent,
CardDescription,
CardFooter,
CardHeader,
CardTitle,
} from "@/components/ui/card";
import {
Alert,
AlertDescription,
AlertTitle,
} from "@/components/ui/alert";
import {
Accordion,
AccordionContent,
AccordionItem,
AccordionTrigger,
} from "@/components/ui/accordion";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
// Define the form schema with Zod for GitHub
const githubConnectorFormSchema = z.object({
name: z.string().min(3, {
message: "Connector name must be at least 3 characters.",
}),
github_pat: z.string()
.min(20, { // Apply min length first
message: "GitHub Personal Access Token seems too short.",
})
.refine(pat => pat.startsWith('ghp_') || pat.startsWith('github_pat_'), { // Then refine the pattern
message: "GitHub PAT should start with 'ghp_' or 'github_pat_'",
}),
});
// Define the type for the form values
type GithubConnectorFormValues = z.infer<typeof githubConnectorFormSchema>;
export default function GithubConnectorPage() {
const router = useRouter();
const params = useParams();
const searchSpaceId = params.search_space_id as string;
const [isSubmitting, setIsSubmitting] = useState(false);
const { createConnector } = useSearchSourceConnectors(); // Assuming this hook exists
// Initialize the form
const form = useForm<GithubConnectorFormValues>({
resolver: zodResolver(githubConnectorFormSchema),
defaultValues: {
name: "GitHub Connector",
github_pat: "",
},
});
// Handle form submission
const onSubmit = async (values: GithubConnectorFormValues) => {
setIsSubmitting(true);
try {
await createConnector({
name: values.name,
connector_type: "GITHUB_CONNECTOR",
config: {
GITHUB_PAT: values.github_pat,
},
is_indexable: true, // GitHub connector is indexable
last_indexed_at: null, // New connector hasn't been indexed
});
toast.success("GitHub connector created successfully!");
// Navigate back to connectors management page (or the add page)
router.push(`/dashboard/${searchSpaceId}/connectors`);
} catch (error) { // Added type check for error
console.error("Error creating GitHub connector:", error);
// Display specific backend error message if available
const errorMessage = error instanceof Error ? error.message : "Failed to create GitHub connector. Please check the PAT and permissions.";
toast.error(errorMessage);
} finally {
setIsSubmitting(false);
}
};
return (
<div className="container mx-auto py-8 max-w-3xl">
<Button
variant="ghost"
className="mb-6"
onClick={() => router.push(`/dashboard/${searchSpaceId}/connectors/add`)}
>
<ArrowLeft className="mr-2 h-4 w-4" />
Back to Add Connectors
</Button>
<motion.div
initial={{ opacity: 0, y: 20 }}
animate={{ opacity: 1, y: 0 }}
transition={{ duration: 0.5 }}
>
<Tabs defaultValue="connect" className="w-full">
<TabsList className="grid w-full grid-cols-2 mb-6">
<TabsTrigger value="connect">Connect GitHub</TabsTrigger>
<TabsTrigger value="documentation">Setup Guide</TabsTrigger>
</TabsList>
<TabsContent value="connect">
<Card className="border-2 border-border">
<CardHeader>
<CardTitle className="text-2xl font-bold flex items-center gap-2"><Github className="h-6 w-6" /> Connect GitHub Account</CardTitle>
<CardDescription>
Integrate with GitHub using a Personal Access Token (PAT) to search and retrieve information from accessible repositories. This connector can index your code and documentation.
</CardDescription>
</CardHeader>
<CardContent>
<Alert className="mb-6 bg-muted">
<Info className="h-4 w-4" />
<AlertTitle>GitHub Personal Access Token (PAT) Required</AlertTitle>
<AlertDescription>
You'll need a GitHub PAT with the appropriate scopes (e.g., 'repo') to use this connector. You can create one from your
<a
href="https://github.com/settings/personal-access-tokens"
target="_blank"
rel="noopener noreferrer"
className="font-medium underline underline-offset-4 ml-1"
>
GitHub Developer Settings
</a>.
</AlertDescription>
</Alert>
<Form {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-6">
<FormField
control={form.control}
name="name"
render={({ field }) => (
<FormItem>
<FormLabel>Connector Name</FormLabel>
<FormControl>
<Input placeholder="My GitHub Connector" {...field} />
</FormControl>
<FormDescription>
A friendly name to identify this GitHub connection.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="github_pat"
render={({ field }) => (
<FormItem>
<FormLabel>GitHub Personal Access Token (PAT)</FormLabel>
<FormControl>
<Input
type="password"
placeholder="ghp_... or github_pat_..."
{...field}
/>
</FormControl>
<FormDescription>
Your GitHub PAT will be encrypted and stored securely. Ensure it has the necessary 'repo' scopes.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<div className="flex justify-end">
<Button
type="submit"
disabled={isSubmitting}
className="w-full sm:w-auto"
>
{isSubmitting ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
Connecting...
</>
) : (
<>
<Check className="mr-2 h-4 w-4" />
Connect GitHub
</>
)}
</Button>
</div>
</form>
</Form>
</CardContent>
<CardFooter className="flex flex-col items-start border-t bg-muted/50 px-6 py-4">
<h4 className="text-sm font-medium">What you get with GitHub integration:</h4>
<ul className="mt-2 list-disc pl-5 text-sm text-muted-foreground">
<li>Search through code and documentation in your repositories</li>
<li>Access READMEs, Markdown files, and common code files</li>
<li>Connect your project knowledge directly to your search space</li>
<li>Index your repositories for enhanced search capabilities</li>
</ul>
</CardFooter>
</Card>
</TabsContent>
<TabsContent value="documentation">
<Card className="border-2 border-border">
<CardHeader>
<CardTitle className="text-2xl font-bold">GitHub Connector Setup Guide</CardTitle>
<CardDescription>
Learn how to generate a Personal Access Token (PAT) and connect your GitHub account.
</CardDescription>
</CardHeader>
<CardContent className="space-y-6">
<div>
<h3 className="text-xl font-semibold mb-2">How it works</h3>
<p className="text-muted-foreground">
The GitHub connector uses a Personal Access Token (PAT) to authenticate with the GitHub API. It fetches information about repositories accessible to the token and indexes relevant files (code, markdown, text).
</p>
<ul className="mt-2 list-disc pl-5 text-muted-foreground">
<li>The connector indexes files based on common code and documentation extensions.</li>
<li>Large files (over 1MB) are skipped during indexing.</li>
<li>Indexing runs periodically (check connector settings for frequency) to keep content up-to-date.</li>
</ul>
</div>
<Accordion type="single" collapsible className="w-full">
<AccordionItem value="create_pat">
<AccordionTrigger className="text-lg font-medium">Step 1: Create a GitHub PAT</AccordionTrigger>
<AccordionContent className="space-y-4">
<Alert className="bg-muted">
<Info className="h-4 w-4" />
<AlertTitle>Token Security</AlertTitle>
<AlertDescription>
Treat your PAT like a password. Store it securely and consider using fine-grained tokens if possible.
</AlertDescription>
</Alert>
<div className="space-y-6">
<div>
<h4 className="font-medium mb-2">Generating a Token:</h4>
<ol className="list-decimal pl-5 space-y-3">
<li>Go to your GitHub <a href="https://github.com/settings/tokens" target="_blank" rel="noopener noreferrer" className="font-medium underline underline-offset-4">Developer settings</a>.</li>
<li>Click on <strong>Personal access tokens</strong>, then choose <strong>Tokens (classic)</strong> or <strong>Fine-grained tokens</strong> (recommended if available and suitable).</li>
<li>Click <strong>Generate new token</strong> (and choose the appropriate type).</li>
<li>Give your token a descriptive name (e.g., "SurfSense Connector").</li>
<li>Set an expiration date for the token (recommended for security).</li>
<li>Under <strong>Select scopes</strong> (for classic tokens) or <strong>Repository access</strong> (for fine-grained), grant the necessary permissions. At minimum, the <strong>`repo`</strong> scope (or equivalent read access to repositories for fine-grained tokens) is required to read repository content.</li>
<li>Click <strong>Generate token</strong>.</li>
<li><strong>Important:</strong> Copy your new PAT immediately. You won't be able to see it again after leaving the page.</li>
</ol>
</div>
</div>
</AccordionContent>
</AccordionItem>
<AccordionItem value="connect_app">
<AccordionTrigger className="text-lg font-medium">Step 2: Connect in SurfSense</AccordionTrigger>
<AccordionContent className="space-y-4">
<ol className="list-decimal pl-5 space-y-3">
<li>Paste the copied GitHub PAT into the "GitHub Personal Access Token (PAT)" field on the "Connect GitHub" tab.</li>
<li>Optionally, give the connector a custom name.</li>
<li>Click the <strong>Connect GitHub</strong> button.</li>
<li>If the connection is successful, you will be redirected and can start indexing from the Connectors page.</li>
</ol>
</AccordionContent>
</AccordionItem>
</Accordion>
</CardContent>
</Card>
</TabsContent>
</Tabs>
</motion.div>
</div>
);
}

View file

@ -14,6 +14,7 @@ import {
IconMail,
IconBrandZoom,
IconChevronRight,
IconWorldWww,
} from "@tabler/icons-react";
import { motion, AnimatePresence } from "framer-motion";
import { useState } from "react";
@ -22,36 +23,43 @@ import Link from "next/link";
import { Button } from "@/components/ui/button";
import { Separator } from "@/components/ui/separator";
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
import { useForm } from "react-hook-form";
// Define the Connector type
interface Connector {
id: string;
title: string;
description: string;
icon: React.ReactNode;
status: "available" | "coming-soon" | "connected"; // Added connected status example
}
interface ConnectorCategory {
id: string;
title: string;
connectors: Connector[];
}
// Define connector categories and their connectors
const connectorCategories = [
const connectorCategories: ConnectorCategory[] = [
{
id: "search-engines",
title: "Search Engines",
description: "Connect to search engines to enhance your research capabilities.",
icon: <IconSearch className="h-5 w-5" />,
connectors: [
{
id: "tavily-api",
title: "Tavily Search API",
description: "Connect to Tavily Search API to search the web.",
icon: <IconSearch className="h-6 w-6" />,
status: "available",
},
{
id: "serper-api",
title: "Serper API",
description: "Connect to Serper API to search the web.",
icon: <IconBrandGoogle className="h-6 w-6" />,
status: "coming-soon",
id: "web-search",
title: "Web Search",
description: "Enable web search capabilities for broader context.",
icon: <IconWorldWww className="h-6 w-6" />,
status: "available", // Example status
// Potentially add config form here if needed (e.g., choosing provider)
},
// Add other search engine connectors like Tavily, Serper if they have UI config
],
},
{
id: "team-chats",
title: "Team Chats",
description: "Connect to your team communication platforms.",
icon: <IconMessages className="h-5 w-5" />,
connectors: [
{
id: "slack-connector",
@ -79,8 +87,6 @@ const connectorCategories = [
{
id: "knowledge-bases",
title: "Knowledge Bases",
description: "Connect to your knowledge bases and documentation.",
icon: <IconDatabase className="h-5 w-5" />,
connectors: [
{
id: "notion-connector",
@ -88,21 +94,20 @@ const connectorCategories = [
description: "Connect to your Notion workspace to access pages and databases.",
icon: <IconBrandNotion className="h-6 w-6" />,
status: "available",
// No form here, assumes it links to its own page
},
{
id: "github",
id: "github-connector", // Keep the id simple
title: "GitHub",
description: "Connect to GitHub repositories to access code and documentation.",
description: "Connect a GitHub PAT to index code and docs from accessible repositories.",
icon: <IconBrandGithub className="h-6 w-6" />,
status: "coming-soon",
status: "available",
},
],
},
{
id: "communication",
title: "Communication",
description: "Connect to your email and meeting platforms.",
icon: <IconMail className="h-5 w-5" />,
connectors: [
{
id: "gmail",
@ -125,7 +130,7 @@ const connectorCategories = [
export default function ConnectorsPage() {
const params = useParams();
const searchSpaceId = params.search_space_id as string;
const [expandedCategories, setExpandedCategories] = useState<string[]>(["search-engines"]);
const [expandedCategories, setExpandedCategories] = useState<string[]>(["search-engines", "knowledge-bases"]);
const toggleCategory = (categoryId: string) => {
setExpandedCategories(prev =>
@ -150,104 +155,68 @@ export default function ConnectorsPage() {
</motion.div>
<div className="space-y-6">
{connectorCategories.map((category, categoryIndex) => (
{connectorCategories.map((category) => (
<Collapsible
key={category.id}
open={expandedCategories.includes(category.id)}
onOpenChange={() => toggleCategory(category.id)}
className="border rounded-lg overflow-hidden bg-card"
className="space-y-2"
>
<CollapsibleTrigger asChild>
<motion.div
initial={{ opacity: 0, y: 10 }}
animate={{ opacity: 1, y: 0 }}
transition={{ duration: 0.3, delay: categoryIndex * 0.1 }}
className="p-4 flex items-center justify-between cursor-pointer hover:bg-accent/50 transition-colors"
>
<div className="flex items-center gap-3">
<div className="p-2 rounded-md bg-primary/10 text-primary">
{category.icon}
</div>
<div>
<h2 className="text-xl font-semibold">{category.title}</h2>
<p className="text-sm text-muted-foreground">{category.description}</p>
</div>
</div>
<IconChevronRight
className={cn(
"h-5 w-5 text-muted-foreground transition-transform duration-200",
expandedCategories.includes(category.id) && "rotate-90"
)}
/>
</motion.div>
</CollapsibleTrigger>
<div className="flex items-center justify-between space-x-4 px-1">
<h3 className="text-lg font-semibold dark:text-gray-200">{category.title}</h3>
<CollapsibleTrigger asChild>
{/* Replace with your preferred expand/collapse icon/button */}
<button className="text-sm text-indigo-600 hover:underline dark:text-indigo-400">
{expandedCategories.includes(category.id) ? "Collapse" : "Expand"}
</button>
</CollapsibleTrigger>
</div>
<CollapsibleContent>
<Separator />
<div className="p-4 grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
<AnimatePresence>
{category.connectors.map((connector, index) => (
<motion.div
key={connector.id}
initial={{ opacity: 0, scale: 0.95 }}
animate={{ opacity: 1, scale: 1 }}
exit={{ opacity: 0, scale: 0.95 }}
transition={{
duration: 0.2,
delay: index * 0.05,
type: "spring",
stiffness: 300,
damping: 30
}}
className={cn(
"relative group flex flex-col p-4 rounded-lg border",
connector.status === "coming-soon" ? "opacity-70" : ""
)}
>
<div className="absolute inset-0 opacity-0 group-hover:opacity-100 transition duration-200 bg-gradient-to-t from-accent/50 to-transparent rounded-lg pointer-events-none" />
<div className="mb-4 relative z-10 text-primary">
{connector.icon}
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3 p-1">
{category.connectors.map((connector) => (
<div key={connector.id} className="col-span-1 flex flex-col divide-y divide-gray-200 dark:divide-gray-700 rounded-lg bg-white dark:bg-gray-800 shadow">
<div className="flex w-full items-center justify-between space-x-6 p-6 flex-grow">
<div className="flex-1 truncate">
<div className="flex items-center space-x-3">
<span className="text-gray-900 dark:text-gray-100">{connector.icon}</span>
<h3 className="truncate text-sm font-medium text-gray-900 dark:text-gray-100">
{connector.title}
</h3>
{connector.status === "coming-soon" && (
<span className="inline-block flex-shrink-0 rounded-full bg-yellow-100 px-2 py-0.5 text-xs font-medium text-yellow-800 dark:bg-yellow-900 dark:text-yellow-200">
Coming soon
</span>
)}
{/* TODO: Add 'Connected' badge based on actual state */}
</div>
<p className="mt-1 truncate text-sm text-gray-500 dark:text-gray-400">
{connector.description}
</p>
</div>
<div className="flex items-center justify-between mb-2">
<h3 className="text-lg font-semibold group-hover:translate-x-1 transition duration-200">
{connector.title}
</h3>
{connector.status === "coming-soon" && (
<span className="text-xs bg-muted px-2 py-1 rounded-full">Coming soon</span>
)}
</div>
<p className="text-sm text-muted-foreground mb-4 flex-grow">
{connector.description}
</p>
{connector.status === "available" ? (
<Link
href={`/dashboard/${searchSpaceId}/connectors/add/${connector.id}`}
className="w-full mt-auto"
>
<Button
variant="default"
className="w-full"
>
</div>
{/* Always render Link button if available */}
{connector.status === 'available' && (
<div className="px-6 py-4 border-t border-gray-200 dark:border-gray-700">
<Link href={`/dashboard/${searchSpaceId}/connectors/add/${connector.id}`}>
<Button variant="default" className="w-full">
Connect
</Button>
</Link>
) : (
<Button
variant="outline"
className="w-full mt-auto"
disabled
>
Notify Me
</div>
)}
{connector.status === 'coming-soon' && (
<div className="px-6 py-4 border-t border-gray-200 dark:border-gray-700">
<Button variant="outline" disabled className="w-full">
Coming Soon
</Button>
)}
</motion.div>
))}
</AnimatePresence>
</div>
)}
{/* TODO: Add logic for 'connected' status */}
</div>
))}
</div>
</CollapsibleContent>
<Separator className="my-4" />
</Collapsible>
))}
</div>