mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-09 15:52:40 +02:00
Merge remote-tracking branch 'upstream/dev' into feat/inbox
This commit is contained in:
commit
200f8732f6
13 changed files with 657 additions and 796 deletions
|
|
@ -24,11 +24,6 @@
|
|||
"enabled": true,
|
||||
"status": "warning",
|
||||
"statusMessage": "Some requests may be blocked if not using Firecrawl."
|
||||
},
|
||||
"GITHUB_CONNECTOR": {
|
||||
"enabled": false,
|
||||
"status": "maintenance",
|
||||
"statusMessage": "Rework in progress."
|
||||
}
|
||||
},
|
||||
"globalSettings": {
|
||||
|
|
|
|||
|
|
@ -1,17 +1,12 @@
|
|||
"use client";
|
||||
|
||||
import { zodResolver } from "@hookform/resolvers/zod";
|
||||
import { Info } from "lucide-react";
|
||||
import { ExternalLink, Info } from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import type { FC } from "react";
|
||||
import { useRef, useState } from "react";
|
||||
import { useForm } from "react-hook-form";
|
||||
import * as z from "zod";
|
||||
import {
|
||||
Accordion,
|
||||
AccordionContent,
|
||||
AccordionItem,
|
||||
AccordionTrigger,
|
||||
} from "@/components/ui/accordion";
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import {
|
||||
|
|
@ -34,8 +29,6 @@ import {
|
|||
} from "@/components/ui/select";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { EnumConnectorName } from "@/contracts/enums/connector";
|
||||
import { DateRangeSelector } from "../../components/date-range-selector";
|
||||
import { getConnectorBenefits } from "../connector-benefits";
|
||||
import type { ConnectFormProps } from "../index";
|
||||
|
||||
const githubConnectorFormSchema = z.object({
|
||||
|
|
@ -44,10 +37,8 @@ const githubConnectorFormSchema = z.object({
|
|||
}),
|
||||
github_pat: z
|
||||
.string()
|
||||
.min(20, {
|
||||
message: "GitHub Personal Access Token seems too short.",
|
||||
})
|
||||
.refine((pat) => pat.startsWith("ghp_") || pat.startsWith("github_pat_"), {
|
||||
.optional()
|
||||
.refine((pat) => !pat || pat.startsWith("ghp_") || pat.startsWith("github_pat_"), {
|
||||
message: "GitHub PAT should start with 'ghp_' or 'github_pat_'",
|
||||
}),
|
||||
repo_full_names: z.string().min(1, {
|
||||
|
|
@ -59,8 +50,6 @@ type GithubConnectorFormValues = z.infer<typeof githubConnectorFormSchema>;
|
|||
|
||||
export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting }) => {
|
||||
const isSubmittingRef = useRef(false);
|
||||
const [startDate, setStartDate] = useState<Date | undefined>(undefined);
|
||||
const [endDate, setEndDate] = useState<Date | undefined>(undefined);
|
||||
const [periodicEnabled, setPeriodicEnabled] = useState(false);
|
||||
const [frequencyMinutes, setFrequencyMinutes] = useState("1440");
|
||||
const form = useForm<GithubConnectorFormValues>({
|
||||
|
|
@ -94,16 +83,18 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
|
|||
name: values.name,
|
||||
connector_type: EnumConnectorName.GITHUB_CONNECTOR,
|
||||
config: {
|
||||
GITHUB_PAT: values.github_pat,
|
||||
GITHUB_PAT: values.github_pat || null, // Optional - only for private repos
|
||||
repo_full_names: repoList,
|
||||
},
|
||||
is_indexable: true,
|
||||
is_active: true,
|
||||
last_indexed_at: null,
|
||||
periodic_indexing_enabled: periodicEnabled,
|
||||
indexing_frequency_minutes: periodicEnabled ? parseInt(frequencyMinutes, 10) : null,
|
||||
next_scheduled_at: null,
|
||||
startDate,
|
||||
endDate,
|
||||
// GitHub indexes full repo snapshots - no date range needed
|
||||
startDate: undefined,
|
||||
endDate: undefined,
|
||||
periodicEnabled,
|
||||
frequencyMinutes,
|
||||
});
|
||||
|
|
@ -117,18 +108,19 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
|
|||
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 p-2 sm:p-3 flex items-center [&>svg]:relative [&>svg]:left-0 [&>svg]:top-0 [&>svg+div]:translate-y-0">
|
||||
<Info className="h-3 w-3 sm:h-4 sm:w-4 shrink-0 ml-1" />
|
||||
<div className="-ml-1">
|
||||
<AlertTitle className="text-xs sm:text-sm">Personal Access Token Required</AlertTitle>
|
||||
<AlertTitle className="text-xs sm:text-sm">Personal Access Token (Optional)</AlertTitle>
|
||||
<AlertDescription className="text-[10px] sm:text-xs !pl-0">
|
||||
You'll need a GitHub Personal Access Token to use this connector. You can create one
|
||||
from{" "}
|
||||
A GitHub PAT is only required for private repositories. Public repos work without a
|
||||
token.{" "}
|
||||
<a
|
||||
href="https://github.com/settings/tokens"
|
||||
href="https://github.com/settings/tokens/new?description=surfsense&scopes=repo"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="font-medium underline underline-offset-4"
|
||||
className="font-medium underline underline-offset-4 inline-flex items-center gap-1.5"
|
||||
>
|
||||
GitHub Settings
|
||||
</a>
|
||||
Get your token
|
||||
<ExternalLink className="h-3 w-3 sm:h-4 sm:w-4" />
|
||||
</a>{" "}
|
||||
</AlertDescription>
|
||||
</div>
|
||||
</Alert>
|
||||
|
|
@ -167,7 +159,10 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
|
|||
name="github_pat"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel className="text-xs sm:text-sm">GitHub Personal Access Token</FormLabel>
|
||||
<FormLabel className="text-xs sm:text-sm">
|
||||
GitHub Personal Access Token{" "}
|
||||
<span className="text-muted-foreground font-normal">(optional)</span>
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="password"
|
||||
|
|
@ -178,8 +173,8 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
|
|||
/>
|
||||
</FormControl>
|
||||
<FormDescription className="text-[10px] sm:text-xs">
|
||||
Your GitHub PAT will be encrypted and stored securely. It typically starts with
|
||||
"ghp_" or "github_pat_".
|
||||
Only required for private repositories. Leave empty if indexing public repos
|
||||
only.
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
|
|
@ -225,15 +220,9 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
|
|||
|
||||
{/* Indexing Configuration */}
|
||||
<div className="space-y-4 pt-4 border-t border-slate-400/20">
|
||||
<h3 className="text-sm sm:text-base font-medium">Indexing Configuration</h3>
|
||||
<h3 className="text-sm sm:text-base font-medium">Sync Configuration</h3>
|
||||
|
||||
{/* Date Range Selector */}
|
||||
<DateRangeSelector
|
||||
startDate={startDate}
|
||||
endDate={endDate}
|
||||
onStartDateChange={setStartDate}
|
||||
onEndDateChange={setEndDate}
|
||||
/>
|
||||
{/* Note: No date range for GitHub - it indexes full repo snapshots */}
|
||||
|
||||
{/* Periodic Sync Config */}
|
||||
<div className="rounded-xl bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6">
|
||||
|
|
@ -301,169 +290,18 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
|
|||
</Form>
|
||||
</div>
|
||||
|
||||
{/* What you get section */}
|
||||
{getConnectorBenefits(EnumConnectorName.GITHUB_CONNECTOR) && (
|
||||
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 px-3 sm:px-6 py-4 space-y-2">
|
||||
<h4 className="text-xs sm:text-sm font-medium">What you get with GitHub integration:</h4>
|
||||
<ul className="list-disc pl-5 text-[10px] sm:text-xs text-muted-foreground space-y-1">
|
||||
{getConnectorBenefits(EnumConnectorName.GITHUB_CONNECTOR)?.map((benefit) => (
|
||||
<li key={benefit}>{benefit}</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Documentation Section */}
|
||||
<Accordion
|
||||
type="single"
|
||||
collapsible
|
||||
className="w-full border border-border rounded-xl bg-slate-400/5 dark:bg-white/5"
|
||||
>
|
||||
<AccordionItem value="documentation" className="border-0">
|
||||
<AccordionTrigger className="text-sm sm:text-base font-medium px-3 sm:px-6 no-underline hover:no-underline">
|
||||
Documentation
|
||||
</AccordionTrigger>
|
||||
<AccordionContent className="px-3 sm:px-6 pb-3 sm:pb-6 space-y-6">
|
||||
<div>
|
||||
<h3 className="text-sm sm:text-base font-semibold mb-2">How it works</h3>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground">
|
||||
The GitHub connector uses a Personal Access Token (PAT) to authenticate with the
|
||||
GitHub API. You provide a comma-separated list of repository full names (e.g.,
|
||||
"owner/repo1, owner/repo2") that you want to index. The connector indexes relevant
|
||||
files (code, markdown, text) from the selected repositories.
|
||||
</p>
|
||||
<ul className="mt-2 list-disc pl-5 text-[10px] sm:text-xs text-muted-foreground space-y-1">
|
||||
<li>
|
||||
The connector indexes files based on common code and documentation extensions.
|
||||
</li>
|
||||
<li>Large files (over 1MB) are skipped during indexing.</li>
|
||||
<li>Only specified repositories are indexed.</li>
|
||||
<li>
|
||||
Indexing runs periodically (check connector settings for frequency) to keep
|
||||
content up-to-date.
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h3 className="text-sm sm:text-base font-semibold mb-2">Authorization</h3>
|
||||
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 mb-4">
|
||||
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
|
||||
<AlertTitle className="text-[10px] sm:text-xs">
|
||||
Personal Access Token Required
|
||||
</AlertTitle>
|
||||
<AlertDescription className="text-[9px] sm:text-[10px]">
|
||||
You'll need a GitHub PAT with the appropriate scopes (e.g., 'repo') to fetch
|
||||
repositories. The PAT will be stored securely to enable indexing.
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
|
||||
<div className="space-y-4 sm:space-y-6">
|
||||
<div>
|
||||
<h4 className="text-[10px] sm:text-xs font-medium mb-2">
|
||||
Step 1: Generate GitHub PAT
|
||||
</h4>
|
||||
<ol className="list-decimal pl-5 space-y-2 text-[10px] sm:text-xs text-muted-foreground">
|
||||
<li>
|
||||
Go to your GitHub{" "}
|
||||
<a
|
||||
href="https://github.com/settings/tokens"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="font-medium underline underline-offset-4"
|
||||
>
|
||||
Developer settings
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
Click on <strong>Personal access tokens</strong>, then choose{" "}
|
||||
<strong>Tokens (classic)</strong> or <strong>Fine-grained tokens</strong>{" "}
|
||||
(recommended if available).
|
||||
</li>
|
||||
<li>
|
||||
Click <strong>Generate new token</strong> (and choose the appropriate type).
|
||||
</li>
|
||||
<li>Give your token a descriptive name (e.g., "SurfSense Connector").</li>
|
||||
<li>Set an expiration date for the token (recommended for security).</li>
|
||||
<li>
|
||||
Under <strong>Select scopes</strong> (for classic tokens) or{" "}
|
||||
<strong>Repository access</strong> (for fine-grained), grant the necessary
|
||||
permissions. At minimum, the <strong>`repo`</strong> scope (or equivalent
|
||||
read access to repositories for fine-grained tokens) is required to read
|
||||
repository content.
|
||||
</li>
|
||||
<li>
|
||||
Click <strong>Generate token</strong>.
|
||||
</li>
|
||||
<li>
|
||||
<strong>Important:</strong> Copy your new PAT immediately. You won't be able
|
||||
to see it again after leaving the page.
|
||||
</li>
|
||||
</ol>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h4 className="text-[10px] sm:text-xs font-medium mb-2">
|
||||
Step 2: Specify repositories
|
||||
</h4>
|
||||
<p className="text-[10px] sm:text-xs text-muted-foreground mb-3">
|
||||
Enter a comma-separated list of repository full names in the format
|
||||
"owner/repo1, owner/repo2". The connector will index files from only the
|
||||
specified repositories.
|
||||
</p>
|
||||
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20">
|
||||
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
|
||||
<AlertTitle className="text-[10px] sm:text-xs">Repository Access</AlertTitle>
|
||||
<AlertDescription className="text-[9px] sm:text-[10px]">
|
||||
Make sure your PAT has access to all repositories you want to index. Private
|
||||
repositories require appropriate permissions.
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h3 className="text-sm sm:text-base font-semibold mb-2">Indexing</h3>
|
||||
<ol className="list-decimal pl-5 space-y-2 text-[10px] sm:text-xs text-muted-foreground mb-4">
|
||||
<li>
|
||||
Navigate to the Connector Dashboard and select the <strong>GitHub</strong>{" "}
|
||||
Connector.
|
||||
</li>
|
||||
<li>
|
||||
Enter your <strong>GitHub Personal Access Token</strong> in the form field.
|
||||
</li>
|
||||
<li>
|
||||
Enter a comma-separated list of <strong>Repository Names</strong> (e.g.,
|
||||
"owner/repo1, owner/repo2").
|
||||
</li>
|
||||
<li>
|
||||
Click <strong>Connect</strong> to establish the connection.
|
||||
</li>
|
||||
<li>Once connected, your GitHub repositories will be indexed automatically.</li>
|
||||
</ol>
|
||||
|
||||
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20">
|
||||
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
|
||||
<AlertTitle className="text-[10px] sm:text-xs">What Gets Indexed</AlertTitle>
|
||||
<AlertDescription className="text-[9px] sm:text-[10px]">
|
||||
<p className="mb-2">The GitHub connector indexes the following data:</p>
|
||||
<ul className="list-disc pl-5 space-y-1">
|
||||
<li>Code files from selected repositories</li>
|
||||
<li>README files and Markdown documentation</li>
|
||||
<li>Common text-based file formats</li>
|
||||
<li>Repository metadata and structure</li>
|
||||
</ul>
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
</div>
|
||||
</div>
|
||||
</AccordionContent>
|
||||
</AccordionItem>
|
||||
</Accordion>
|
||||
{/* Documentation Link */}
|
||||
<div>
|
||||
<Link
|
||||
href="/docs/connectors/github"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-xs sm:text-sm font-medium underline underline-offset-4 hover:text-primary transition-colors inline-flex items-center gap-1.5"
|
||||
>
|
||||
View GitHub Connector Documentation
|
||||
<ExternalLink className="h-3 w-3 sm:h-4 sm:w-4" />
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import { KeyRound } from "lucide-react";
|
||||
import type { FC } from "react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
|
|
@ -12,25 +12,29 @@ export interface GithubConfigProps extends ConnectorConfigProps {
|
|||
onNameChange?: (name: string) => void;
|
||||
}
|
||||
|
||||
// Helper functions moved outside component to avoid useEffect dependency issues
|
||||
const stringToArray = (arr: string[] | string | undefined): string[] => {
|
||||
if (Array.isArray(arr)) return arr;
|
||||
if (typeof arr === "string") {
|
||||
return arr
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0);
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const arrayToString = (arr: string[]): string => {
|
||||
return arr.join(", ");
|
||||
};
|
||||
|
||||
export const GithubConfig: FC<GithubConfigProps> = ({
|
||||
connector,
|
||||
onConfigChange,
|
||||
onNameChange,
|
||||
}) => {
|
||||
const stringToArray = (arr: string[] | string | undefined): string[] => {
|
||||
if (Array.isArray(arr)) return arr;
|
||||
if (typeof arr === "string") {
|
||||
return arr
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0);
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const arrayToString = (arr: string[]): string => {
|
||||
return arr.join(", ");
|
||||
};
|
||||
// Track internal changes to prevent useEffect from overwriting user input
|
||||
const isInternalChange = useRef(false);
|
||||
|
||||
const [githubPat, setGithubPat] = useState<string>(
|
||||
(connector.config?.GITHUB_PAT as string) || ""
|
||||
|
|
@ -40,8 +44,13 @@ export const GithubConfig: FC<GithubConfigProps> = ({
|
|||
);
|
||||
const [name, setName] = useState<string>(connector.name || "");
|
||||
|
||||
// Update values when connector changes
|
||||
// Update values when connector changes externally (not from our own input)
|
||||
useEffect(() => {
|
||||
// Skip if this is our own internal change
|
||||
if (isInternalChange.current) {
|
||||
isInternalChange.current = false;
|
||||
return;
|
||||
}
|
||||
const pat = (connector.config?.GITHUB_PAT as string) || "";
|
||||
const repos = arrayToString(stringToArray(connector.config?.repo_full_names));
|
||||
setGithubPat(pat);
|
||||
|
|
@ -50,6 +59,7 @@ export const GithubConfig: FC<GithubConfigProps> = ({
|
|||
}, [connector.config, connector.name]);
|
||||
|
||||
const handleGithubPatChange = (value: string) => {
|
||||
isInternalChange.current = true;
|
||||
setGithubPat(value);
|
||||
if (onConfigChange) {
|
||||
onConfigChange({
|
||||
|
|
@ -60,6 +70,7 @@ export const GithubConfig: FC<GithubConfigProps> = ({
|
|||
};
|
||||
|
||||
const handleRepoFullNamesChange = (value: string) => {
|
||||
isInternalChange.current = true;
|
||||
setRepoFullNames(value);
|
||||
const repoList = stringToArray(value);
|
||||
if (onConfigChange) {
|
||||
|
|
@ -71,6 +82,7 @@ export const GithubConfig: FC<GithubConfigProps> = ({
|
|||
};
|
||||
|
||||
const handleNameChange = (value: string) => {
|
||||
isInternalChange.current = true;
|
||||
setName(value);
|
||||
if (onNameChange) {
|
||||
onNameChange(value);
|
||||
|
|
@ -105,7 +117,7 @@ export const GithubConfig: FC<GithubConfigProps> = ({
|
|||
<div className="space-y-2">
|
||||
<Label className="flex items-center gap-2 text-xs sm:text-sm">
|
||||
<KeyRound className="h-4 w-4" />
|
||||
GitHub Personal Access Token
|
||||
GitHub Personal Access Token (optional)
|
||||
</Label>
|
||||
<Input
|
||||
type="password"
|
||||
|
|
|
|||
|
|
@ -206,9 +206,10 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
|
|||
{/* Date range selector and periodic sync - only shown for indexable connectors */}
|
||||
{connector.is_indexable && (
|
||||
<>
|
||||
{/* Date range selector - not shown for Google Drive (uses folder selection) or Webcrawler (uses config) */}
|
||||
{/* Date range selector - not shown for Google Drive, Webcrawler, or GitHub (indexes full repo snapshots) */}
|
||||
{connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" &&
|
||||
connector.connector_type !== "WEBCRAWLER_CONNECTOR" && (
|
||||
connector.connector_type !== "WEBCRAWLER_CONNECTOR" &&
|
||||
connector.connector_type !== "GITHUB_CONNECTOR" && (
|
||||
<DateRangeSelector
|
||||
startDate={startDate}
|
||||
endDate={endDate}
|
||||
|
|
|
|||
|
|
@ -151,9 +151,10 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
|
|||
{/* Date range selector and periodic sync - only shown for indexable connectors */}
|
||||
{connector?.is_indexable && (
|
||||
<>
|
||||
{/* Date range selector - not shown for Google Drive (uses folder selection) or Webcrawler (uses config) */}
|
||||
{/* Date range selector - not shown for Google Drive, Webcrawler, or GitHub (indexes full repo snapshots) */}
|
||||
{config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" &&
|
||||
config.connectorType !== "WEBCRAWLER_CONNECTOR" && (
|
||||
config.connectorType !== "WEBCRAWLER_CONNECTOR" &&
|
||||
config.connectorType !== "GITHUB_CONNECTOR" && (
|
||||
<DateRangeSelector
|
||||
startDate={startDate}
|
||||
endDate={endDate}
|
||||
|
|
|
|||
|
|
@ -489,8 +489,8 @@ export function SourceDetailPanel({
|
|||
>
|
||||
{idx + 1}
|
||||
{isCited && (
|
||||
<span className="absolute -top-1 -right-1 w-3 h-3 bg-primary rounded-full border-2 border-background">
|
||||
<Sparkles className="h-2 w-2 text-primary-foreground absolute top-0.5 left-0.5" />
|
||||
<span className="absolute -top-1.5 -right-1.5 flex items-center justify-center w-4 h-4 bg-primary rounded-full border-2 border-background shadow-sm">
|
||||
<Sparkles className="h-2.5 w-2.5 text-primary-foreground" />
|
||||
</span>
|
||||
)}
|
||||
</motion.button>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue