Merge remote-tracking branch 'upstream/dev' into feat/inbox

This commit is contained in:
Anish Sarkar 2026-01-22 02:13:42 +05:30
commit 200f8732f6
13 changed files with 657 additions and 796 deletions

View file

@ -24,11 +24,6 @@
"enabled": true,
"status": "warning",
"statusMessage": "Some requests may be blocked if not using Firecrawl."
},
"GITHUB_CONNECTOR": {
"enabled": false,
"status": "maintenance",
"statusMessage": "Rework in progress."
}
},
"globalSettings": {

View file

@ -1,17 +1,12 @@
"use client";
import { zodResolver } from "@hookform/resolvers/zod";
import { Info } from "lucide-react";
import { ExternalLink, Info } from "lucide-react";
import Link from "next/link";
import type { FC } from "react";
import { useRef, useState } from "react";
import { useForm } from "react-hook-form";
import * as z from "zod";
import {
Accordion,
AccordionContent,
AccordionItem,
AccordionTrigger,
} from "@/components/ui/accordion";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import { Badge } from "@/components/ui/badge";
import {
@ -34,8 +29,6 @@ import {
} from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
import { EnumConnectorName } from "@/contracts/enums/connector";
import { DateRangeSelector } from "../../components/date-range-selector";
import { getConnectorBenefits } from "../connector-benefits";
import type { ConnectFormProps } from "../index";
const githubConnectorFormSchema = z.object({
@ -44,10 +37,8 @@ const githubConnectorFormSchema = z.object({
}),
github_pat: z
.string()
.min(20, {
message: "GitHub Personal Access Token seems too short.",
})
.refine((pat) => pat.startsWith("ghp_") || pat.startsWith("github_pat_"), {
.optional()
.refine((pat) => !pat || pat.startsWith("ghp_") || pat.startsWith("github_pat_"), {
message: "GitHub PAT should start with 'ghp_' or 'github_pat_'",
}),
repo_full_names: z.string().min(1, {
@ -59,8 +50,6 @@ type GithubConnectorFormValues = z.infer<typeof githubConnectorFormSchema>;
export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting }) => {
const isSubmittingRef = useRef(false);
const [startDate, setStartDate] = useState<Date | undefined>(undefined);
const [endDate, setEndDate] = useState<Date | undefined>(undefined);
const [periodicEnabled, setPeriodicEnabled] = useState(false);
const [frequencyMinutes, setFrequencyMinutes] = useState("1440");
const form = useForm<GithubConnectorFormValues>({
@ -94,16 +83,18 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
name: values.name,
connector_type: EnumConnectorName.GITHUB_CONNECTOR,
config: {
GITHUB_PAT: values.github_pat,
GITHUB_PAT: values.github_pat || null, // Optional - only for private repos
repo_full_names: repoList,
},
is_indexable: true,
is_active: true,
last_indexed_at: null,
periodic_indexing_enabled: periodicEnabled,
indexing_frequency_minutes: periodicEnabled ? parseInt(frequencyMinutes, 10) : null,
next_scheduled_at: null,
startDate,
endDate,
// GitHub indexes full repo snapshots - no date range needed
startDate: undefined,
endDate: undefined,
periodicEnabled,
frequencyMinutes,
});
@ -117,18 +108,19 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 p-2 sm:p-3 flex items-center [&>svg]:relative [&>svg]:left-0 [&>svg]:top-0 [&>svg+div]:translate-y-0">
<Info className="h-3 w-3 sm:h-4 sm:w-4 shrink-0 ml-1" />
<div className="-ml-1">
<AlertTitle className="text-xs sm:text-sm">Personal Access Token Required</AlertTitle>
<AlertTitle className="text-xs sm:text-sm">Personal Access Token (Optional)</AlertTitle>
<AlertDescription className="text-[10px] sm:text-xs !pl-0">
You'll need a GitHub Personal Access Token to use this connector. You can create one
from{" "}
A GitHub PAT is only required for private repositories. Public repos work without a
token.{" "}
<a
href="https://github.com/settings/tokens"
href="https://github.com/settings/tokens/new?description=surfsense&scopes=repo"
target="_blank"
rel="noopener noreferrer"
className="font-medium underline underline-offset-4"
className="font-medium underline underline-offset-4 inline-flex items-center gap-1.5"
>
GitHub Settings
</a>
Get your token
<ExternalLink className="h-3 w-3 sm:h-4 sm:w-4" />
</a>{" "}
</AlertDescription>
</div>
</Alert>
@ -167,7 +159,10 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
name="github_pat"
render={({ field }) => (
<FormItem>
<FormLabel className="text-xs sm:text-sm">GitHub Personal Access Token</FormLabel>
<FormLabel className="text-xs sm:text-sm">
GitHub Personal Access Token{" "}
<span className="text-muted-foreground font-normal">(optional)</span>
</FormLabel>
<FormControl>
<Input
type="password"
@ -178,8 +173,8 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
/>
</FormControl>
<FormDescription className="text-[10px] sm:text-xs">
Your GitHub PAT will be encrypted and stored securely. It typically starts with
"ghp_" or "github_pat_".
Only required for private repositories. Leave empty if indexing public repos
only.
</FormDescription>
<FormMessage />
</FormItem>
@ -225,15 +220,9 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
{/* Indexing Configuration */}
<div className="space-y-4 pt-4 border-t border-slate-400/20">
<h3 className="text-sm sm:text-base font-medium">Indexing Configuration</h3>
<h3 className="text-sm sm:text-base font-medium">Sync Configuration</h3>
{/* Date Range Selector */}
<DateRangeSelector
startDate={startDate}
endDate={endDate}
onStartDateChange={setStartDate}
onEndDateChange={setEndDate}
/>
{/* Note: No date range for GitHub - it indexes full repo snapshots */}
{/* Periodic Sync Config */}
<div className="rounded-xl bg-slate-400/5 dark:bg-white/5 p-3 sm:p-6">
@ -301,169 +290,18 @@ export const GithubConnectForm: FC<ConnectFormProps> = ({ onSubmit, isSubmitting
</Form>
</div>
{/* What you get section */}
{getConnectorBenefits(EnumConnectorName.GITHUB_CONNECTOR) && (
<div className="rounded-xl border border-border bg-slate-400/5 dark:bg-white/5 px-3 sm:px-6 py-4 space-y-2">
<h4 className="text-xs sm:text-sm font-medium">What you get with GitHub integration:</h4>
<ul className="list-disc pl-5 text-[10px] sm:text-xs text-muted-foreground space-y-1">
{getConnectorBenefits(EnumConnectorName.GITHUB_CONNECTOR)?.map((benefit) => (
<li key={benefit}>{benefit}</li>
))}
</ul>
</div>
)}
{/* Documentation Section */}
<Accordion
type="single"
collapsible
className="w-full border border-border rounded-xl bg-slate-400/5 dark:bg-white/5"
>
<AccordionItem value="documentation" className="border-0">
<AccordionTrigger className="text-sm sm:text-base font-medium px-3 sm:px-6 no-underline hover:no-underline">
Documentation
</AccordionTrigger>
<AccordionContent className="px-3 sm:px-6 pb-3 sm:pb-6 space-y-6">
<div>
<h3 className="text-sm sm:text-base font-semibold mb-2">How it works</h3>
<p className="text-[10px] sm:text-xs text-muted-foreground">
The GitHub connector uses a Personal Access Token (PAT) to authenticate with the
GitHub API. You provide a comma-separated list of repository full names (e.g.,
"owner/repo1, owner/repo2") that you want to index. The connector indexes relevant
files (code, markdown, text) from the selected repositories.
</p>
<ul className="mt-2 list-disc pl-5 text-[10px] sm:text-xs text-muted-foreground space-y-1">
<li>
The connector indexes files based on common code and documentation extensions.
</li>
<li>Large files (over 1MB) are skipped during indexing.</li>
<li>Only specified repositories are indexed.</li>
<li>
Indexing runs periodically (check connector settings for frequency) to keep
content up-to-date.
</li>
</ul>
</div>
<div className="space-y-4">
<div>
<h3 className="text-sm sm:text-base font-semibold mb-2">Authorization</h3>
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20 mb-4">
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
<AlertTitle className="text-[10px] sm:text-xs">
Personal Access Token Required
</AlertTitle>
<AlertDescription className="text-[9px] sm:text-[10px]">
You'll need a GitHub PAT with the appropriate scopes (e.g., 'repo') to fetch
repositories. The PAT will be stored securely to enable indexing.
</AlertDescription>
</Alert>
<div className="space-y-4 sm:space-y-6">
<div>
<h4 className="text-[10px] sm:text-xs font-medium mb-2">
Step 1: Generate GitHub PAT
</h4>
<ol className="list-decimal pl-5 space-y-2 text-[10px] sm:text-xs text-muted-foreground">
<li>
Go to your GitHub{" "}
<a
href="https://github.com/settings/tokens"
target="_blank"
rel="noopener noreferrer"
className="font-medium underline underline-offset-4"
>
Developer settings
</a>
</li>
<li>
Click on <strong>Personal access tokens</strong>, then choose{" "}
<strong>Tokens (classic)</strong> or <strong>Fine-grained tokens</strong>{" "}
(recommended if available).
</li>
<li>
Click <strong>Generate new token</strong> (and choose the appropriate type).
</li>
<li>Give your token a descriptive name (e.g., "SurfSense Connector").</li>
<li>Set an expiration date for the token (recommended for security).</li>
<li>
Under <strong>Select scopes</strong> (for classic tokens) or{" "}
<strong>Repository access</strong> (for fine-grained), grant the necessary
permissions. At minimum, the <strong>`repo`</strong> scope (or equivalent
read access to repositories for fine-grained tokens) is required to read
repository content.
</li>
<li>
Click <strong>Generate token</strong>.
</li>
<li>
<strong>Important:</strong> Copy your new PAT immediately. You won't be able
to see it again after leaving the page.
</li>
</ol>
</div>
<div>
<h4 className="text-[10px] sm:text-xs font-medium mb-2">
Step 2: Specify repositories
</h4>
<p className="text-[10px] sm:text-xs text-muted-foreground mb-3">
Enter a comma-separated list of repository full names in the format
"owner/repo1, owner/repo2". The connector will index files from only the
specified repositories.
</p>
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20">
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
<AlertTitle className="text-[10px] sm:text-xs">Repository Access</AlertTitle>
<AlertDescription className="text-[9px] sm:text-[10px]">
Make sure your PAT has access to all repositories you want to index. Private
repositories require appropriate permissions.
</AlertDescription>
</Alert>
</div>
</div>
</div>
</div>
<div className="space-y-4">
<div>
<h3 className="text-sm sm:text-base font-semibold mb-2">Indexing</h3>
<ol className="list-decimal pl-5 space-y-2 text-[10px] sm:text-xs text-muted-foreground mb-4">
<li>
Navigate to the Connector Dashboard and select the <strong>GitHub</strong>{" "}
Connector.
</li>
<li>
Enter your <strong>GitHub Personal Access Token</strong> in the form field.
</li>
<li>
Enter a comma-separated list of <strong>Repository Names</strong> (e.g.,
"owner/repo1, owner/repo2").
</li>
<li>
Click <strong>Connect</strong> to establish the connection.
</li>
<li>Once connected, your GitHub repositories will be indexed automatically.</li>
</ol>
<Alert className="bg-slate-400/5 dark:bg-white/5 border-slate-400/20">
<Info className="h-3 w-3 sm:h-4 sm:w-4" />
<AlertTitle className="text-[10px] sm:text-xs">What Gets Indexed</AlertTitle>
<AlertDescription className="text-[9px] sm:text-[10px]">
<p className="mb-2">The GitHub connector indexes the following data:</p>
<ul className="list-disc pl-5 space-y-1">
<li>Code files from selected repositories</li>
<li>README files and Markdown documentation</li>
<li>Common text-based file formats</li>
<li>Repository metadata and structure</li>
</ul>
</AlertDescription>
</Alert>
</div>
</div>
</AccordionContent>
</AccordionItem>
</Accordion>
{/* Documentation Link */}
<div>
<Link
href="/docs/connectors/github"
target="_blank"
rel="noopener noreferrer"
className="text-xs sm:text-sm font-medium underline underline-offset-4 hover:text-primary transition-colors inline-flex items-center gap-1.5"
>
View GitHub Connector Documentation
<ExternalLink className="h-3 w-3 sm:h-4 sm:w-4" />
</Link>
</div>
</div>
);
};

View file

@ -2,7 +2,7 @@
import { KeyRound } from "lucide-react";
import type { FC } from "react";
import { useEffect, useState } from "react";
import { useEffect, useRef, useState } from "react";
import { Badge } from "@/components/ui/badge";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
@ -12,25 +12,29 @@ export interface GithubConfigProps extends ConnectorConfigProps {
onNameChange?: (name: string) => void;
}
// Helper functions moved outside component to avoid useEffect dependency issues
const stringToArray = (arr: string[] | string | undefined): string[] => {
if (Array.isArray(arr)) return arr;
if (typeof arr === "string") {
return arr
.split(",")
.map((item) => item.trim())
.filter((item) => item.length > 0);
}
return [];
};
const arrayToString = (arr: string[]): string => {
return arr.join(", ");
};
export const GithubConfig: FC<GithubConfigProps> = ({
connector,
onConfigChange,
onNameChange,
}) => {
const stringToArray = (arr: string[] | string | undefined): string[] => {
if (Array.isArray(arr)) return arr;
if (typeof arr === "string") {
return arr
.split(",")
.map((item) => item.trim())
.filter((item) => item.length > 0);
}
return [];
};
const arrayToString = (arr: string[]): string => {
return arr.join(", ");
};
// Track internal changes to prevent useEffect from overwriting user input
const isInternalChange = useRef(false);
const [githubPat, setGithubPat] = useState<string>(
(connector.config?.GITHUB_PAT as string) || ""
@ -40,8 +44,13 @@ export const GithubConfig: FC<GithubConfigProps> = ({
);
const [name, setName] = useState<string>(connector.name || "");
// Update values when connector changes
// Update values when connector changes externally (not from our own input)
useEffect(() => {
// Skip if this is our own internal change
if (isInternalChange.current) {
isInternalChange.current = false;
return;
}
const pat = (connector.config?.GITHUB_PAT as string) || "";
const repos = arrayToString(stringToArray(connector.config?.repo_full_names));
setGithubPat(pat);
@ -50,6 +59,7 @@ export const GithubConfig: FC<GithubConfigProps> = ({
}, [connector.config, connector.name]);
const handleGithubPatChange = (value: string) => {
isInternalChange.current = true;
setGithubPat(value);
if (onConfigChange) {
onConfigChange({
@ -60,6 +70,7 @@ export const GithubConfig: FC<GithubConfigProps> = ({
};
const handleRepoFullNamesChange = (value: string) => {
isInternalChange.current = true;
setRepoFullNames(value);
const repoList = stringToArray(value);
if (onConfigChange) {
@ -71,6 +82,7 @@ export const GithubConfig: FC<GithubConfigProps> = ({
};
const handleNameChange = (value: string) => {
isInternalChange.current = true;
setName(value);
if (onNameChange) {
onNameChange(value);
@ -105,7 +117,7 @@ export const GithubConfig: FC<GithubConfigProps> = ({
<div className="space-y-2">
<Label className="flex items-center gap-2 text-xs sm:text-sm">
<KeyRound className="h-4 w-4" />
GitHub Personal Access Token
GitHub Personal Access Token (optional)
</Label>
<Input
type="password"

View file

@ -206,9 +206,10 @@ export const ConnectorEditView: FC<ConnectorEditViewProps> = ({
{/* Date range selector and periodic sync - only shown for indexable connectors */}
{connector.is_indexable && (
<>
{/* Date range selector - not shown for Google Drive (uses folder selection) or Webcrawler (uses config) */}
{/* Date range selector - not shown for Google Drive, Webcrawler, or GitHub (indexes full repo snapshots) */}
{connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" &&
connector.connector_type !== "WEBCRAWLER_CONNECTOR" && (
connector.connector_type !== "WEBCRAWLER_CONNECTOR" &&
connector.connector_type !== "GITHUB_CONNECTOR" && (
<DateRangeSelector
startDate={startDate}
endDate={endDate}

View file

@ -151,9 +151,10 @@ export const IndexingConfigurationView: FC<IndexingConfigurationViewProps> = ({
{/* Date range selector and periodic sync - only shown for indexable connectors */}
{connector?.is_indexable && (
<>
{/* Date range selector - not shown for Google Drive (uses folder selection) or Webcrawler (uses config) */}
{/* Date range selector - not shown for Google Drive, Webcrawler, or GitHub (indexes full repo snapshots) */}
{config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" &&
config.connectorType !== "WEBCRAWLER_CONNECTOR" && (
config.connectorType !== "WEBCRAWLER_CONNECTOR" &&
config.connectorType !== "GITHUB_CONNECTOR" && (
<DateRangeSelector
startDate={startDate}
endDate={endDate}

View file

@ -489,8 +489,8 @@ export function SourceDetailPanel({
>
{idx + 1}
{isCited && (
<span className="absolute -top-1 -right-1 w-3 h-3 bg-primary rounded-full border-2 border-background">
<Sparkles className="h-2 w-2 text-primary-foreground absolute top-0.5 left-0.5" />
<span className="absolute -top-1.5 -right-1.5 flex items-center justify-center w-4 h-4 bg-primary rounded-full border-2 border-background shadow-sm">
<Sparkles className="h-2.5 w-2.5 text-primary-foreground" />
</span>
)}
</motion.button>