mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-04-27 17:56:25 +02:00
feat: add podcast generation capabilities to SurfSense deep agent and UI integration
This commit is contained in:
Anish Sarkar
parent
3906ba52e0
commit
4c4e4b3c4c
9 changed files with 985 additions and 22 deletions
310
surfsense_web/components/tool-ui/audio.tsx
Normal file
310
surfsense_web/components/tool-ui/audio.tsx
Normal file
|
|
@ -0,0 +1,310 @@
|
|||
"use client";
|
||||
|
||||
import { DownloadIcon, PauseIcon, PlayIcon, Volume2Icon, VolumeXIcon } from "lucide-react";
|
||||
import Image from "next/image";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Slider } from "@/components/ui/slider";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
interface AudioProps {
|
||||
id: string;
|
||||
assetId?: string;
|
||||
src: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
artwork?: string;
|
||||
durationMs?: number;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
function formatTime(seconds: number): string {
|
||||
if (!Number.isFinite(seconds) || seconds < 0) return "0:00";
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = Math.floor(seconds % 60);
|
||||
return `${mins}:${secs.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
export function Audio({
|
||||
id,
|
||||
src,
|
||||
title,
|
||||
description,
|
||||
artwork,
|
||||
durationMs,
|
||||
className,
|
||||
}: AudioProps) {
|
||||
const audioRef = useRef<HTMLAudioElement>(null);
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [currentTime, setCurrentTime] = useState(0);
|
||||
const [duration, setDuration] = useState(durationMs ? durationMs / 1000 : 0);
|
||||
const [volume, setVolume] = useState(1);
|
||||
const [isMuted, setIsMuted] = useState(false);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
// Handle play/pause
|
||||
const togglePlayPause = useCallback(() => {
|
||||
const audio = audioRef.current;
|
||||
if (!audio) return;
|
||||
|
||||
if (isPlaying) {
|
||||
audio.pause();
|
||||
} else {
|
||||
audio.play().catch((err) => {
|
||||
console.error("Error playing audio:", err);
|
||||
setError("Failed to play audio");
|
||||
});
|
||||
}
|
||||
}, [isPlaying]);
|
||||
|
||||
// Handle seek
|
||||
const handleSeek = useCallback((value: number[]) => {
|
||||
const audio = audioRef.current;
|
||||
if (!audio || !Number.isFinite(value[0])) return;
|
||||
audio.currentTime = value[0];
|
||||
setCurrentTime(value[0]);
|
||||
}, []);
|
||||
|
||||
// Handle volume change
|
||||
const handleVolumeChange = useCallback((value: number[]) => {
|
||||
const audio = audioRef.current;
|
||||
if (!audio || !Number.isFinite(value[0])) return;
|
||||
const newVolume = value[0];
|
||||
audio.volume = newVolume;
|
||||
setVolume(newVolume);
|
||||
setIsMuted(newVolume === 0);
|
||||
}, []);
|
||||
|
||||
// Toggle mute
|
||||
const toggleMute = useCallback(() => {
|
||||
const audio = audioRef.current;
|
||||
if (!audio) return;
|
||||
|
||||
if (isMuted) {
|
||||
audio.volume = volume || 1;
|
||||
setIsMuted(false);
|
||||
} else {
|
||||
audio.volume = 0;
|
||||
setIsMuted(true);
|
||||
}
|
||||
}, [isMuted, volume]);
|
||||
|
||||
// Handle download
|
||||
const handleDownload = useCallback(async () => {
|
||||
try {
|
||||
const response = await fetch(src);
|
||||
const blob = await response.blob();
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = `${title.replace(/[^a-zA-Z0-9]/g, "_")}.mp3`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
window.URL.revokeObjectURL(url);
|
||||
} catch (err) {
|
||||
console.error("Error downloading audio:", err);
|
||||
}
|
||||
}, [src, title]);
|
||||
|
||||
// Set up audio event listeners
|
||||
useEffect(() => {
|
||||
const audio = audioRef.current;
|
||||
if (!audio) return;
|
||||
|
||||
const handleLoadedMetadata = () => {
|
||||
setDuration(audio.duration);
|
||||
setIsLoading(false);
|
||||
};
|
||||
|
||||
const handleTimeUpdate = () => {
|
||||
setCurrentTime(audio.currentTime);
|
||||
};
|
||||
|
||||
const handlePlay = () => setIsPlaying(true);
|
||||
const handlePause = () => setIsPlaying(false);
|
||||
const handleEnded = () => {
|
||||
setIsPlaying(false);
|
||||
setCurrentTime(0);
|
||||
};
|
||||
const handleError = () => {
|
||||
setError("Failed to load audio");
|
||||
setIsLoading(false);
|
||||
};
|
||||
const handleCanPlay = () => setIsLoading(false);
|
||||
|
||||
audio.addEventListener("loadedmetadata", handleLoadedMetadata);
|
||||
audio.addEventListener("timeupdate", handleTimeUpdate);
|
||||
audio.addEventListener("play", handlePlay);
|
||||
audio.addEventListener("pause", handlePause);
|
||||
audio.addEventListener("ended", handleEnded);
|
||||
audio.addEventListener("error", handleError);
|
||||
audio.addEventListener("canplay", handleCanPlay);
|
||||
|
||||
return () => {
|
||||
audio.removeEventListener("loadedmetadata", handleLoadedMetadata);
|
||||
audio.removeEventListener("timeupdate", handleTimeUpdate);
|
||||
audio.removeEventListener("play", handlePlay);
|
||||
audio.removeEventListener("pause", handlePause);
|
||||
audio.removeEventListener("ended", handleEnded);
|
||||
audio.removeEventListener("error", handleError);
|
||||
audio.removeEventListener("canplay", handleCanPlay);
|
||||
};
|
||||
}, []);
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"flex items-center gap-4 rounded-xl border border-destructive/20 bg-destructive/5 p-4",
|
||||
className,
|
||||
)}
|
||||
>
|
||||
<div className="flex size-16 items-center justify-center rounded-lg bg-destructive/10">
|
||||
<Volume2Icon className="size-8 text-destructive" />
|
||||
</div>
|
||||
<div className="flex-1">
|
||||
<p className="font-medium text-destructive">{title}</p>
|
||||
<p className="text-destructive/70 text-sm">{error}</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
id={id}
|
||||
className={cn(
|
||||
"group relative overflow-hidden rounded-xl border bg-gradient-to-br from-background to-muted/30 p-4 shadow-sm transition-all hover:shadow-md",
|
||||
className,
|
||||
)}
|
||||
>
|
||||
{/* Hidden audio element */}
|
||||
<audio ref={audioRef} src={src} preload="metadata">
|
||||
<track kind="captions" srcLang="en" label="English captions" default />
|
||||
</audio>
|
||||
|
||||
<div className="flex gap-4">
|
||||
{/* Artwork */}
|
||||
<div className="relative shrink-0">
|
||||
<div className="relative size-20 overflow-hidden rounded-lg bg-gradient-to-br from-primary/20 to-primary/5 shadow-inner">
|
||||
{artwork ? (
|
||||
<Image
|
||||
src={artwork}
|
||||
alt={title}
|
||||
fill
|
||||
className="object-cover"
|
||||
unoptimized
|
||||
/>
|
||||
) : (
|
||||
<div className="flex size-full items-center justify-center">
|
||||
<Volume2Icon className="size-8 text-primary/50" />
|
||||
</div>
|
||||
)}
|
||||
{/* Play overlay on artwork */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={togglePlayPause}
|
||||
className="absolute inset-0 flex items-center justify-center bg-black/0 opacity-0 transition-all group-hover:bg-black/30 group-hover:opacity-100"
|
||||
aria-label={isPlaying ? "Pause" : "Play"}
|
||||
>
|
||||
{isPlaying ? (
|
||||
<PauseIcon className="size-8 text-white drop-shadow-lg" />
|
||||
) : (
|
||||
<PlayIcon className="size-8 text-white drop-shadow-lg" />
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Content */}
|
||||
<div className="flex min-w-0 flex-1 flex-col justify-between">
|
||||
{/* Title and description */}
|
||||
<div className="min-w-0">
|
||||
<h3 className="truncate font-semibold text-foreground">{title}</h3>
|
||||
{description && (
|
||||
<p className="mt-0.5 line-clamp-1 text-muted-foreground text-sm">
|
||||
{description}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Progress bar */}
|
||||
<div className="mt-2 space-y-1">
|
||||
<Slider
|
||||
value={[currentTime]}
|
||||
max={duration || 100}
|
||||
step={0.1}
|
||||
onValueChange={handleSeek}
|
||||
className="cursor-pointer"
|
||||
disabled={isLoading}
|
||||
/>
|
||||
<div className="flex justify-between text-muted-foreground text-xs">
|
||||
<span>{formatTime(currentTime)}</span>
|
||||
<span>{formatTime(duration)}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Controls */}
|
||||
<div className="mt-3 flex items-center justify-between border-t pt-3">
|
||||
<div className="flex items-center gap-2">
|
||||
{/* Play/Pause button */}
|
||||
<Button
|
||||
variant="default"
|
||||
size="sm"
|
||||
onClick={togglePlayPause}
|
||||
disabled={isLoading}
|
||||
className="gap-2"
|
||||
>
|
||||
{isLoading ? (
|
||||
<div className="size-4 animate-spin rounded-full border-2 border-current border-t-transparent" />
|
||||
) : isPlaying ? (
|
||||
<PauseIcon className="size-4" />
|
||||
) : (
|
||||
<PlayIcon className="size-4" />
|
||||
)}
|
||||
{isPlaying ? "Pause" : "Play"}
|
||||
</Button>
|
||||
|
||||
{/* Volume control */}
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={toggleMute}
|
||||
className="size-8"
|
||||
>
|
||||
{isMuted ? (
|
||||
<VolumeXIcon className="size-4" />
|
||||
) : (
|
||||
<Volume2Icon className="size-4" />
|
||||
)}
|
||||
</Button>
|
||||
<Slider
|
||||
value={[isMuted ? 0 : volume]}
|
||||
max={1}
|
||||
step={0.01}
|
||||
onValueChange={handleVolumeChange}
|
||||
className="w-20"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Download button */}
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={handleDownload}
|
||||
className="gap-2"
|
||||
>
|
||||
<DownloadIcon className="size-4" />
|
||||
Download
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
288
surfsense_web/components/tool-ui/generate-podcast.tsx
Normal file
288
surfsense_web/components/tool-ui/generate-podcast.tsx
Normal file
|
|
@ -0,0 +1,288 @@
|
|||
"use client";
|
||||
|
||||
import { makeAssistantToolUI } from "@assistant-ui/react";
|
||||
import { AlertCircleIcon, Loader2Icon, MicIcon } from "lucide-react";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { Audio } from "@/components/tool-ui/audio";
|
||||
import { podcastsApiService } from "@/lib/apis/podcasts-api.service";
|
||||
|
||||
/**
|
||||
* Type definitions for the generate_podcast tool
|
||||
*/
|
||||
interface GeneratePodcastArgs {
|
||||
source_content: string;
|
||||
podcast_title?: string;
|
||||
user_prompt?: string;
|
||||
}
|
||||
|
||||
interface GeneratePodcastResult {
|
||||
status: "success" | "error";
|
||||
podcast_id?: number;
|
||||
title?: string;
|
||||
transcript?: string;
|
||||
duration_ms?: number;
|
||||
transcript_entries?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loading state component shown while podcast is being generated
|
||||
*/
|
||||
function PodcastGeneratingState({ title }: { title: string }) {
|
||||
return (
|
||||
<div className="my-4 overflow-hidden rounded-xl border border-primary/20 bg-gradient-to-br from-primary/5 to-primary/10 p-6">
|
||||
<div className="flex items-center gap-4">
|
||||
<div className="relative">
|
||||
<div className="flex size-16 items-center justify-center rounded-full bg-primary/20">
|
||||
<MicIcon className="size-8 text-primary" />
|
||||
</div>
|
||||
{/* Animated rings */}
|
||||
<div className="absolute inset-0 animate-ping rounded-full bg-primary/20" />
|
||||
</div>
|
||||
<div className="flex-1">
|
||||
<h3 className="font-semibold text-foreground text-lg">{title}</h3>
|
||||
<div className="mt-2 flex items-center gap-2 text-muted-foreground">
|
||||
<Loader2Icon className="size-4 animate-spin" />
|
||||
<span className="text-sm">Generating podcast... This may take a few minutes</span>
|
||||
</div>
|
||||
<div className="mt-3">
|
||||
<div className="h-1.5 w-full overflow-hidden rounded-full bg-primary/10">
|
||||
<div className="h-full w-1/3 animate-pulse rounded-full bg-primary" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Error state component shown when podcast generation fails
|
||||
*/
|
||||
function PodcastErrorState({ title, error }: { title: string; error: string }) {
|
||||
return (
|
||||
<div className="my-4 overflow-hidden rounded-xl border border-destructive/20 bg-destructive/5 p-6">
|
||||
<div className="flex items-center gap-4">
|
||||
<div className="flex size-16 shrink-0 items-center justify-center rounded-full bg-destructive/10">
|
||||
<AlertCircleIcon className="size-8 text-destructive" />
|
||||
</div>
|
||||
<div className="flex-1">
|
||||
<h3 className="font-semibold text-foreground">{title}</h3>
|
||||
<p className="mt-1 text-destructive text-sm">Failed to generate podcast</p>
|
||||
<p className="mt-2 text-muted-foreground text-sm">{error}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Audio loading state component
|
||||
*/
|
||||
function AudioLoadingState({ title }: { title: string }) {
|
||||
return (
|
||||
<div className="my-4 overflow-hidden rounded-xl border bg-muted/30 p-6">
|
||||
<div className="flex items-center gap-4">
|
||||
<div className="flex size-16 items-center justify-center rounded-full bg-primary/10">
|
||||
<MicIcon className="size-8 text-primary/50" />
|
||||
</div>
|
||||
<div className="flex-1">
|
||||
<h3 className="font-semibold text-foreground">{title}</h3>
|
||||
<div className="mt-2 flex items-center gap-2 text-muted-foreground">
|
||||
<Loader2Icon className="size-4 animate-spin" />
|
||||
<span className="text-sm">Loading audio...</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Podcast Player Component - Fetches audio with authentication
|
||||
*/
|
||||
function PodcastPlayer({
|
||||
podcastId,
|
||||
title,
|
||||
description,
|
||||
durationMs,
|
||||
transcript,
|
||||
transcriptEntries,
|
||||
}: {
|
||||
podcastId: number;
|
||||
title: string;
|
||||
description: string;
|
||||
durationMs?: number;
|
||||
transcript?: string;
|
||||
transcriptEntries?: number;
|
||||
}) {
|
||||
const [audioSrc, setAudioSrc] = useState<string | null>(null);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const objectUrlRef = useRef<string | null>(null);
|
||||
|
||||
// Cleanup object URL on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (objectUrlRef.current) {
|
||||
URL.revokeObjectURL(objectUrlRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Fetch audio with authentication
|
||||
const loadAudio = useCallback(async () => {
|
||||
setIsLoading(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Revoke previous object URL if exists
|
||||
if (objectUrlRef.current) {
|
||||
URL.revokeObjectURL(objectUrlRef.current);
|
||||
objectUrlRef.current = null;
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 60000); // 60s timeout
|
||||
|
||||
try {
|
||||
// Fetch audio blob with authentication
|
||||
const response = await podcastsApiService.loadPodcast({
|
||||
request: { id: podcastId },
|
||||
controller,
|
||||
});
|
||||
|
||||
// Create object URL from blob
|
||||
const objectUrl = URL.createObjectURL(response);
|
||||
objectUrlRef.current = objectUrl;
|
||||
setAudioSrc(objectUrl);
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error loading podcast audio:", err);
|
||||
if (err instanceof DOMException && err.name === "AbortError") {
|
||||
setError("Request timed out. Please try again.");
|
||||
} else {
|
||||
setError(err instanceof Error ? err.message : "Failed to load audio");
|
||||
}
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}, [podcastId]);
|
||||
|
||||
// Load audio when component mounts
|
||||
useEffect(() => {
|
||||
loadAudio();
|
||||
}, [loadAudio]);
|
||||
|
||||
if (isLoading) {
|
||||
return <AudioLoadingState title={title} />;
|
||||
}
|
||||
|
||||
if (error || !audioSrc) {
|
||||
return <PodcastErrorState title={title} error={error || "Failed to load audio"} />;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="my-4">
|
||||
<Audio
|
||||
id={`podcast-${podcastId}`}
|
||||
src={audioSrc}
|
||||
title={title}
|
||||
description={description}
|
||||
durationMs={durationMs}
|
||||
className="w-full"
|
||||
/>
|
||||
{/* Full transcript */}
|
||||
{transcript && (
|
||||
<details className="mt-3 rounded-lg border bg-muted/30 p-3">
|
||||
<summary className="cursor-pointer font-medium text-muted-foreground text-sm hover:text-foreground">
|
||||
View full transcript{transcriptEntries ? ` (${transcriptEntries} entries)` : ""}
|
||||
</summary>
|
||||
<pre className="mt-2 max-h-96 overflow-y-auto whitespace-pre-wrap text-muted-foreground text-xs">
|
||||
{transcript}
|
||||
</pre>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Podcast Tool UI Component
|
||||
*
|
||||
* This component is registered with assistant-ui to render custom UI
|
||||
* when the generate_podcast tool is called by the agent.
|
||||
*
|
||||
* It fetches the podcast audio with authentication (like the old system)
|
||||
* and displays it using the Audio component.
|
||||
*/
|
||||
export const GeneratePodcastToolUI = makeAssistantToolUI<
|
||||
GeneratePodcastArgs,
|
||||
GeneratePodcastResult
|
||||
>({
|
||||
toolName: "generate_podcast",
|
||||
render: function GeneratePodcastUI({ args, result, status }) {
|
||||
const title = args.podcast_title || "SurfSense Podcast";
|
||||
|
||||
// Loading state - podcast is being generated
|
||||
if (status.type === "running" || status.type === "requires-action") {
|
||||
return <PodcastGeneratingState title={title} />;
|
||||
}
|
||||
|
||||
// Incomplete/cancelled state
|
||||
if (status.type === "incomplete") {
|
||||
if (status.reason === "cancelled") {
|
||||
return (
|
||||
<div className="my-4 rounded-xl border border-muted p-4 text-muted-foreground">
|
||||
<p className="flex items-center gap-2">
|
||||
<MicIcon className="size-4" />
|
||||
<span className="line-through">Podcast generation cancelled</span>
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (status.reason === "error") {
|
||||
return (
|
||||
<PodcastErrorState
|
||||
title={title}
|
||||
error={typeof status.error === "string" ? status.error : "An error occurred"}
|
||||
/>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// No result yet
|
||||
if (!result) {
|
||||
return <PodcastGeneratingState title={title} />;
|
||||
}
|
||||
|
||||
// Error result
|
||||
if (result.status === "error") {
|
||||
return <PodcastErrorState title={title} error={result.error || "Unknown error"} />;
|
||||
}
|
||||
|
||||
// Success - need podcast_id to fetch with auth
|
||||
if (!result.podcast_id) {
|
||||
return <PodcastErrorState title={title} error="Missing podcast ID" />;
|
||||
}
|
||||
|
||||
// Render the podcast player (handles auth fetch internally)
|
||||
return (
|
||||
<PodcastPlayer
|
||||
podcastId={result.podcast_id}
|
||||
title={result.title || title}
|
||||
description={
|
||||
result.transcript_entries
|
||||
? `${result.transcript_entries} dialogue entries`
|
||||
: "SurfSense AI-generated podcast"
|
||||
}
|
||||
durationMs={result.duration_ms}
|
||||
transcript={result.transcript}
|
||||
transcriptEntries={result.transcript_entries}
|
||||
/>
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
11
surfsense_web/components/tool-ui/index.ts
Normal file
11
surfsense_web/components/tool-ui/index.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
/**
|
||||
* Tool UI Components
|
||||
*
|
||||
* This module exports custom UI components for assistant tools.
|
||||
* These components are registered with assistant-ui to render
|
||||
* rich UI when specific tools are called by the agent.
|
||||
*/
|
||||
|
||||
export { Audio } from "./audio";
|
||||
export { GeneratePodcastToolUI } from "./generate-podcast";
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue