mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-04-26 17:06:23 +02:00
feat: voice notes with instant transcription and knowledge graph integration
- Voice memos now create notes immediately in knowledge/Voice Memos/<date>/ - Transcription shows directly in the note (Recording... → Transcribing... → transcript) - Graph builder processes voice memos from knowledge directory - Note creation agents now use workspace tools (writeFile, edit, grep, glob) instead of executeCommand - Removes executeCommand dependency - no more permission prompts blocking knowledge updates Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
3e2ed4cbc4
commit
d7b84f87d0
6 changed files with 720 additions and 142 deletions
|
|
@ -1461,6 +1461,29 @@ function AppContent({ auth }: { auth: ReturnType<typeof useRowboatAuth> }) {
|
|||
},
|
||||
}), [tree, selectedPath, workspaceRoot, collectDirPaths])
|
||||
|
||||
// Handler for when a voice note is created/updated
|
||||
const handleVoiceNoteCreated = useCallback(async (notePath: string) => {
|
||||
// Refresh the tree to show the new file/folder
|
||||
const newTree = await loadDirectory()
|
||||
setTree(newTree)
|
||||
|
||||
// Expand parent directories to show the file
|
||||
const parts = notePath.split('/')
|
||||
const parentPaths: string[] = []
|
||||
for (let i = 1; i < parts.length; i++) {
|
||||
parentPaths.push(parts.slice(0, i).join('/'))
|
||||
}
|
||||
setExpandedPaths(prev => {
|
||||
const newSet = new Set(prev)
|
||||
parentPaths.forEach(p => newSet.add(p))
|
||||
return newSet
|
||||
})
|
||||
|
||||
// Select the file to show it in the editor
|
||||
setIsGraphOpen(false)
|
||||
setSelectedPath(notePath)
|
||||
}, [loadDirectory])
|
||||
|
||||
const ensureWikiFile = useCallback(async (wikiPath: string) => {
|
||||
const resolvedPath = toKnowledgePath(wikiPath)
|
||||
if (!resolvedPath) return null
|
||||
|
|
@ -1707,6 +1730,7 @@ function AppContent({ auth }: { auth: ReturnType<typeof useRowboatAuth> }) {
|
|||
expandedPaths={expandedPaths}
|
||||
onSelectFile={toggleExpand}
|
||||
knowledgeActions={knowledgeActions}
|
||||
onVoiceNoteCreated={handleVoiceNoteCreated}
|
||||
runs={runs}
|
||||
currentRunId={runId}
|
||||
tasksActions={{
|
||||
|
|
|
|||
|
|
@ -12,8 +12,10 @@ import {
|
|||
Folder,
|
||||
FolderPlus,
|
||||
MessageSquare,
|
||||
Mic,
|
||||
Network,
|
||||
Pencil,
|
||||
Square,
|
||||
SquarePen,
|
||||
Trash2,
|
||||
} from "lucide-react"
|
||||
|
|
@ -88,6 +90,7 @@ type SidebarContentPanelProps = {
|
|||
expandedPaths: Set<string>
|
||||
onSelectFile: (path: string, kind: "file" | "dir") => void
|
||||
knowledgeActions: KnowledgeActions
|
||||
onVoiceNoteCreated?: (path: string) => void
|
||||
runs?: RunListItem[]
|
||||
currentRunId?: string | null
|
||||
tasksActions?: TasksActions
|
||||
|
|
@ -104,6 +107,7 @@ export function SidebarContentPanel({
|
|||
expandedPaths,
|
||||
onSelectFile,
|
||||
knowledgeActions,
|
||||
onVoiceNoteCreated,
|
||||
runs = [],
|
||||
currentRunId,
|
||||
tasksActions,
|
||||
|
|
@ -126,6 +130,7 @@ export function SidebarContentPanel({
|
|||
expandedPaths={expandedPaths}
|
||||
onSelectFile={onSelectFile}
|
||||
actions={knowledgeActions}
|
||||
onVoiceNoteCreated={onVoiceNoteCreated}
|
||||
/>
|
||||
)}
|
||||
{activeSection === "tasks" && (
|
||||
|
|
@ -141,6 +146,227 @@ export function SidebarContentPanel({
|
|||
)
|
||||
}
|
||||
|
||||
async function transcribeWithDeepgram(audioBlob: Blob): Promise<string | null> {
|
||||
try {
|
||||
const configResult = await window.ipc.invoke('workspace:readFile', {
|
||||
path: 'config/deepgram.json',
|
||||
encoding: 'utf8',
|
||||
})
|
||||
const { apiKey } = JSON.parse(configResult.data) as { apiKey: string }
|
||||
if (!apiKey) throw new Error('No apiKey in deepgram.json')
|
||||
|
||||
const response = await fetch(
|
||||
'https://api.deepgram.com/v1/listen?model=nova-2&smart_format=true',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Token ${apiKey}`,
|
||||
'Content-Type': audioBlob.type,
|
||||
},
|
||||
body: audioBlob,
|
||||
},
|
||||
)
|
||||
|
||||
if (!response.ok) throw new Error(`Deepgram API error: ${response.status}`)
|
||||
const result = await response.json()
|
||||
return result.results?.channels?.[0]?.alternatives?.[0]?.transcript ?? null
|
||||
} catch (err) {
|
||||
console.error('Deepgram transcription failed:', err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// Voice Note Recording Button
|
||||
function VoiceNoteButton({ onNoteCreated }: { onNoteCreated?: (path: string) => void }) {
|
||||
const [isRecording, setIsRecording] = React.useState(false)
|
||||
const mediaRecorderRef = React.useRef<MediaRecorder | null>(null)
|
||||
const chunksRef = React.useRef<Blob[]>([])
|
||||
const notePathRef = React.useRef<string | null>(null)
|
||||
const timestampRef = React.useRef<string | null>(null)
|
||||
const relativePathRef = React.useRef<string | null>(null)
|
||||
|
||||
const startRecording = async () => {
|
||||
try {
|
||||
// Generate timestamp and paths immediately
|
||||
const now = new Date()
|
||||
const timestamp = now.toISOString().replace(/[:.]/g, '-')
|
||||
const dateStr = now.toISOString().split('T')[0] // YYYY-MM-DD
|
||||
const noteName = `voice-memo-${timestamp}`
|
||||
const notePath = `knowledge/Voice Memos/${dateStr}/${noteName}.md`
|
||||
|
||||
timestampRef.current = timestamp
|
||||
notePathRef.current = notePath
|
||||
// Relative path for linking (from knowledge/ root, without .md extension)
|
||||
const relativePath = `Voice Memos/${dateStr}/${noteName}`
|
||||
relativePathRef.current = relativePath
|
||||
|
||||
// Create the note immediately with a "Recording..." placeholder
|
||||
await window.ipc.invoke('workspace:mkdir', {
|
||||
path: `knowledge/Voice Memos/${dateStr}`,
|
||||
recursive: true,
|
||||
})
|
||||
|
||||
const initialContent = `# Voice Memo
|
||||
|
||||
**Type:** voice memo
|
||||
**Recorded:** ${now.toLocaleString()}
|
||||
**Path:** ${relativePath}
|
||||
|
||||
## Transcript
|
||||
|
||||
*Recording in progress...*
|
||||
`
|
||||
await window.ipc.invoke('workspace:writeFile', {
|
||||
path: notePath,
|
||||
data: initialContent,
|
||||
opts: { encoding: 'utf8' },
|
||||
})
|
||||
|
||||
// Select the note so the user can see it
|
||||
onNoteCreated?.(notePath)
|
||||
|
||||
// Start actual recording
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
const mimeType = MediaRecorder.isTypeSupported('audio/mp4')
|
||||
? 'audio/mp4'
|
||||
: 'audio/webm'
|
||||
const recorder = new MediaRecorder(stream, { mimeType })
|
||||
chunksRef.current = []
|
||||
|
||||
recorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) chunksRef.current.push(e.data)
|
||||
}
|
||||
|
||||
recorder.onstop = async () => {
|
||||
stream.getTracks().forEach((t) => t.stop())
|
||||
const blob = new Blob(chunksRef.current, { type: mimeType })
|
||||
const ext = mimeType === 'audio/mp4' ? 'm4a' : 'webm'
|
||||
const audioFilename = `voice-memo-${timestampRef.current}.${ext}`
|
||||
|
||||
// Save audio file to voice_memos folder (for backup/reference)
|
||||
try {
|
||||
await window.ipc.invoke('workspace:mkdir', {
|
||||
path: 'voice_memos',
|
||||
recursive: true,
|
||||
})
|
||||
|
||||
const arrayBuffer = await blob.arrayBuffer()
|
||||
const base64 = btoa(
|
||||
new Uint8Array(arrayBuffer).reduce(
|
||||
(data, byte) => data + String.fromCharCode(byte),
|
||||
'',
|
||||
),
|
||||
)
|
||||
|
||||
await window.ipc.invoke('workspace:writeFile', {
|
||||
path: `voice_memos/${audioFilename}`,
|
||||
data: base64,
|
||||
opts: { encoding: 'base64' },
|
||||
})
|
||||
} catch {
|
||||
console.error('Failed to save audio file')
|
||||
}
|
||||
|
||||
// Update note to show transcribing status
|
||||
const currentNotePath = notePathRef.current
|
||||
const currentRelativePath = relativePathRef.current
|
||||
if (currentNotePath && currentRelativePath) {
|
||||
const transcribingContent = `# Voice Memo
|
||||
|
||||
**Type:** voice memo
|
||||
**Recorded:** ${new Date().toLocaleString()}
|
||||
**Path:** ${currentRelativePath}
|
||||
|
||||
## Transcript
|
||||
|
||||
*Transcribing...*
|
||||
`
|
||||
await window.ipc.invoke('workspace:writeFile', {
|
||||
path: currentNotePath,
|
||||
data: transcribingContent,
|
||||
opts: { encoding: 'utf8' },
|
||||
})
|
||||
}
|
||||
|
||||
// Transcribe and update the note with the transcript
|
||||
const transcript = await transcribeWithDeepgram(blob)
|
||||
if (currentNotePath && currentRelativePath) {
|
||||
const finalContent = transcript
|
||||
? `# Voice Memo
|
||||
|
||||
**Type:** voice memo
|
||||
**Recorded:** ${new Date().toLocaleString()}
|
||||
**Path:** ${currentRelativePath}
|
||||
|
||||
## Transcript
|
||||
|
||||
${transcript}
|
||||
`
|
||||
: `# Voice Memo
|
||||
|
||||
**Type:** voice memo
|
||||
**Recorded:** ${new Date().toLocaleString()}
|
||||
**Path:** ${currentRelativePath}
|
||||
|
||||
## Transcript
|
||||
|
||||
*Transcription failed. Please try again.*
|
||||
`
|
||||
await window.ipc.invoke('workspace:writeFile', {
|
||||
path: currentNotePath,
|
||||
data: finalContent,
|
||||
opts: { encoding: 'utf8' },
|
||||
})
|
||||
|
||||
// Re-select to trigger refresh
|
||||
onNoteCreated?.(currentNotePath)
|
||||
|
||||
if (transcript) {
|
||||
toast('Voice note transcribed', 'success')
|
||||
} else {
|
||||
toast('Transcription failed', 'error')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
recorder.start()
|
||||
mediaRecorderRef.current = recorder
|
||||
setIsRecording(true)
|
||||
toast('Recording started', 'success')
|
||||
} catch {
|
||||
toast('Could not access microphone', 'error')
|
||||
}
|
||||
}
|
||||
|
||||
const stopRecording = () => {
|
||||
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
|
||||
mediaRecorderRef.current.stop()
|
||||
}
|
||||
mediaRecorderRef.current = null
|
||||
setIsRecording(false)
|
||||
}
|
||||
|
||||
return (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
onClick={isRecording ? stopRecording : startRecording}
|
||||
className="text-sidebar-foreground/70 hover:text-sidebar-foreground hover:bg-sidebar-accent rounded p-1.5 transition-colors"
|
||||
>
|
||||
{isRecording ? (
|
||||
<Square className="size-4 fill-red-500 text-red-500 animate-pulse" />
|
||||
) : (
|
||||
<Mic className="size-4" />
|
||||
)}
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom">
|
||||
{isRecording ? 'Stop Recording' : 'New Voice Note'}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)
|
||||
}
|
||||
|
||||
// Knowledge Section
|
||||
function KnowledgeSection({
|
||||
tree,
|
||||
|
|
@ -148,15 +374,17 @@ function KnowledgeSection({
|
|||
expandedPaths,
|
||||
onSelectFile,
|
||||
actions,
|
||||
onVoiceNoteCreated,
|
||||
}: {
|
||||
tree: TreeNode[]
|
||||
selectedPath: string | null
|
||||
expandedPaths: Set<string>
|
||||
onSelectFile: (path: string, kind: "file" | "dir") => void
|
||||
actions: KnowledgeActions
|
||||
onVoiceNoteCreated?: (path: string) => void
|
||||
}) {
|
||||
const isExpanded = expandedPaths.size > 0
|
||||
|
||||
|
||||
const quickActions = [
|
||||
{ icon: FilePlus, label: "New Note", action: () => actions.createNote() },
|
||||
{ icon: FolderPlus, label: "New Folder", action: () => actions.createFolder() },
|
||||
|
|
@ -181,6 +409,7 @@ function KnowledgeSection({
|
|||
<TooltipContent side="bottom">{action.label}</TooltipContent>
|
||||
</Tooltip>
|
||||
))}
|
||||
<VoiceNoteButton onNoteCreated={onVoiceNoteCreated} />
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue